'Image stores' to enable better image caching

assets/style.css

@@ -1,5 +1,5 @@
 body {
-  max-width: 640px;
+  max-width: 720px;
   margin: 0 0 0 auto;
   padding: 8px;
   font-family: sans-serif;
@@ -33,7 +33,8 @@ main li b {
 }
 
 img {
-  margin: 10px auto;
+  margin: 10px 0 10px 50%;
+  transform: translateX(-50%);
   max-width: 100%;
 }
 
@@ -51,7 +52,7 @@ hr {
 }
 
 footer a {
-  padding-bottom: 30vh;
+  padding-bottom: 3in;
 }
 
 @media (prefers-color-scheme: dark) {

default/cache.json

@@ -1,3 +0,0 @@
-{
-  "nitter": {}
-}

lib.js

@@ -3,7 +3,6 @@
 import Path from "path"
 import FS from "fs/promises"
 import { JSDOM } from "jsdom"
-import mime from "mime-types"
 
 
 //     |   o |
@@ -40,14 +39,14 @@ export const write = async (path, content) => {
 }
 
 export const createNetworkingError = response => {
-  return new Error(`Request failed, ${response.status}: ${response.statusText}`)
+  return new Error(`Request failed for ${response.url}, ${response.status}: ${response.statusText}`)
 }
 
 export const getLinkExtname = link =>
   Path.extname(new URL(link).pathname)
 
 export const getImageBasePath = (source, postId) =>
-  `images/${source.name}-${postId}`
+  `${source.name}-${postId}`
 
 export const writeStylesheet = (path, { directory, batch }) =>
   batch.add(
@@ -67,12 +66,33 @@ export const testBlacklist = (array, blacklist) =>
   blacklist.find(tag => array.includes(tag)) !== undefined
 
 export const createView = (directory, pageSize, extra = {}) => {
-  return {
+  let view = {
     batch: new PromiseBatch(),
     directory,
     pageSize,
     ...extra
   }
+
+  return view
+}
+
+export const openImageStore = async view => {
+  let imageStorePath = Path.join(view.directory, view.imageStoreDirectory)
+  let dirents = await FS.readdir(imageStorePath, { withFileTypes: true })
+  view.imageStore = new Map()
+
+  for(let dirent of dirents) {
+    if(dirent.isFile()) {
+      let basename = dirent.name.slice(0, dirent.name.lastIndexOf('.'))
+      view.imageStore.set(basename, Path.join(view.imageStoreDirectory, dirent.name))
+    }
+  }
+
+  return view
+}
+
+export const isUnset = (value) => {
+  return typeof value === "undefined" || value === null
 }
 
 //
@@ -80,17 +100,18 @@ export const createView = (directory, pageSize, extra = {}) => {
 // |   `-. `-.
 // '   `-' `-'
 
-export async function fetchRss(source) {
+export async function fetchChannel(source) {
   let { hostname } = source
   let error
   let response
   let rss
+  let channel
 
   try {
     response = await delayedFetch(
       new URL(source.pathname, 'https://' + hostname),
       {},
-      source.courtesyWait ?? 5 * 1000
+      source.courtesyWait
     )
   } catch(err) {
     error = err
@@ -99,12 +120,24 @@ export async function fetchRss(source) {
   source.errored = error !== undefined || !response.ok
   if(source.errored) {
     source.error = error ?? createNetworkingError(response)
-  } else {
+    return
-    rss = await response.text()
+  }
-    console.log(`Found ${source.name} at ${hostname}`)
+  
+  console.log(`Found ${source.name} at ${hostname}`)
+  
+  try {
+    channel = createChannel(await response.text())
+  } catch(err) {
+    error = err
   }
 
-  return rss
+  source.errored = error !== undefined
+  if(source.errored) {
+    source.error = error
+    return
+  }
+
+  return channel
 }
 
 let waitingList = new Map()
@@ -115,17 +148,33 @@ export const delayedFetch = async (url, options, courtesyWait = 5 * 1000) => {
 
   waitingList.set(domain, waitFor + courtesyWait)
   if(waitFor !== 0) {
+    console.log(`Waiting ${waitFor}ms to download ${url}`)
     await sleep(waitFor)
   }
 
   return await fetch(url, options)
 }
 
+export const retryDelayedFetch = async (url, options, courtesyWait, retryAttempts) => {
+  let attemptsTried = 0
+  let response = undefined
+
+  while(isUnset(response) && attemptsTried <= (retryAttempts ?? 3)) {
+    if(attemptsTried > 0)
+      console.error(`Failed to fetch ${url}, retrying...`)
+
+    response = await delayedFetch(url, options, courtesyWait)
+    attemptsTried++
+  }
+
+  return response
+}
+
 export const getCacheFilename = (source) =>
   source.name + '.xml'
 
-export const getCachePath = (source, { directory }) =>
+export const getCachePath = (source, cache) =>
-  Path.join(directory, getCacheFilename(source))
+  Path.join(cache.directory, getCacheFilename(source))
 
 export const cacheSource = (source, cache) =>
   write(getCachePath(source, cache), renderCache(source, cache))
@@ -149,19 +198,21 @@ export const openCache = async (source, cache) => {
   if(exists)
     rss = await FS.readFile(path, { encoding: 'utf8' })
 
-  if(exists & rss) {
+  if(exists && rss) {
+    // if(source.user == 'nanoraptor') {
+    //   source.asdf = 'b'
+    //   source.cache.asdf = 'b'
+    // }
     let channel = createChannel(rss)
-    let date = readPubDate(channel.querySelector('pubDate'))
-    let link = new URL(channel.querySelector('link').textContent)
 
     source.cache = {
       channel,
-      date,
+      date: readPubDate(channel.querySelector('pubDate')),
-      link
+      link: new URL(channel.querySelector('link').textContent),
     }
   } else {
     source.cache = {
-      date: new Date(0),
+      date: new Date(0)
     }
 
     if(source.hostname)
@@ -176,6 +227,7 @@ export const openCache = async (source, cache) => {
 export const buildCacheLink = source =>
   new URL('https://' + source.hostname)
 
+// .replaceAll(/\n\s*/g, '')
 export const renderCache = (source, cache) => `\
 <?xml version="1.0" encoding="UTF-8" ?>
 <rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
@@ -206,32 +258,39 @@ export const readPubDate = (pubDate) =>
 export const createPosts = async (channel, source, fromDate, reducerCallback) => {
   let items = channel.querySelectorAll('item')
 
-  // if(items.length === 0) {
-  //   // throw new NoMatchesError('Got no matches')
-  //   return source
-  // }
-
   let promises = []
 
   for(let item of items) {
+    let post = createPost(item, source)
+
+    if(post.date <= fromDate)
+      continue
+
     source.items.push(item)
-    let promise = createPost(item, source, reducerCallback)
-      .then(post => {
-        if(post && post.date > fromDate) {
-          source.posts.push(post)
-        }
 
-        return post
+    let postResolvable = reducerCallback(post)
-      })
+    
+    if(postResolvable instanceof Promise) {
+      postResolvable
+        .then(post => {
+          if(post) {
+            source.posts.push(post)
+          }
+        })
+    } else {
+      if(postResolvable) {
+        source.posts.push(postResolvable)
+      }
+    }
 
-    promises.push(promise)
+    promises.push(postResolvable)
   }
 
   await Promise.all(promises)
   return source
 }
 
-export const createPost = async (item, source, reducerCallback) => {
+export const createPost = (item, source) => {
   let description = new JSDOM(item.querySelector('description').textContent).window.document
   let date = readPubDate(item.querySelector('pubDate'))
   let link = item.querySelector('link').textContent
@@ -249,7 +308,7 @@ export const createPost = async (item, source, reducerCallback) => {
     occurances: []
   }
   
-  return await reducerCallback(post)
+  return post
 }
 
 export const processCategories = (post) => {
@@ -263,7 +322,7 @@ export const processCategories = (post) => {
   return post
 }
 
-export const extractImages = (post, cache = true) => {
+export const extractImages = (post) => {
   let images = post.description.querySelectorAll('img')
 
   if(images) {
@@ -272,7 +331,7 @@ export const extractImages = (post, cache = true) => {
     for(let image of images) {
       let { src } = image
 
-      if(!src) {
+      if(isUnset(src)) {
         let finalSrc = image.srcset.split(', ').pop()
 
         src = finalSrc.slice(0, finalSrc.indexOf(' ') )
@@ -403,7 +462,7 @@ export const renderPostDetail = (name, value) =>
 
 export const renderImage = href => {
   return `\
-<a href="${href}" download><img src="${href}" loading="lazy"></img></a>`
+<a href="${href}"><img src="${href}" loading="lazy"></img></a>`
 }
 
 export const renderDate = date =>
@@ -479,21 +538,36 @@ export const renderNavEntry = (list) => {
 // |   | | | | | | |   | | | | |
 // `-' `-' ' ' `-` `-' ' `-' ' '
 
-export const downloadImage = async (url, basename, courtesyWait, { batch, directory }) => {
+export const downloadImage = async (url, basename, { courtesyWait, retryAttempts }, { batch, directory, imageStoreDirectory }) => {
-  let response = await delayedFetch(url, {}, courtesyWait)
+  let response = await retryDelayedFetch(url, {}, courtesyWait, retryAttempts)
     .catch(err => console.error(`Failed download of ${url}:`, err, err.errors) )
 
+  if(response == undefined) {
+    console.error('Could not download image: ' + url)
+    return url
+  }
+
   if(response.ok) {
-    let relativePath = basename + imageExtensions[response.headers.get('Content-Type')]
+    let mimetype = response.headers.get('Content-Type').split(';')[0]
-    let path = Path.join(directory, relativePath)
+    let extension = imageExtensions[mimetype]
+    
+    if(typeof extension !== 'string') {
+      console.error(`Unknown mimetype for ${url}: ${mimetype}. Cannot download`)
+      return url
+    }
+
+    let relativePath = basename + extension
+    let path = Path.join(directory, imageStoreDirectory, relativePath)
 
     const download = () => write(path, response.body)
       .then(annotate( `Downloaded ${relativePath}`))
 
+    // TODO: See if the image is downloaded before even trying to download it
     batch.add(FS.access(path).catch(download))
     return relativePath
   } else {
-    throw createNetworkingError(response)
+    console.error( createNetworkingError(response) )
+    return url
   }
 }
 
@@ -503,8 +577,13 @@ export const downloadImages = (images, source, postId, view) => {
 
   for(let i = 0; i < images.length; i++) {
     let basename = images.length > 1 ? basePath + '-' + i : basePath
+    let pathname = view.imageStore.get(basename)
 
-    pathnames.push(downloadImage(images[i], basename, source.courtesyWait, view))
+    if(isUnset(pathname)) {
+      pathname = downloadImage(images[i], basename, source, view)
+    }
+
+    pathnames.push(pathname)
   }
   
   return Promise.all(pathnames)
@@ -546,19 +625,19 @@ export const createFeed = (name, sources, main = false) => {
   }
 }
 
-export const fetchRssFromInstances = async (source) => {
+export const fetchChannelFromInstances = async (source) => {
   let index = 0
   let instances = source.instances
   let cachedLink = source.cache.link
-  let rss
+  let channel
 
   if(cachedLink) {
     instances.unshift(cachedLink.hostname)
   }
 
-  while(!rss && index != instances.length) {
+  while(!channel && index != instances.length) {
     source.hostname = instances[index]
-    rss = await fetchRss(source)
+    channel = await fetchChannel(source)
 
     if(source.errored) {
       console.error(`Failed to fetch ${source.name} from ${source.hostname}: `, source.error)
@@ -568,15 +647,15 @@ export const fetchRssFromInstances = async (source) => {
     }
   }
 
-  return rss
+  return channel
 }
 
-export const populateSource = async (rss, source, postReducerCallback, useCache = true) => {
+export const populateSource = async (channel, source, postReducerCallback, cache) => {
   let fromDate = 0
   source.items = []
   source.posts = []
 
-  if(useCache) {
+  if(cache.enabled) {
     fromDate = source.latestPostDate
 
     if(source.cache.channel)
@@ -590,8 +669,8 @@ export const populateSource = async (rss, source, postReducerCallback, useCache
     return postReducerCallback(post)
   }
 
-  if(rss ?? false)
+  if(channel ?? false)
-    source = await createPosts(createChannel(rss), source, fromDate, remoteReducerCallback)
+    source = await createPosts(channel, source, fromDate, remoteReducerCallback)
 
   return source
 }
@@ -615,14 +694,26 @@ export const writeView = (sources, feeds, view) => {
   writeStylesheet(Path.join(import.meta.dirname, 'assets/style.css'), view)
 }
 
-export const createSource = async (source, getRss, postReducerCallback, cache) => {
+export const createSource = async (source, getChannel, postReducerCallback, cache) => {
-  source = await openCache(source, cache)
+  if(cache.enabled)
-  source = await populateSource(await getRss(source), source, postReducerCallback, cache.populate)
+    source = await openCache(source, cache)
+  source = await populateSource(await getChannel(source), source, postReducerCallback, cache)
 
-  cache.batch.add(cacheSource(source, cache))
+  if(cache.enabled)
+    cache.batch.add(cacheSource(source, cache))
   return source
 }
 
+export const createSourceOptions = (options, view) => {
+  if(isUnset(options.courtesyWait))
+    options.courtesyWait = 1000
+
+  if(isUnset(options.retryAttempts))
+    options.retryAttempts = 3
+
+  return options
+}
+
 //     |     |    ,-
 // ;-. | ,-: |-   |  ,-. ;-. ;-.-. ,-.
 // | | | | | |    |- | | |   | | | `-.
@@ -630,20 +721,20 @@ export const createSource = async (source, getRss, postReducerCallback, cache) =
 // '             -'
 
 export const tumblr = {
-  createSource(user, courtesyWait, postReducerCallback, cache) {
+  createSource(user, options, postReducerCallback, cache) {
     let lowercaseUser = user.toLowerCase()
     let source = {
       type: 'tumblr',
       description: `Aggregate feed for @${lowercaseUser} on tumblr.com`,
       hostname: lowercaseUser + '.tumblr.com',
       pathname: 'rss',
-      courtesyWait,
       name: `tumblr-${lowercaseUser}`,
       displayName: user,
       user: lowercaseUser,
+      ...createSourceOptions(options)
     }
 
-    return createSource(source, fetchRss, postReducerCallback, cache)
+    return createSource(source, fetchChannel, postReducerCallback, cache)
   },
 
   createSources(users, ...args) {
@@ -672,19 +763,19 @@ export const tumblr = {
 }
 
 export const nitter = {
-  createSource(user, instances, courtesyWait, postReducerCallback, cache) {
+  createSource(user, options, instances, postReducerCallback, cache) {
     let source = {
       type: 'nitter',
       description: `Aggregate feed for @${user} on twitter.com`,
       instances,
       pathname: user + '/rss',
-      courtesyWait,
       name: `nitter-${user}`,
       displayName: user,
-      user
+      user,
+      ...createSourceOptions(options)
     }
 
-    return createSource(source, fetchRssFromInstances, postReducerCallback, cache)
+    return createSource(source, fetchChannelFromInstances, postReducerCallback, cache)
   },
 
   createSources(users, ...args) {
@@ -697,17 +788,36 @@ export const nitter = {
     return creator.innerHTML.slice(1) !== post.source.user
   },
 
-  pullImages
+  async pullImages (post, view, imageMirrorDomain, discardPostIfNoImages = false, getPostId = getPostIdFromPathname) {
+    let images = extractImages(post)
+    let mirroredImages = []
+    const mirrorImage = nitter.createImageMirrorer(post, imageMirrorDomain)
+
+    if(!discardPostIfNoImages || images.length > 0) {
+      post.images = await downloadImages(
+        images.map(mirrorImage),
+        post.source,
+        getPostId(post),
+        view
+      )
+      return post
+    }
+  },
+
+  createImageMirrorer(post, imageMirrorDomain) {
+    let mirrorUrl = new URL(imageMirrorDomain)
+    let basePathname = new URL(post.guid).pathname
+
+    return (image, index, images) => {
+      mirrorUrl.pathname = Path.join(basePathname, 'photo', (index + 1).toString())
+
+      return mirrorUrl.href
+    }
+  }
 }
 
-// TODO: Mastodon support
-//
-// "Turns out Mastodon has built-in RSS; your feed URL is [instance]/@[username].rss, so for example I'm
-// https://mastodon.social/@brownpau.rss (note the "@")"
-// - https://mastodon.social/@brownpau/100523448408374430
-
 export const mastodon = {
-  createSource(usertag, courtesyWait, postReducerCallback, cache) {
+  createSource(usertag, options, postReducerCallback, cache) {
     let [ user, hostname ] = usertag.toLowerCase().split('@')
     
     let source = {
@@ -715,13 +825,13 @@ export const mastodon = {
       description: `Aggregate feed for @${user} at ${hostname}`,
       hostname,
       pathname: '@' + user + ".rss",
-      courtesyWait,
       name: `${hostname}-${user}`,
       displayName: user,
       user,
+      ...createSourceOptions(options)
     }
 
-    return createSource(source, fetchRss, postReducerCallback, cache)
+    return createSource(source, fetchChannel, postReducerCallback, cache)
   },
 
   isRepost(post) {
@@ -747,4 +857,4 @@ export const mastodon = {
       return post
     }
   }
-}
+}

package.json

@@ -10,8 +10,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "jsdom": "^22.1.0",
+    "jsdom": "^22.1.0"
-    "mime-types": "^2.1.35"
   },
   "type": "module"
 }

shell.nix

@@ -1,13 +0,0 @@
-{ pkgs ? import <nixpkgs> {} }:
-
-pkgs.stdenv.mkDerivation {
-    name = "rssssing-dev";
-    buildInputs = [
-        pkgs.nodejs_21
-        pkgs.yarn
-    ];
-    shellHook = ''
-        export PATH="$PWD/node_modules/.bin/:$PATH"
-        export NPM_PACKAGES="$HOME/.npm-packages"
-    '';
-}