Fix delayed fetching, retry requests, image mirroring for nitter, better RSS error handling

lib.js

@@ -3,7 +3,6 @@
 import Path from "path"
 import FS from "fs/promises"
 import { JSDOM } from "jsdom"
-import mime from "mime-types"
 
 
 //     |   o |
@@ -40,7 +39,7 @@ export const write = async (path, content) => {
 }
 
 export const createNetworkingError = response => {
-  return new Error(`Request failed, ${response.status}: ${response.statusText}`)
+  return new Error(`Request failed for ${response.url}, ${response.status}: ${response.statusText}`)
 }
 
 export const getLinkExtname = link =>
@@ -75,22 +74,27 @@ export const createView = (directory, pageSize, extra = {}) => {
   }
 }
 
+export const isUnset = (value) => {
+  return typeof value === "undefined" || value === null
+}
+
 //
 // ;-. ,-. ,-.
 // |   `-. `-.
 // '   `-' `-'
 
-export async function fetchRss(source) {
+export async function fetchChannel(source) {
   let { hostname } = source
   let error
   let response
   let rss
+  let channel
 
   try {
     response = await delayedFetch(
       new URL(source.pathname, 'https://' + hostname),
       {},
-      source.courtesyWait ?? 5 * 1000
+      source.courtesyWait
     )
   } catch(err) {
     error = err
@@ -99,12 +103,24 @@ export async function fetchRss(source) {
   source.errored = error !== undefined || !response.ok
   if(source.errored) {
     source.error = error ?? createNetworkingError(response)
-  } else {
+    return
-    rss = await response.text()
+  }
-    console.log(`Found ${source.name} at ${hostname}`)
+  
+  console.log(`Found ${source.name} at ${hostname}`)
+  
+  try {
+    channel = createChannel(await response.text())
+  } catch(err) {
+    error = err
   }
 
-  return rss
+  source.errored = error !== undefined
+  if(source.errored) {
+    source.error = error
+    return
+  }
+
+  return channel
 }
 
 let waitingList = new Map()
@@ -121,6 +137,21 @@ export const delayedFetch = async (url, options, courtesyWait = 5 * 1000) => {
   return await fetch(url, options)
 }
 
+export const retryDelayedFetch = async (url, options, courtesyWait, retryAttempts) => {
+  let attemptsTried = 0
+  let response = undefined
+
+  while(isUnset(response) && attemptsTried <= (retryAttempts ?? 3)) {
+    if(attemptsTried > 0)
+      console.error(`Failed to fetch ${url}, retrying...`)
+
+    response = await delayedFetch(url, options, courtesyWait)
+    attemptsTried++
+  }
+
+  return response
+}
+
 export const getCacheFilename = (source) =>
   source.name + '.xml'
 
@@ -206,11 +237,6 @@ export const readPubDate = (pubDate) =>
 export const createPosts = async (channel, source, fromDate, reducerCallback) => {
   let items = channel.querySelectorAll('item')
 
-  // if(items.length === 0) {
-  //   // throw new NoMatchesError('Got no matches')
-  //   return source
-  // }
-
   let promises = []
 
   for(let item of items) {
@@ -272,7 +298,7 @@ export const extractImages = (post, cache = true) => {
     for(let image of images) {
       let { src } = image
 
-      if(!src) {
+      if(isUnset(src)) {
         let finalSrc = image.srcset.split(', ').pop()
 
         src = finalSrc.slice(0, finalSrc.indexOf(' ') )
@@ -403,7 +429,7 @@ export const renderPostDetail = (name, value) =>
 
 export const renderImage = href => {
   return `\
-<a href="${href}" download><img src="${href}" loading="lazy"></img></a>`
+<a href="${href}"><img src="${href}" loading="lazy"></img></a>`
 }
 
 export const renderDate = date =>
@@ -479,21 +505,36 @@ export const renderNavEntry = (list) => {
 // |   | | | | | | |   | | | | |
 // `-' `-' ' ' `-` `-' ' `-' ' '
 
-export const downloadImage = async (url, basename, courtesyWait, { batch, directory }) => {
+export const downloadImage = async (url, basename, { courtesyWait, retryAttempts }, { batch, directory }) => {
-  let response = await delayedFetch(url, {}, courtesyWait)
+  let response = await retryDelayedFetch(url, {}, courtesyWait, retryAttempts)
     .catch(err => console.error(`Failed download of ${url}:`, err, err.errors) )
 
+  if(response == undefined) {
+    console.error('Could not download image: ' + url)
+    return url
+  }
+
   if(response.ok) {
-    let relativePath = basename + imageExtensions[response.headers.get('Content-Type')]
+    let mimetype = response.headers.get('Content-Type').split(';')[0]
+    let extension = imageExtensions[mimetype]
+    
+    if(typeof extension !== 'string') {
+      console.error(`Unknown image mimetype for ${url}: ${mimetype}. Cannot download`)
+      return url
+    }
+
+    let relativePath = basename + extension
     let path = Path.join(directory, relativePath)
 
     const download = () => write(path, response.body)
       .then(annotate( `Downloaded ${relativePath}`))
 
+    // TODO: See if the image is downloaded before even trying to download it
     batch.add(FS.access(path).catch(download))
     return relativePath
   } else {
-    throw createNetworkingError(response)
+    console.error( createNetworkingError(response) )
+    return url
   }
 }
 
@@ -504,7 +545,7 @@ export const downloadImages = (images, source, postId, view) => {
   for(let i = 0; i < images.length; i++) {
     let basename = images.length > 1 ? basePath + '-' + i : basePath
 
-    pathnames.push(downloadImage(images[i], basename, source.courtesyWait, view))
+    pathnames.push(downloadImage(images[i], basename, source, view))
   }
   
   return Promise.all(pathnames)
@@ -546,19 +587,19 @@ export const createFeed = (name, sources, main = false) => {
   }
 }
 
-export const fetchRssFromInstances = async (source) => {
+export const fetchChannelFromInstances = async (source) => {
   let index = 0
   let instances = source.instances
   let cachedLink = source.cache.link
-  let rss
+  let channel
 
   if(cachedLink) {
     instances.unshift(cachedLink.hostname)
   }
 
-  while(!rss && index != instances.length) {
+  while(!channel && index != instances.length) {
     source.hostname = instances[index]
-    rss = await fetchRss(source)
+    channel = await fetchChannel(source)
 
     if(source.errored) {
       console.error(`Failed to fetch ${source.name} from ${source.hostname}: `, source.error)
@@ -568,10 +609,10 @@ export const fetchRssFromInstances = async (source) => {
     }
   }
 
-  return rss
+  return channel
 }
 
-export const populateSource = async (rss, source, postReducerCallback, useCache = true) => {
+export const populateSource = async (channel, source, postReducerCallback, useCache = true) => {
   let fromDate = 0
   source.items = []
   source.posts = []
@@ -590,8 +631,8 @@ export const populateSource = async (rss, source, postReducerCallback, useCache
     return postReducerCallback(post)
   }
 
-  if(rss ?? false)
+  if(channel ?? false)
-    source = await createPosts(createChannel(rss), source, fromDate, remoteReducerCallback)
+    source = await createPosts(channel, source, fromDate, remoteReducerCallback)
 
   return source
 }
@@ -615,14 +656,24 @@ export const writeView = (sources, feeds, view) => {
   writeStylesheet(Path.join(import.meta.dirname, 'assets/style.css'), view)
 }
 
-export const createSource = async (source, getRss, postReducerCallback, cache) => {
+export const createSource = async (source, getChannel, postReducerCallback, cache) => {
   source = await openCache(source, cache)
-  source = await populateSource(await getRss(source), source, postReducerCallback, cache.populate)
+  source = await populateSource(await getChannel(source), source, postReducerCallback, cache.populate)
 
   cache.batch.add(cacheSource(source, cache))
   return source
 }
 
+export const createSourceOptions = (options) => {
+  if(isUnset(options.courtesyWait))
+    options.courtesyWait = 1000
+
+  if(isUnset(options.retryAttempts))
+    options.retryAttempts = 3
+
+  return options
+}
+
 //     |     |    ,-
 // ;-. | ,-: |-   |  ,-. ;-. ;-.-. ,-.
 // | | | | | |    |- | | |   | | | `-.
@@ -630,20 +681,20 @@ export const createSource = async (source, getRss, postReducerCallback, cache) =
 // '             -'
 
 export const tumblr = {
-  createSource(user, courtesyWait, postReducerCallback, cache) {
+  createSource(user, options, postReducerCallback, cache) {
     let lowercaseUser = user.toLowerCase()
     let source = {
       type: 'tumblr',
       description: `Aggregate feed for @${lowercaseUser} on tumblr.com`,
       hostname: lowercaseUser + '.tumblr.com',
       pathname: 'rss',
-      courtesyWait,
       name: `tumblr-${lowercaseUser}`,
       displayName: user,
       user: lowercaseUser,
+      ...createSourceOptions(options)
     }
 
-    return createSource(source, fetchRss, postReducerCallback, cache)
+    return createSource(source, fetchChannel, postReducerCallback, cache)
   },
 
   createSources(users, ...args) {
@@ -672,19 +723,19 @@ export const tumblr = {
 }
 
 export const nitter = {
-  createSource(user, instances, courtesyWait, postReducerCallback, cache) {
+  createSource(user, options, instances, postReducerCallback, cache) {
     let source = {
       type: 'nitter',
       description: `Aggregate feed for @${user} on twitter.com`,
       instances,
       pathname: user + '/rss',
-      courtesyWait,
       name: `nitter-${user}`,
       displayName: user,
-      user
+      user,
+      ...createSourceOptions(options)
     }
 
-    return createSource(source, fetchRssFromInstances, postReducerCallback, cache)
+    return createSource(source, fetchChannelFromInstances, postReducerCallback, cache)
   },
 
   createSources(users, ...args) {
@@ -697,17 +748,36 @@ export const nitter = {
     return creator.innerHTML.slice(1) !== post.source.user
   },
 
-  pullImages
+  async pullImages (post, view, imageMirrorDomain, discardPostIfNoImages = false, getPostId = getPostIdFromPathname) {
+    let images = extractImages(post)
+    let mirroredImages = []
+    const mirrorImage = nitter.createImageMirrorer(post, imageMirrorDomain)
+
+    if(!discardPostIfNoImages || images.length > 0) {
+      post.images = await downloadImages(
+        images.map(mirrorImage),
+        post.source,
+        getPostId(post),
+        view
+      )
+      return post
+    }
+  },
+
+  createImageMirrorer(post, imageMirrorDomain) {
+    let mirrorUrl = new URL(imageMirrorDomain)
+    let basePathname = new URL(post.guid).pathname
+
+    return (image, index, images) => {
+      mirrorUrl.pathname = Path.join(basePathname, 'photo', (index + 1).toString())
+
+      return mirrorUrl.href
+    }
+  }
 }
 
-// TODO: Mastodon support
-//
-// "Turns out Mastodon has built-in RSS; your feed URL is [instance]/@[username].rss, so for example I'm
-// https://mastodon.social/@brownpau.rss (note the "@")"
-// - https://mastodon.social/@brownpau/100523448408374430
-
 export const mastodon = {
-  createSource(usertag, courtesyWait, postReducerCallback, cache) {
+  createSource(usertag, options, postReducerCallback, cache) {
     let [ user, hostname ] = usertag.toLowerCase().split('@')
     
     let source = {
@@ -715,13 +785,13 @@ export const mastodon = {
       description: `Aggregate feed for @${user} at ${hostname}`,
       hostname,
       pathname: '@' + user + ".rss",
-      courtesyWait,
       name: `${hostname}-${user}`,
       displayName: user,
       user,
+      ...createSourceOptions(options)
     }
 
-    return createSource(source, fetchRss, postReducerCallback, cache)
+    return createSource(source, fetchChannel, postReducerCallback, cache)
   },
 
   isRepost(post) {
@@ -747,4 +817,4 @@ export const mastodon = {
       return post
     }
   }
-}
+}

package.json

@@ -10,8 +10,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "jsdom": "^22.1.0",
+    "jsdom": "^22.1.0"
-    "mime-types": "^2.1.35"
   },
   "type": "module"
 }