Remove lock system in favor of caching

lib.js

@@ -4,9 +4,6 @@ import Path from "path"
 import FS from "fs/promises"
 import { JSDOM } from "jsdom"
 
-let cache = await FS.readFile('./cache.json', { encoding: 'utf-8' })
-  .then(json => JSON.parse(json) )
-
 
 //     |   o |
 // . . |-  . | ,-.
@@ -79,31 +76,6 @@ export const postIdFromPathname = post => {
   return pathname.slice(pathname.lastIndexOf('/') + 1)
 }
 
-export const createLock = async (path) => {
-  let lockExists = false
-
-  try {
-    await FS.access(path)
-    lockExists = true
-  } catch(err) {
-    lockExists = false
-  }
-
-  let lock = {
-    sources: {},
-    lists: {}
-  }
-
-  if(lockExists) {
-    Object.assign(lock, JSON.parse(await FS.readFile(path, { encoding: 'utf8' })))
-  }
-
-  return lock
-}
-
-export const writeLock = (lock, path) =>
-  write(path, JSON.stringify(lock) )
-
 export const testWhitelist = (array, whitelist) =>
   whitelist.find(tag => !array.includes(tag)) !== undefined
 
@@ -166,17 +138,94 @@ export const delayedFetch = async (url, options, courtesyWait = 5 * 1000) => {
   return await fetch(url, options)
 }
 
-class NoMatchesError extends Error {}
+export const getCachePath = (source, { directory }) =>
-export const processRss = (source, fromDate, reducerCallback) => {
+  Path.join(directory, source.name + '.xml')
-  let { document } = new JSDOM(source.rss, { contentType: 'text/xml' }).window
-  let items = document.querySelectorAll('channel item')
 
-  if(items.length == 0) {
+export const cacheSource = (source, cache) =>
-    throw new NoMatchesError('Got no matches')
+  write(getCachePath(source, cache), createCache(source, cache))
+
+export const cacheSources = (sources, cache) =>
+  Promise.all(sources.map(source => cacheSource(source, cache)))
+
+export const openCache = async (source, cache) => {
+  let path = getCachePath(source, cache)
+  let exists
+
+  try {
+    await FS.access(path)
+    exists = true
+  } catch(err) {
+    exists = false
   }
 
+  if(exists) {
+    let rss = await FS.readFile(path, { encoding: 'utf8' })
+    let channel = createChannel(rss)
+    let date = readPubDate(channel.querySelector('pubDate'))
+    let link = new URL(channel.querySelector('link').textContent)
+
+    source.cache = {
+      channel,
+      date,
+      link
+    }
+  } else {
+    source.cache = {
+      date: new Date(0),
+    }
+
+    if(source.hostname)
+      source.cache.link = buildCacheLink(source)
+  }
+
+  source.latestPostDate = source.cache.date
+
+  return source
+}
+
+export const buildCacheLink = source =>
+  new URL('https://' + source.hostname)
+
+// TODO: Support atom links
+// https://validator.w3.org/feed/docs/warning/MissingAtomSelfLink.html
+// TODO: Add a description of some kind
+export const createCache = (source, cache) => `\
+<?xml version="1.0" encoding="UTF-8" ?>
+<rss version="2.0">
+
+<channel>
+
+<title>${source.displayName}</title>
+<link>${buildCacheLink(source)}</link>
+<pubDate>${new Date(source.latestPostDate).toUTCString()}</pubDate>
+<generator>rssssing</generator>
+${source.posts.map(post => post.item.outerHTML.replaceAll(/\n\s*/g, '')).join('\n')}
+
+</channel>
+
+</rss>`
+
+export const createChannel = rss => {
+  let { document } = new JSDOM(rss, { contentType: 'text/xml' }).window
+
+  return document.querySelector('channel')
+}
+
+export const readPubDate = (pubDate) =>
+  pubDate ? new Date(pubDate.textContent).valueOf() : 0
+
+class NoMatchesError extends Error {}
+export const createPosts = (channel, source, fromDate, reducerCallback) => {
+  // let { document } = new JSDOM(rss, { contentType: 'text/xml' }).window
+  let items = channel.querySelectorAll('item')
+
+  // if(items.length === 0) {
+  //   // throw new NoMatchesError('Got no matches')
+  //   return source
+  // }
+
   for(let item of items) {
-    let post = createPost(source, item, reducerCallback)
+    let post = createPost(item, source, reducerCallback)
 
     if(post && post.date > fromDate) {
       source.posts.push(post)
@@ -186,9 +235,9 @@ export const processRss = (source, fromDate, reducerCallback) => {
   return source
 }
 
-export const createPost = (source, item, reducerCallback) => {
+export const createPost = (item, source, reducerCallback) => {
   let description = new JSDOM(item.querySelector('description').textContent).window.document
-  let dateString = item.querySelector('pubDate').textContent
+  let date = readPubDate(item.querySelector('pubDate'))
   let link = item.querySelector('link').textContent
   let guid = item.querySelector('guid')?.textContent
   let title = item.querySelector('title')?.textContent
@@ -197,8 +246,7 @@ export const createPost = (source, item, reducerCallback) => {
     source,
     item,
     description,
-    dateString,
+    date,
-    date: new Date(dateString).valueOf() ?? 0,
     link,
     guid,
     title,
@@ -451,13 +499,13 @@ export const createFeed = (name, sources, main = false) => {
   }
 }
 
-export const fetchRssFromInstances = async (source, lock) => {
+export const fetchRssFromInstances = async (source) => {
   let index = 0
   let instances = source.instances
-  let lockHostname = lock.sources[source.name]?.hostname
+  let cachedLink = source.cache.link
 
-  if(lockHostname) {
+  if(cachedLink) {
-    instances.unshift(lockHostname)
+    instances.unshift(cachedLink.hostname)
   }
 
   while(!source.rss && index != instances.length) {
@@ -475,47 +523,29 @@ export const fetchRssFromInstances = async (source, lock) => {
   return source
 }
 
-export const populateSource = (source, postReducerCallback, lock) => {
+export const populateSource = (source, postReducerCallback, useCache = true) => {
-  let sourceLock = lock.sources[source.name] ??= {}
+  let fromDate = 0
-
   source.posts = []
-  source = processRss(source, sourceLock.timestamp ?? 0, postReducerCallback, lock)
 
-  if(sourceLock.items) {
+  if(useCache) {
-    for(let itemText of sourceLock.items) {
+    fromDate = source.latestPostDate
-      let item = new JSDOM(itemText, { contentType: 'text/xml' }).window.document.documentElement
+
-  
+    if(source.cache.channel)
-      source.posts.push(createPost(source, item, postReducerCallback))
+      source = createPosts(source.cache.channel, source, 0, postReducerCallback)
-    }
   }
 
-  lock.sources[source.name] = sourceLock
+  let remoteReducerCallback = post => {
-  lockSource(source, lock)
+    if(post.date > source.latestPostDate)
+      source.latestPostDate = post.date
+
+    return postReducerCallback(post)
+  }
+
+  source = createPosts(createChannel(source.rss), source, fromDate, remoteReducerCallback)
+
   return source
 }
 
-export const lockSource = (source, lock) => {
-  let date = 0
-  let items = []
-
-  for(let post of source.posts) {
-    if(post.date > date)
-      date = post.date
-
-    items.push(post.item.outerHTML)
-  }
-
-  lock.sources[source.name] = {
-    hostname: source.hostname,
-    timestamp: date,
-    items
-  }
-}
-
-export const lockSources = (sources, lock) => {
-  sources.forEach(source => lockSource(source, lock))
-}
-
 export const writeView = (sources, feeds, view) => {
   view.header = renderNav(feeds, sources)
   let pages = []
@@ -535,6 +565,13 @@ export const writeView = (sources, feeds, view) => {
   writeStylesheet(Path.join(import.meta.dirname, 'assets/style.css'), view)
 }
 
+export const createSource = async (source, getRss, postReducerCallback, cache) => {
+  source = await openCache(source, cache)
+  source = await getRss(source)
+  source = populateSource(source, postReducerCallback, cache.populate)
+  return source
+}
+
 //     |     |    ,-
 // ;-. | ,-: |-   |  ,-. ;-. ;-.-. ,-.
 // | | | | | |    |- | | |   | | | `-.
@@ -542,7 +579,7 @@ export const writeView = (sources, feeds, view) => {
 // '             -'
 
 export const tumblr = {
-  async createSource(user, courtesyWait, postReducerCallback, lock) {
+  createSource(user, courtesyWait, postReducerCallback, cache) {
     let lowercaseUser = user.toLowerCase()
     let source = {
       hostname: lowercaseUser + '.tumblr.com',
@@ -553,9 +590,7 @@ export const tumblr = {
       user: lowercaseUser,
     }
 
-    source = await fetchRss(source)
+    return createSource(source, fetchRss, postReducerCallback, cache)
-    source = populateSource(source, postReducerCallback, lock)
-    return source
   },
 
   createSources(users, ...args) {
@@ -584,7 +619,7 @@ export const tumblr = {
 }
 
 export const nitter = {
-  async createSource(user, instances, courtesyWait, postReducerCallback, lock) {
+  createSource(user, instances, courtesyWait, postReducerCallback, cache) {
     let source = {
       instances,
       pathname: user + '/rss',
@@ -594,9 +629,7 @@ export const nitter = {
       user
     }
 
-    source = await fetchRssFromInstances(source, lock)
+    return createSource(source, fetchRssFromInstances, postReducerCallback, cache)
-    source = populateSource(source, postReducerCallback, lock)
-    return source
   },
 
   createSources(users, ...args) {