diff --git a/lib.js b/lib.js index 9733db4..22ace52 100644 --- a/lib.js +++ b/lib.js @@ -4,9 +4,6 @@ import Path from "path" import FS from "fs/promises" import { JSDOM } from "jsdom" -let cache = await FS.readFile('./cache.json', { encoding: 'utf-8' }) - .then(json => JSON.parse(json) ) - // | o | // . . |- . | ,-. @@ -79,31 +76,6 @@ export const postIdFromPathname = post => { return pathname.slice(pathname.lastIndexOf('/') + 1) } -export const createLock = async (path) => { - let lockExists = false - - try { - await FS.access(path) - lockExists = true - } catch(err) { - lockExists = false - } - - let lock = { - sources: {}, - lists: {} - } - - if(lockExists) { - Object.assign(lock, JSON.parse(await FS.readFile(path, { encoding: 'utf8' }))) - } - - return lock -} - -export const writeLock = (lock, path) => - write(path, JSON.stringify(lock) ) - export const testWhitelist = (array, whitelist) => whitelist.find(tag => !array.includes(tag)) !== undefined @@ -166,17 +138,94 @@ export const delayedFetch = async (url, options, courtesyWait = 5 * 1000) => { return await fetch(url, options) } -class NoMatchesError extends Error {} -export const processRss = (source, fromDate, reducerCallback) => { - let { document } = new JSDOM(source.rss, { contentType: 'text/xml' }).window - let items = document.querySelectorAll('channel item') +export const getCachePath = (source, { directory }) => + Path.join(directory, source.name + '.xml') - if(items.length == 0) { - throw new NoMatchesError('Got no matches') +export const cacheSource = (source, cache) => + write(getCachePath(source, cache), createCache(source, cache)) + +export const cacheSources = (sources, cache) => + Promise.all(sources.map(source => cacheSource(source, cache))) + +export const openCache = async (source, cache) => { + let path = getCachePath(source, cache) + let exists + + try { + await FS.access(path) + exists = true + } catch(err) { + exists = false } + if(exists) { + let rss = await FS.readFile(path, { encoding: 'utf8' }) + let channel = createChannel(rss) + let date = readPubDate(channel.querySelector('pubDate')) + let link = new URL(channel.querySelector('link').textContent) + + source.cache = { + channel, + date, + link + } + } else { + source.cache = { + date: new Date(0), + } + + if(source.hostname) + source.cache.link = buildCacheLink(source) + } + + source.latestPostDate = source.cache.date + + return source +} + +export const buildCacheLink = source => + new URL('https://' + source.hostname) + +// TODO: Support atom links +// https://validator.w3.org/feed/docs/warning/MissingAtomSelfLink.html +// TODO: Add a description of some kind +export const createCache = (source, cache) => `\ + + + + + +${source.displayName} +${buildCacheLink(source)} +${new Date(source.latestPostDate).toUTCString()} +rssssing +${source.posts.map(post => post.item.outerHTML.replaceAll(/\n\s*/g, '')).join('\n')} + + + +` + +export const createChannel = rss => { + let { document } = new JSDOM(rss, { contentType: 'text/xml' }).window + + return document.querySelector('channel') +} + +export const readPubDate = (pubDate) => + pubDate ? new Date(pubDate.textContent).valueOf() : 0 + +class NoMatchesError extends Error {} +export const createPosts = (channel, source, fromDate, reducerCallback) => { + // let { document } = new JSDOM(rss, { contentType: 'text/xml' }).window + let items = channel.querySelectorAll('item') + + // if(items.length === 0) { + // // throw new NoMatchesError('Got no matches') + // return source + // } + for(let item of items) { - let post = createPost(source, item, reducerCallback) + let post = createPost(item, source, reducerCallback) if(post && post.date > fromDate) { source.posts.push(post) @@ -186,9 +235,9 @@ export const processRss = (source, fromDate, reducerCallback) => { return source } -export const createPost = (source, item, reducerCallback) => { +export const createPost = (item, source, reducerCallback) => { let description = new JSDOM(item.querySelector('description').textContent).window.document - let dateString = item.querySelector('pubDate').textContent + let date = readPubDate(item.querySelector('pubDate')) let link = item.querySelector('link').textContent let guid = item.querySelector('guid')?.textContent let title = item.querySelector('title')?.textContent @@ -197,8 +246,7 @@ export const createPost = (source, item, reducerCallback) => { source, item, description, - dateString, - date: new Date(dateString).valueOf() ?? 0, + date, link, guid, title, @@ -451,13 +499,13 @@ export const createFeed = (name, sources, main = false) => { } } -export const fetchRssFromInstances = async (source, lock) => { +export const fetchRssFromInstances = async (source) => { let index = 0 let instances = source.instances - let lockHostname = lock.sources[source.name]?.hostname + let cachedLink = source.cache.link - if(lockHostname) { - instances.unshift(lockHostname) + if(cachedLink) { + instances.unshift(cachedLink.hostname) } while(!source.rss && index != instances.length) { @@ -475,47 +523,29 @@ export const fetchRssFromInstances = async (source, lock) => { return source } -export const populateSource = (source, postReducerCallback, lock) => { - let sourceLock = lock.sources[source.name] ??= {} - +export const populateSource = (source, postReducerCallback, useCache = true) => { + let fromDate = 0 source.posts = [] - source = processRss(source, sourceLock.timestamp ?? 0, postReducerCallback, lock) - if(sourceLock.items) { - for(let itemText of sourceLock.items) { - let item = new JSDOM(itemText, { contentType: 'text/xml' }).window.document.documentElement - - source.posts.push(createPost(source, item, postReducerCallback)) - } + if(useCache) { + fromDate = source.latestPostDate + + if(source.cache.channel) + source = createPosts(source.cache.channel, source, 0, postReducerCallback) } - lock.sources[source.name] = sourceLock - lockSource(source, lock) + let remoteReducerCallback = post => { + if(post.date > source.latestPostDate) + source.latestPostDate = post.date + + return postReducerCallback(post) + } + + source = createPosts(createChannel(source.rss), source, fromDate, remoteReducerCallback) + return source } -export const lockSource = (source, lock) => { - let date = 0 - let items = [] - - for(let post of source.posts) { - if(post.date > date) - date = post.date - - items.push(post.item.outerHTML) - } - - lock.sources[source.name] = { - hostname: source.hostname, - timestamp: date, - items - } -} - -export const lockSources = (sources, lock) => { - sources.forEach(source => lockSource(source, lock)) -} - export const writeView = (sources, feeds, view) => { view.header = renderNav(feeds, sources) let pages = [] @@ -535,6 +565,13 @@ export const writeView = (sources, feeds, view) => { writeStylesheet(Path.join(import.meta.dirname, 'assets/style.css'), view) } +export const createSource = async (source, getRss, postReducerCallback, cache) => { + source = await openCache(source, cache) + source = await getRss(source) + source = populateSource(source, postReducerCallback, cache.populate) + return source +} + // | | ,- // ;-. | ,-: |- | ,-. ;-. ;-.-. ,-. // | | | | | | |- | | | | | | `-. @@ -542,7 +579,7 @@ export const writeView = (sources, feeds, view) => { // ' -' export const tumblr = { - async createSource(user, courtesyWait, postReducerCallback, lock) { + createSource(user, courtesyWait, postReducerCallback, cache) { let lowercaseUser = user.toLowerCase() let source = { hostname: lowercaseUser + '.tumblr.com', @@ -553,9 +590,7 @@ export const tumblr = { user: lowercaseUser, } - source = await fetchRss(source) - source = populateSource(source, postReducerCallback, lock) - return source + return createSource(source, fetchRss, postReducerCallback, cache) }, createSources(users, ...args) { @@ -584,7 +619,7 @@ export const tumblr = { } export const nitter = { - async createSource(user, instances, courtesyWait, postReducerCallback, lock) { + createSource(user, instances, courtesyWait, postReducerCallback, cache) { let source = { instances, pathname: user + '/rss', @@ -594,9 +629,7 @@ export const nitter = { user } - source = await fetchRssFromInstances(source, lock) - source = populateSource(source, postReducerCallback, lock) - return source + return createSource(source, fetchRssFromInstances, postReducerCallback, cache) }, createSources(users, ...args) {