Remove lock system in favor of caching
This commit is contained in:
parent
b29b7e6405
commit
fa7f27d7ce
203
lib.js
203
lib.js
@ -4,9 +4,6 @@ import Path from "path"
|
||||
import FS from "fs/promises"
|
||||
import { JSDOM } from "jsdom"
|
||||
|
||||
let cache = await FS.readFile('./cache.json', { encoding: 'utf-8' })
|
||||
.then(json => JSON.parse(json) )
|
||||
|
||||
|
||||
// | o |
|
||||
// . . |- . | ,-.
|
||||
@ -79,31 +76,6 @@ export const postIdFromPathname = post => {
|
||||
return pathname.slice(pathname.lastIndexOf('/') + 1)
|
||||
}
|
||||
|
||||
export const createLock = async (path) => {
|
||||
let lockExists = false
|
||||
|
||||
try {
|
||||
await FS.access(path)
|
||||
lockExists = true
|
||||
} catch(err) {
|
||||
lockExists = false
|
||||
}
|
||||
|
||||
let lock = {
|
||||
sources: {},
|
||||
lists: {}
|
||||
}
|
||||
|
||||
if(lockExists) {
|
||||
Object.assign(lock, JSON.parse(await FS.readFile(path, { encoding: 'utf8' })))
|
||||
}
|
||||
|
||||
return lock
|
||||
}
|
||||
|
||||
export const writeLock = (lock, path) =>
|
||||
write(path, JSON.stringify(lock) )
|
||||
|
||||
export const testWhitelist = (array, whitelist) =>
|
||||
whitelist.find(tag => !array.includes(tag)) !== undefined
|
||||
|
||||
@ -166,17 +138,94 @@ export const delayedFetch = async (url, options, courtesyWait = 5 * 1000) => {
|
||||
return await fetch(url, options)
|
||||
}
|
||||
|
||||
class NoMatchesError extends Error {}
|
||||
export const processRss = (source, fromDate, reducerCallback) => {
|
||||
let { document } = new JSDOM(source.rss, { contentType: 'text/xml' }).window
|
||||
let items = document.querySelectorAll('channel item')
|
||||
export const getCachePath = (source, { directory }) =>
|
||||
Path.join(directory, source.name + '.xml')
|
||||
|
||||
if(items.length == 0) {
|
||||
throw new NoMatchesError('Got no matches')
|
||||
export const cacheSource = (source, cache) =>
|
||||
write(getCachePath(source, cache), createCache(source, cache))
|
||||
|
||||
export const cacheSources = (sources, cache) =>
|
||||
Promise.all(sources.map(source => cacheSource(source, cache)))
|
||||
|
||||
export const openCache = async (source, cache) => {
|
||||
let path = getCachePath(source, cache)
|
||||
let exists
|
||||
|
||||
try {
|
||||
await FS.access(path)
|
||||
exists = true
|
||||
} catch(err) {
|
||||
exists = false
|
||||
}
|
||||
|
||||
if(exists) {
|
||||
let rss = await FS.readFile(path, { encoding: 'utf8' })
|
||||
let channel = createChannel(rss)
|
||||
let date = readPubDate(channel.querySelector('pubDate'))
|
||||
let link = new URL(channel.querySelector('link').textContent)
|
||||
|
||||
source.cache = {
|
||||
channel,
|
||||
date,
|
||||
link
|
||||
}
|
||||
} else {
|
||||
source.cache = {
|
||||
date: new Date(0),
|
||||
}
|
||||
|
||||
if(source.hostname)
|
||||
source.cache.link = buildCacheLink(source)
|
||||
}
|
||||
|
||||
source.latestPostDate = source.cache.date
|
||||
|
||||
return source
|
||||
}
|
||||
|
||||
export const buildCacheLink = source =>
|
||||
new URL('https://' + source.hostname)
|
||||
|
||||
// TODO: Support atom links
|
||||
// https://validator.w3.org/feed/docs/warning/MissingAtomSelfLink.html
|
||||
// TODO: Add a description of some kind
|
||||
export const createCache = (source, cache) => `\
|
||||
<?xml version="1.0" encoding="UTF-8" ?>
|
||||
<rss version="2.0">
|
||||
|
||||
<channel>
|
||||
|
||||
<title>${source.displayName}</title>
|
||||
<link>${buildCacheLink(source)}</link>
|
||||
<pubDate>${new Date(source.latestPostDate).toUTCString()}</pubDate>
|
||||
<generator>rssssing</generator>
|
||||
${source.posts.map(post => post.item.outerHTML.replaceAll(/\n\s*/g, '')).join('\n')}
|
||||
|
||||
</channel>
|
||||
|
||||
</rss>`
|
||||
|
||||
export const createChannel = rss => {
|
||||
let { document } = new JSDOM(rss, { contentType: 'text/xml' }).window
|
||||
|
||||
return document.querySelector('channel')
|
||||
}
|
||||
|
||||
export const readPubDate = (pubDate) =>
|
||||
pubDate ? new Date(pubDate.textContent).valueOf() : 0
|
||||
|
||||
class NoMatchesError extends Error {}
|
||||
export const createPosts = (channel, source, fromDate, reducerCallback) => {
|
||||
// let { document } = new JSDOM(rss, { contentType: 'text/xml' }).window
|
||||
let items = channel.querySelectorAll('item')
|
||||
|
||||
// if(items.length === 0) {
|
||||
// // throw new NoMatchesError('Got no matches')
|
||||
// return source
|
||||
// }
|
||||
|
||||
for(let item of items) {
|
||||
let post = createPost(source, item, reducerCallback)
|
||||
let post = createPost(item, source, reducerCallback)
|
||||
|
||||
if(post && post.date > fromDate) {
|
||||
source.posts.push(post)
|
||||
@ -186,9 +235,9 @@ export const processRss = (source, fromDate, reducerCallback) => {
|
||||
return source
|
||||
}
|
||||
|
||||
export const createPost = (source, item, reducerCallback) => {
|
||||
export const createPost = (item, source, reducerCallback) => {
|
||||
let description = new JSDOM(item.querySelector('description').textContent).window.document
|
||||
let dateString = item.querySelector('pubDate').textContent
|
||||
let date = readPubDate(item.querySelector('pubDate'))
|
||||
let link = item.querySelector('link').textContent
|
||||
let guid = item.querySelector('guid')?.textContent
|
||||
let title = item.querySelector('title')?.textContent
|
||||
@ -197,8 +246,7 @@ export const createPost = (source, item, reducerCallback) => {
|
||||
source,
|
||||
item,
|
||||
description,
|
||||
dateString,
|
||||
date: new Date(dateString).valueOf() ?? 0,
|
||||
date,
|
||||
link,
|
||||
guid,
|
||||
title,
|
||||
@ -451,13 +499,13 @@ export const createFeed = (name, sources, main = false) => {
|
||||
}
|
||||
}
|
||||
|
||||
export const fetchRssFromInstances = async (source, lock) => {
|
||||
export const fetchRssFromInstances = async (source) => {
|
||||
let index = 0
|
||||
let instances = source.instances
|
||||
let lockHostname = lock.sources[source.name]?.hostname
|
||||
let cachedLink = source.cache.link
|
||||
|
||||
if(lockHostname) {
|
||||
instances.unshift(lockHostname)
|
||||
if(cachedLink) {
|
||||
instances.unshift(cachedLink.hostname)
|
||||
}
|
||||
|
||||
while(!source.rss && index != instances.length) {
|
||||
@ -475,47 +523,29 @@ export const fetchRssFromInstances = async (source, lock) => {
|
||||
return source
|
||||
}
|
||||
|
||||
export const populateSource = (source, postReducerCallback, lock) => {
|
||||
let sourceLock = lock.sources[source.name] ??= {}
|
||||
|
||||
export const populateSource = (source, postReducerCallback, useCache = true) => {
|
||||
let fromDate = 0
|
||||
source.posts = []
|
||||
source = processRss(source, sourceLock.timestamp ?? 0, postReducerCallback, lock)
|
||||
|
||||
if(sourceLock.items) {
|
||||
for(let itemText of sourceLock.items) {
|
||||
let item = new JSDOM(itemText, { contentType: 'text/xml' }).window.document.documentElement
|
||||
|
||||
source.posts.push(createPost(source, item, postReducerCallback))
|
||||
}
|
||||
if(useCache) {
|
||||
fromDate = source.latestPostDate
|
||||
|
||||
if(source.cache.channel)
|
||||
source = createPosts(source.cache.channel, source, 0, postReducerCallback)
|
||||
}
|
||||
|
||||
lock.sources[source.name] = sourceLock
|
||||
lockSource(source, lock)
|
||||
let remoteReducerCallback = post => {
|
||||
if(post.date > source.latestPostDate)
|
||||
source.latestPostDate = post.date
|
||||
|
||||
return postReducerCallback(post)
|
||||
}
|
||||
|
||||
source = createPosts(createChannel(source.rss), source, fromDate, remoteReducerCallback)
|
||||
|
||||
return source
|
||||
}
|
||||
|
||||
export const lockSource = (source, lock) => {
|
||||
let date = 0
|
||||
let items = []
|
||||
|
||||
for(let post of source.posts) {
|
||||
if(post.date > date)
|
||||
date = post.date
|
||||
|
||||
items.push(post.item.outerHTML)
|
||||
}
|
||||
|
||||
lock.sources[source.name] = {
|
||||
hostname: source.hostname,
|
||||
timestamp: date,
|
||||
items
|
||||
}
|
||||
}
|
||||
|
||||
export const lockSources = (sources, lock) => {
|
||||
sources.forEach(source => lockSource(source, lock))
|
||||
}
|
||||
|
||||
export const writeView = (sources, feeds, view) => {
|
||||
view.header = renderNav(feeds, sources)
|
||||
let pages = []
|
||||
@ -535,6 +565,13 @@ export const writeView = (sources, feeds, view) => {
|
||||
writeStylesheet(Path.join(import.meta.dirname, 'assets/style.css'), view)
|
||||
}
|
||||
|
||||
export const createSource = async (source, getRss, postReducerCallback, cache) => {
|
||||
source = await openCache(source, cache)
|
||||
source = await getRss(source)
|
||||
source = populateSource(source, postReducerCallback, cache.populate)
|
||||
return source
|
||||
}
|
||||
|
||||
// | | ,-
|
||||
// ;-. | ,-: |- | ,-. ;-. ;-.-. ,-.
|
||||
// | | | | | | |- | | | | | | `-.
|
||||
@ -542,7 +579,7 @@ export const writeView = (sources, feeds, view) => {
|
||||
// ' -'
|
||||
|
||||
export const tumblr = {
|
||||
async createSource(user, courtesyWait, postReducerCallback, lock) {
|
||||
createSource(user, courtesyWait, postReducerCallback, cache) {
|
||||
let lowercaseUser = user.toLowerCase()
|
||||
let source = {
|
||||
hostname: lowercaseUser + '.tumblr.com',
|
||||
@ -553,9 +590,7 @@ export const tumblr = {
|
||||
user: lowercaseUser,
|
||||
}
|
||||
|
||||
source = await fetchRss(source)
|
||||
source = populateSource(source, postReducerCallback, lock)
|
||||
return source
|
||||
return createSource(source, fetchRss, postReducerCallback, cache)
|
||||
},
|
||||
|
||||
createSources(users, ...args) {
|
||||
@ -584,7 +619,7 @@ export const tumblr = {
|
||||
}
|
||||
|
||||
export const nitter = {
|
||||
async createSource(user, instances, courtesyWait, postReducerCallback, lock) {
|
||||
createSource(user, instances, courtesyWait, postReducerCallback, cache) {
|
||||
let source = {
|
||||
instances,
|
||||
pathname: user + '/rss',
|
||||
@ -594,9 +629,7 @@ export const nitter = {
|
||||
user
|
||||
}
|
||||
|
||||
source = await fetchRssFromInstances(source, lock)
|
||||
source = populateSource(source, postReducerCallback, lock)
|
||||
return source
|
||||
return createSource(source, fetchRssFromInstances, postReducerCallback, cache)
|
||||
},
|
||||
|
||||
createSources(users, ...args) {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user