// Ascii font used is "Shimrod" import Path from "path" import FS from "fs/promises" import { JSDOM } from "jsdom" let cache = await FS.readFile('./cache.json', { encoding: 'utf-8' }) .then(json => JSON.parse(json) ) // | o | // . . |- . | ,-. // | | | | | `-. // `-` `-' ' ' `-' export function PromiseBatch() { let promises = [] this.add = (promise) => promises.push(promise) this.complete = () => Promise.all(promises) } export const annotate = annotation => data => { console.log(annotation), data } export const write = async (path, content) => { let dir = Path.dirname(path) try { await FS.access(dir) } catch(e) { await FS.mkdir(dir, { recursive: true }) } return await FS.writeFile(path, content) } export const download = async (url, path, courtesyWait) => { let response = await delayedFetch(url, {}, courtesyWait) .catch(err => console.error(`Failed download of ${url}:`, err) ) if(response.ok) { await write(path, response.body) } else { throw createNetworkingError(response) } } export const createNetworkingError = response => { return new Error(`Request failed, ${response.status}: ${response.statusText}`) } export const getLinkExtname = link => Path.extname(new URL(link).pathname) export const buildImagePathHandler = (source, id) => (url, i, array) => { let path = `images/${source.name}-${id}` if(array.length > 1) path += `-${i}` return path + getLinkExtname(url) } export const addStylesheet = (path, { viewDir, batch }) => batch.add( FS.readFile(path) .then(content => write(Path.join(viewDir, 'style.css'), content)) ) export const postIdFromPathname = post => { let { pathname } = new URL(post.link) return pathname.slice(pathname.lastIndexOf('/') + 1) } export const createLock = async renderer => { let lockExists = false try { await FS.access(renderer.lockPath) lockExists = true } catch(err) { lockExists = false } renderer.lock = { sources: {}, lists: {} } if(lockExists) { let lock = JSON.parse(await FS.readFile(renderer.lockPath, { encoding: 'utf8' })) Object.assign(renderer.lock, lock) } } export const writeLock = renderer => write(renderer.lockPath, JSON.stringify(renderer.lock) ) // // ;-. ,-. ,-. // | `-. `-. // ' `-' `-' class NoMatchesError extends Error {} export const processRss = (source, reducerCallback) => { let { document } = new JSDOM(source.rss, { contentType: 'text/xml' }).window let items = document.querySelectorAll('channel item') if(items.length == 0) { throw new NoMatchesError('Got no matches') } source.posts = [] for(let item of items) { let description = new JSDOM(item.querySelector('description').textContent).window.document let dateString = item.querySelector('pubDate').textContent let link = item.querySelector('link').textContent let guid = item.querySelector('guid').textContent let post = { source, item, description, dateString, date: new Date(dateString).valueOf() ?? 0, link, guid } post = reducerCallback(post) if(post) { source.posts.push(post) } } return source } let waitingList = new Map() export const sleep = delay => new Promise(resolve => setTimeout(() => resolve(), delay) ) export const delayedFetch = async (url, options, courtesyWait = 5 * 1000) => { let [ domain ] = /[\w-]+.[\w-]+$/.exec(url.hostname) let timeout = waitingList.get(domain) ?? 0 let now = Date.now() if(timeout == null || timeout <= now) { waitingList.set(domain, timeout + courtesyWait) } else { await sleep(timeout - now) } return await fetch(url, options) } export async function fetchRss(source) { let { hostname } = source let error let response try { response = await delayedFetch( new URL(source.pathname, 'https://' + hostname), {}, source.courtesyWait ?? 5 * 1000 ) } catch(err) { error = err } source.errored = error !== undefined || !response.ok if(source.errored) { source.error = error ?? createNetworkingError(response) source.rss = '' } else { source.rss = await response.text() } return source } export const extractImages = (post, cache = true) => { let images = post.description.querySelectorAll('img') if(images) { let imageUrls = [] for(let image of images) { let { src } = image if(!src) { let finalSrc = image.srcset.split(', ').pop() src = finalSrc.slice(0, finalSrc.indexOf(' ') ) } // Sending through URL prevents potential XSS imageUrls.push(new URL(src).href) } return imageUrls } } // o // . , . ,-. , , , // |/ | |-' |/|/ // ' ' `-' ' ' export const createPages = (list, { pageSize, header = '', viewDir, batch, getPageFilename, getPageTitle, lock }) => { let posts = [] let lastPageLink = 'about:blank' let pageIndex = 0 // let pageIndex = Math.ceil(list.posts.length / pageSize) // let { // index: pageIndex = 0, // lastPostDate // } = lock.lists[list.name]?.lastPage ?? {} // let sinceDate = posts[0]?.date ?? 0 // posts = list.posts // .filter(post => post.date > sinceDate) // .concat(posts) // .sort((a, b) => b.date - a.date) // let firstPageSize = list.posts.sort((a, b) => a.date - b.date) for(let i = 0; i < list.posts.length; i++) { // for(let i = list.posts.length - 1; i >= 0; i--) { posts.push(list.posts[i]) if(i % pageSize == 0) { let isLastPage = list.main && i < pageSize let title = getPageTitle(list, pageIndex) let html = renderPage(title, posts.reverse(), header, renderNextPageLink(lastPageLink)) let filename = isLastPage ? 'index.html' : getPageFilename(list, pageIndex) let promise = write(Path.join(viewDir, filename), html) batch.add(promise.then(annotate(`Created "${title}" (${filename})`))) posts = [] lastPageLink = filename pageIndex++ } } // lock.lists[list.name] = { // pageIndex, // lastPostDate: posts[0]?.date ?? lastPostDate // } } export const renderPage = (title, posts, header, footer) => `\ ${title}
${header}
${posts.map(renderPost).join('\n')} ` export const renderPost = post => { let date = new Date(post.date) return `\ ${post.images.map(renderImage).join('\n')}

${post.source.displayName} ${renderDate(date)} open


` } export const renderImage = href => { return `\ ` } export const renderDate = date => (date.getMonth() + 1) + '.' + date.getDate() + '.' + date.getFullYear() export const renderNextPageLink = link => `\ next` export const renderNav = (feeds, sources) => `\
Feeds


` export const renderNavEntry = (list) => { let extra = '' if(list.error) { extra += ' (errored)' } else if (list.posts.length == 0) { extra += ' (empty)' } return `
  • ${list.displayName}${extra}
  • ` } // | | | o // ,-. ,-. | | ,-: |- . ,-. ;-. // | | | | | | | | | | | | | // `-' `-' ' ' `-` `-' ' `-' ' ' export const downloadImages = (images, getImagePath, courtesyWait, { viewDir, batch }) => { let out = [] for(let i = 0; i < images.length; i ++) { let url = images[i] let relativePath = getImagePath(url, i, images) let fullPath = Path.join(viewDir, relativePath) let promise = FS.access(fullPath) .catch(() => download(url, fullPath, courtesyWait) .then(annotate( `Downloaded ${relativePath}`)) ) out.push(relativePath) batch.add(promise) } return out } export const pullImages = (post, renderer, discardPostIfNoImages = false, getPostId = postIdFromPathname) => { let images = extractImages(post) if(!discardPostIfNoImages || images.length > 0) { post.images = downloadImages( images, buildImagePathHandler(post.source, getPostId(post)), post.source.courtesyWait, renderer ) return post } } export const createFeed = (name, sources) => { return { name, posts: sources.reduce((posts, source) => posts.concat(source.posts), []) } } export const fetchRssFromInstances = async (source, renderer) => { let index = 0 let instances = source.instances let lockHostname = renderer.lock.sources[source.name]?.hostname if(lockHostname) { instances.unshift(lockHostname) } while(!source.rss && index != instances.length) { source.hostname = instances[index] source = await fetchRss(source) if(source.errored) { console.error(`Failed to fetch ${source.name} from ${source.hostname}: `, source.error) index++ } else { break } } (renderer.lock.sources[source.name] ??= {}).hostname = source.hostname return source } const addPostsToLock = (source, renderer) => { (renderer.lock.sources[source.name] ??= {}).postData = source.posts.map(post => post.description) } // | | ,- // ;-. | ,-: |- | ,-. ;-. ;-.-. ,-. // | | | | | | |- | | | | | | `-. // |-' ' `-` `-' | `-' ' ' ' ' `-' // ' -' export const tumblr = { async createSource(user, courtesyWait, postReducerCallback, renderer) { let source = { hostname: user + '.tumblr.com', pathname: 'rss', courtesyWait, name: `tumblr-${user}`, displayName: user, user } source = await fetchRss(source) source = processRss(source, postReducerCallback) addPostsToLock(source, renderer) return source }, createSources(users, ...args) { return Promise.all(users.map(user => tumblr.createSource(user, ...args))) }, isRepost(post) { let reblog = post.description.querySelector('p > a.tumblr_blog') return reblog && reblog.innerHTML !== post.source.user }, pullImages } export const nitter = { async createSource(user, instances, courtesyWait, postReducerCallback, renderer) { let source = { instances, pathname: user + '/rss', courtesyWait, name: `nitter-${user}`, displayName: user, user } source = await fetchRssFromInstances(source, renderer) source = processRss(source, postReducerCallback) return source }, createSources(users, ...args) { return Promise.all(users.map(user => nitter.createSource(user, ...args))) }, isRepost(post) { let creator = post.item.getElementsByTagName('dc:creator')[0] return creator.innerHTML.slice(1) === post.source.user }, pullImages } // TODO: Mastodon support // // "Turns out Mastodon has built-in RSS; your feed URL is [instance]/@[username].rss, so for example I'm // https://mastodon.social/@brownpau.rss (note the "@")" // - https://mastodon.social/@brownpau/100523448408374430