// Ascii font used is "Shimrod" import Path from "path" import FS from "fs/promises" import { JSDOM } from "jsdom" let cache = await FS.readFile('./cache.json', { encoding: 'utf-8' }) .then(json => JSON.parse(json) ) // | o | // . . |- . | ,-. // | | | | | `-. // `-` `-' ' ' `-' export function PromiseBatch() { let promises = [] this.add = (promise) => promises.push(promise) this.complete = () => Promise.all(promises) } export const annotate = annotation => data => { console.log(annotation), data } export const write = async (path, content) => { let dir = Path.dirname(path) try { await FS.access(dir) } catch(e) { await FS.mkdir(dir, { recursive: true }) } return await FS.writeFile(path, content) } export const download = async (url, path, courtesyWait) => { let response = await delayedFetch(url, {}, courtesyWait) .catch(err => console.error(`Failed download of ${url}:`, err) ) if(response.ok) { await write(path, response.body) } else { throw createNetworkingError(response) } } export const createNetworkingError = response => { return new Error(`Request failed, ${response.status}: ${response.statusText}`) } export const getLinkExtname = link => Path.extname(new URL(link).pathname) export const buildImagePathHandler = (source, id) => (url, i, array) => { let path = `images/${source.name}-${id}` if(array.length > 1) path += `-${i}` return path + getLinkExtname(url) } export const addStylesheet = (path, { viewDir, batch }) => batch.add( FS.readFile(path) .then(content => write(Path.join(viewDir, 'style.css'), content)) ) export const postIdFromPathname = post => { let { pathname } = new URL(post.link) return pathname.slice(pathname.lastIndexOf('/') + 1) } export const createLock = async renderer => { let lockExists = false try { await FS.access(renderer.lockPath) lockExists = true } catch(err) { lockExists = false } renderer.lock = { sources: {}, lists: {} } if(lockExists) { let lock = JSON.parse(await FS.readFile(renderer.lockPath, { encoding: 'utf8' })) Object.assign(renderer.lock, lock) } } export const writeLock = renderer => write(renderer.lockPath, JSON.stringify(renderer.lock) ) // // ;-. ,-. ,-. // | `-. `-. // ' `-' `-' class NoMatchesError extends Error {} export const processRss = (source, reducerCallback) => { let { document } = new JSDOM(source.rss, { contentType: 'text/xml' }).window let items = document.querySelectorAll('channel item') if(items.length == 0) { throw new NoMatchesError('Got no matches') } source.posts = [] for(let item of items) { let description = new JSDOM(item.querySelector('description').textContent).window.document let dateString = item.querySelector('pubDate').textContent let link = item.querySelector('link').textContent let guid = item.querySelector('guid').textContent let post = { source, item, description, dateString, date: new Date(dateString).valueOf() ?? 0, link, guid } post = reducerCallback(post) if(post) { source.posts.push(post) } } return source } let waitingList = new Map() export const sleep = delay => new Promise(resolve => setTimeout(() => resolve(), delay) ) export const delayedFetch = async (url, options, courtesyWait = 5 * 1000) => { let [ domain ] = /[\w-]+.[\w-]+$/.exec(url.hostname) let timeout = waitingList.get(domain) ?? 0 let now = Date.now() if(timeout == null || timeout <= now) { waitingList.set(domain, timeout + courtesyWait) } else { await sleep(timeout - now) } return await fetch(url, options) } export async function fetchRss(source) { let { hostname } = source let error let response try { response = await delayedFetch( new URL(source.pathname, 'https://' + hostname), {}, source.courtesyWait ?? 5 * 1000 ) } catch(err) { error = err } source.errored = error !== undefined || !response.ok if(source.errored) { source.error = error ?? createNetworkingError(response) source.rss = '' } else { source.rss = await response.text() } return source } export const extractImages = (post, cache = true) => { let images = post.description.querySelectorAll('img') if(images) { let imageUrls = [] for(let image of images) { let { src } = image if(!src) { let finalSrc = image.srcset.split(', ').pop() src = finalSrc.slice(0, finalSrc.indexOf(' ') ) } // Sending through URL prevents potential XSS imageUrls.push(new URL(src).href) } return imageUrls } } // o // . , . ,-. , , , // |/ | |-' |/|/ // ' ' `-' ' ' export const createPages = (list, { pageSize, header = '', viewDir, batch, getPageFilename, getPageTitle, lock }) => { let posts = [] let lastPageLink = 'about:blank' let pageIndex = 0 // let pageIndex = Math.ceil(list.posts.length / pageSize) // let { // index: pageIndex = 0, // lastPostDate // } = lock.lists[list.name]?.lastPage ?? {} // let sinceDate = posts[0]?.date ?? 0 // posts = list.posts // .filter(post => post.date > sinceDate) // .concat(posts) // .sort((a, b) => b.date - a.date) // let firstPageSize = list.posts.sort((a, b) => a.date - b.date) for(let i = 0; i < list.posts.length; i++) { // for(let i = list.posts.length - 1; i >= 0; i--) { posts.push(list.posts[i]) if(i % pageSize == 0) { let isLastPage = list.main && i < pageSize let title = getPageTitle(list, pageIndex) let html = renderPage(title, posts.reverse(), header, renderNextPageLink(lastPageLink)) let filename = isLastPage ? 'index.html' : getPageFilename(list, pageIndex) let promise = write(Path.join(viewDir, filename), html) batch.add(promise.then(annotate(`Created "${title}" (${filename})`))) posts = [] lastPageLink = filename pageIndex++ } } // lock.lists[list.name] = { // pageIndex, // lastPostDate: posts[0]?.date ?? lastPostDate // } } export const renderPage = (title, posts, header, footer) => `\
${post.source.displayName} ${renderDate(date)} open