diff --git a/assets/style.css b/assets/style.css new file mode 100644 index 0000000..13187d5 --- /dev/null +++ b/assets/style.css @@ -0,0 +1,38 @@ +body { + max-width: 640px; + margin: 0 0 0 auto; + padding: 8px; + font-family: sans-serif; +} + +ul { + padding-inline-start: 30px; + list-style-type: none; +} + +p { + padding: 30px; +} + +img { + margin: 10px auto; + max-width: 100%; +} + +p a, footer a { + float: right +} + +hr { + clear: both +} + +footer a { + padding-bottom: 10px; +} + +@media (prefers-color-scheme: dark) { + body { background: #000; color: #eee } + a { color: #ccf } + hr { border-color: #555 } +} \ No newline at end of file diff --git a/default/config.js b/default/config.js index bc48a5c..6c4983c 100644 --- a/default/config.js +++ b/default/config.js @@ -105,7 +105,7 @@ const sources = { ] } -module.exports = { +export default { feeds, sources, courtesyWait, diff --git a/index.js b/index.js index 57bb71b..a9dd7ac 100644 --- a/index.js +++ b/index.js @@ -175,7 +175,7 @@ const printFeed = async (sources, directory, header, viewOptions, error) => { } } - feed = feed.sort((a, b) => a.date < b.date) + feed = feed.sort((a, b) => a.date > b.date) // Render @@ -187,11 +187,12 @@ const printFeed = async (sources, directory, header, viewOptions, error) => { // Write + let lastIndex = getLastIndex() let promises = [] const writePage = (index, content) => promises.push( - write(Path.join(directory, index == 0 ? 'index' : index.toString() ) + '.html', content) + write(Path.join(directory, index == (feed.length - 1) ? 'index' : index.toString() ) + '.html', content) ) for(let i = 0; i < pages.length; i++) { diff --git a/lib.js b/lib.js new file mode 100644 index 0000000..846eeec --- /dev/null +++ b/lib.js @@ -0,0 +1,489 @@ +// Ascii font used is "Shimrod" + +import Path from "path" +import FS from "fs/promises" +import { JSDOM } from "jsdom" + +let cache = await FS.readFile('./cache.json', { encoding: 'utf-8' }) + .then(json => JSON.parse(json) ) + + +// | o | +// . . |- . | ,-. +// | | | | | `-. +// `-` `-' ' ' `-' + +export function PromiseBatch() { + let promises = [] + + this.add = (promise) => + promises.push(promise) + + this.complete = () => + Promise.all(promises) +} + +export const annotate = annotation => + data => { + console.log(annotation), + data + } + +export const write = async (path, content) => { + let dir = Path.dirname(path) + + try { + await FS.access(dir) + } catch(e) { + await FS.mkdir(dir, { recursive: true }) + } + + return await FS.writeFile(path, content) +} + +export const download = async (url, path, courtesyWait) => { + let response = await delayedFetch(url, {}, courtesyWait) + .catch(err => console.error(`Failed download of ${url}:`, err) ) + + if(response.ok) { + await write(path, response.body) + } else { + throw createNetworkingError(response) + } +} + +export const createNetworkingError = response => { + return new Error(`Request failed, ${response.status}: ${response.statusText}`) +} + +export const getLinkExtname = link => + Path.extname(new URL(link).pathname) + +export const buildImagePathHandler = (source, id) => (url, i, array) => { + let path = `images/${source.name}-${id}` + + if(array.length > 1) + path += `-${i}` + + return path + getLinkExtname(url) +} + +export const addStylesheet = (path, { viewDir, batch }) => + batch.add( + FS.readFile(path) + .then(content => write(Path.join(viewDir, 'style.css'), content)) + ) + +export const postIdFromPathname = post => { + let { pathname } = new URL(post.link) + return pathname.slice(pathname.lastIndexOf('/') + 1) +} + +export const createLock = async renderer => { + let lockExists = false + + try { + await FS.access(renderer.lockPath) + lockExists = true + } catch(err) { + lockExists = false + } + + renderer.lock = { + sources: {}, + lists: {} + } + + if(lockExists) { + let lock = JSON.parse(await FS.readFile(renderer.lockPath, { encoding: 'utf8' })) + + Object.assign(renderer.lock, lock) + } +} + +export const writeLock = renderer => + write(renderer.lockPath, JSON.stringify(renderer.lock) ) + + +// +// ;-. ,-. ,-. +// | `-. `-. +// ' `-' `-' + +class NoMatchesError extends Error {} +export const processRss = (source, reducerCallback) => { + let { document } = new JSDOM(source.rss, { + contentType: 'text/xml' + }).window + let items = document.querySelectorAll('channel item') + + if(items.length == 0) { + throw new NoMatchesError('Got no matches') + } + + source.posts = [] + + for(let item of items) { + let description = new JSDOM(item.querySelector('description').textContent).window.document + let dateString = item.querySelector('pubDate').textContent + let link = item.querySelector('link').textContent + let guid = item.querySelector('guid').textContent + + let post = { + source, + item, + description, + dateString, + date: new Date(dateString).valueOf() ?? 0, + link, + guid + } + + post = reducerCallback(post) + + if(post) { + source.posts.push(post) + } + } + + return source +} + +let waitingList = new Map() +export const sleep = delay => new Promise(resolve => setTimeout(() => resolve(), delay) ) +export const delayedFetch = async (url, options, courtesyWait = 5 * 1000) => { + let [ domain ] = /[\w-]+.[\w-]+$/.exec(url.hostname) + let timeout = waitingList.get(domain) ?? 0 + let now = Date.now() + + if(timeout == null || timeout <= now) { + waitingList.set(domain, timeout + courtesyWait) + } else { + await sleep(timeout - now) + } + + return await fetch(url, options) +} + +export async function fetchRss(source) { + let { hostname } = source + let error + let response + + try { + response = await delayedFetch( + new URL(source.pathname, 'https://' + hostname), + {}, + source.courtesyWait ?? 5 * 1000 + ) + } catch(err) { + error = err + } + + source.errored = error !== undefined || !response.ok + if(source.errored) { + source.error = error ?? createNetworkingError(response) + source.rss = '' + } else { + source.rss = await response.text() + } + + return source +} + +export const extractImages = (post, cache = true) => { + let images = post.description.querySelectorAll('img') + + if(images) { + let imageUrls = [] + + for(let image of images) { + let { src } = image + + if(!src) { + let finalSrc = image.srcset.split(', ').pop() + + src = finalSrc.slice(0, finalSrc.indexOf(' ') ) + } + + // Sending through URL prevents potential XSS + imageUrls.push(new URL(src).href) + } + + return imageUrls + } +} + + +// o +// . , . ,-. , , , +// |/ | |-' |/|/ +// ' ' `-' ' ' + +export const createPages = (list, { pageSize, header = '', viewDir, batch, getPageFilename, getPageTitle, lock }) => { + let posts = [] + let lastPageLink = 'about:blank' + let pageIndex = 0 + // let pageIndex = Math.ceil(list.posts.length / pageSize) + // let { + // index: pageIndex = 0, + // lastPostDate + // } = lock.lists[list.name]?.lastPage ?? {} + + // let sinceDate = posts[0]?.date ?? 0 + // posts = list.posts + // .filter(post => post.date > sinceDate) + // .concat(posts) + // .sort((a, b) => b.date - a.date) + + // let firstPageSize = + + list.posts.sort((a, b) => a.date - b.date) + + for(let i = 0; i < list.posts.length; i++) { + // for(let i = list.posts.length - 1; i >= 0; i--) { + posts.push(list.posts[i]) + + if(i % pageSize == 0) { + let isLastPage = list.main && i < pageSize + let title = getPageTitle(list, pageIndex) + let html = renderPage(title, posts.reverse(), header, renderNextPageLink(lastPageLink)) + let filename = isLastPage ? 'index.html' : getPageFilename(list, pageIndex) + let promise = write(Path.join(viewDir, filename), html) + + batch.add(promise.then(annotate(`Created "${title}" (${filename})`))) + posts = [] + lastPageLink = filename + pageIndex++ + } + } + + // lock.lists[list.name] = { + // pageIndex, + // lastPostDate: posts[0]?.date ?? lastPostDate + // } +} + +export const renderPage = (title, posts, header, footer) => `\ + + + +${title} + + + + + + +
+${header} +
+${posts.map(renderPost).join('\n')} + + + +` + +export const renderPost = post => { + let date = new Date(post.date) + + return `\ +${post.images.map(renderImage).join('\n')} +

${post.source.displayName} ${renderDate(date)} open

` +} + +export const renderImage = href => { + return `\ +` +} + +export const renderDate = date => + (date.getMonth() + 1) + '.' + date.getDate() + '.' + date.getFullYear() + +export const renderNextPageLink = link => `\ +next` + +export const renderNav = (feeds, sources) => `\ +
+ +Feeds +
+ +
+ +
+ +
+
` + +export const renderNavEntry = (list) => { + let extra = '' + + if(list.error) { + extra += ' (errored)' + } else if (list.posts.length == 0) { + extra += ' (empty)' + } + + return `
  • ${list.displayName}${extra}
    • ` +} + + +// | | | o +// ,-. ,-. | | ,-: |- . ,-. ;-. +// | | | | | | | | | | | | | +// `-' `-' ' ' `-` `-' ' `-' ' ' + +export const downloadImages = (images, getImagePath, courtesyWait, { viewDir, batch }) => { + let out = [] + + for(let i = 0; i < images.length; i ++) { + let url = images[i] + let relativePath = getImagePath(url, i, images) + let fullPath = Path.join(viewDir, relativePath) + + let promise = FS.access(fullPath) + .catch(() => + download(url, fullPath, courtesyWait) + .then(annotate( `Downloaded ${relativePath}`)) + ) + + out.push(relativePath) + batch.add(promise) + } + + return out +} + +export const pullImages = (post, renderer, discardPostIfNoImages = false, getPostId = postIdFromPathname) => { + let images = extractImages(post) + + if(!discardPostIfNoImages || images.length > 0) { + post.images = downloadImages( + images, + buildImagePathHandler(post.source, getPostId(post)), + post.source.courtesyWait, + renderer + ) + return post + } +} + +export const createFeed = (name, sources) => { + return { + name, + posts: sources.reduce((posts, source) => posts.concat(source.posts), []) + } +} + +export const fetchRssFromInstances = async (source, renderer) => { + let index = 0 + let instances = source.instances + let lockHostname = renderer.lock.sources[source.name]?.hostname + + if(lockHostname) { + instances.unshift(lockHostname) + } + + while(!source.rss && index != instances.length) { + source.hostname = instances[index] + source = await fetchRss(source) + + if(source.errored) { + console.error(`Failed to fetch ${source.name} from ${source.hostname}: `, source.error) + index++ + } else { + break + } + } + + (renderer.lock.sources[source.name] ??= {}).hostname = source.hostname + + return source +} + +const addPostsToLock = (source, renderer) => { + (renderer.lock.sources[source.name] ??= {}).postData = source.posts.map(post => post.description) +} + + +// | | ,- +// ;-. | ,-: |- | ,-. ;-. ;-.-. ,-. +// | | | | | | |- | | | | | | `-. +// |-' ' `-` `-' | `-' ' ' ' ' `-' +// ' -' + +export const tumblr = { + async createSource(user, courtesyWait, postReducerCallback, renderer) { + let source = { + hostname: user + '.tumblr.com', + pathname: 'rss', + courtesyWait, + name: `tumblr-${user}`, + displayName: user, + user + } + + source = await fetchRss(source) + source = processRss(source, postReducerCallback) + addPostsToLock(source, renderer) + return source + }, + + createSources(users, ...args) { + return Promise.all(users.map(user => tumblr.createSource(user, ...args))) + }, + + isRepost(post) { + let reblog = post.description.querySelector('p > a.tumblr_blog') + + return reblog && reblog.innerHTML !== post.source.user + }, + + pullImages +} + +export const nitter = { + async createSource(user, instances, courtesyWait, postReducerCallback, renderer) { + let source = { + instances, + pathname: user + '/rss', + courtesyWait, + name: `nitter-${user}`, + displayName: user, + user + } + + source = await fetchRssFromInstances(source, renderer) + source = processRss(source, postReducerCallback) + return source + }, + + createSources(users, ...args) { + return Promise.all(users.map(user => nitter.createSource(user, ...args))) + }, + + isRepost(post) { + let creator = post.item.getElementsByTagName('dc:creator')[0] + + return creator.innerHTML.slice(1) === post.source.user + }, + + pullImages +} + +// TODO: Mastodon support +// +// "Turns out Mastodon has built-in RSS; your feed URL is [instance]/@[username].rss, so for example I'm +// https://mastodon.social/@brownpau.rss (note the "@")" +// - https://mastodon.social/@brownpau/100523448408374430 \ No newline at end of file diff --git a/package.json b/package.json index a9d0b55..49f0e01 100644 --- a/package.json +++ b/package.json @@ -10,8 +10,7 @@ "author": "", "license": "ISC", "dependencies": { - "jsdom": "^22.1.0", - "node-fetch": "^3.3.1" + "jsdom": "^22.1.0" }, "type": "module" } diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000..c6691be --- /dev/null +++ b/shell.nix @@ -0,0 +1,13 @@ +{ pkgs ? import {} }: + +pkgs.stdenv.mkDerivation { + name = "rssssing-dev"; + buildInputs = [ + pkgs.nodejs_21 + pkgs.yarn + ]; + shellHook = '' + export PATH="$PWD/node_modules/.bin/:$PATH" + export NPM_PACKAGES="$HOME/.npm-packages" + ''; +} \ No newline at end of file diff --git a/test.html b/test.html new file mode 100644 index 0000000..a48f92b --- /dev/null +++ b/test.html @@ -0,0 +1,43 @@ + + + + + +muses + + + + + + +
    + +
    +
    + +
    +
    art regurgitor
    + + + + + \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 11f9dea..ba8bae8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -38,11 +38,6 @@ cssstyle@^3.0.0: dependencies: rrweb-cssom "^0.6.0" -data-uri-to-buffer@^4.0.0: - version "4.0.1" - resolved "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz" - integrity sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A== - data-urls@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/data-urls/-/data-urls-4.0.0.tgz" @@ -81,14 +76,6 @@ entities@^4.4.0: resolved "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz" integrity sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw== -fetch-blob@^3.1.2, fetch-blob@^3.1.4: - version "3.2.0" - resolved "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz" - integrity sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ== - dependencies: - node-domexception "^1.0.0" - web-streams-polyfill "^3.0.3" - form-data@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz" @@ -98,13 +85,6 @@ form-data@^4.0.0: combined-stream "^1.0.8" mime-types "^2.1.12" -formdata-polyfill@^4.0.10: - version "4.0.10" - resolved "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz" - integrity sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g== - dependencies: - fetch-blob "^3.1.2" - html-encoding-sniffer@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz" @@ -187,20 +167,6 @@ ms@2.1.2: resolved "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -node-domexception@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz" - integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== - -node-fetch@^3.3.1: - version "3.3.2" - resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz" - integrity sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA== - dependencies: - data-uri-to-buffer "^4.0.0" - fetch-blob "^3.1.4" - formdata-polyfill "^4.0.10" - nwsapi@^2.2.4: version "2.2.7" resolved "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.7.tgz" @@ -292,11 +258,6 @@ w3c-xmlserializer@^4.0.0: dependencies: xml-name-validator "^4.0.0" -web-streams-polyfill@^3.0.3: - version "3.2.1" - resolved "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz" - integrity sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q== - webidl-conversions@^7.0.0: version "7.0.0" resolved "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz"