v2 mostly working

This commit is contained in:
dakedres 2024-02-07 19:15:12 -07:00
parent 040bd8f6fa
commit 2288085649
8 changed files with 588 additions and 44 deletions

38
assets/style.css Normal file
View File

@ -0,0 +1,38 @@
body {
max-width: 640px;
margin: 0 0 0 auto;
padding: 8px;
font-family: sans-serif;
}
ul {
padding-inline-start: 30px;
list-style-type: none;
}
p {
padding: 30px;
}
img {
margin: 10px auto;
max-width: 100%;
}
p a, footer a {
float: right
}
hr {
clear: both
}
footer a {
padding-bottom: 10px;
}
@media (prefers-color-scheme: dark) {
body { background: #000; color: #eee }
a { color: #ccf }
hr { border-color: #555 }
}

View File

@ -105,7 +105,7 @@ const sources = {
]
}
module.exports = {
export default {
feeds,
sources,
courtesyWait,

View File

@ -175,7 +175,7 @@ const printFeed = async (sources, directory, header, viewOptions, error) => {
}
}
feed = feed.sort((a, b) => a.date < b.date)
feed = feed.sort((a, b) => a.date > b.date)
// Render
@ -187,11 +187,12 @@ const printFeed = async (sources, directory, header, viewOptions, error) => {
// Write
let lastIndex = getLastIndex()
let promises = []
const writePage = (index, content) =>
promises.push(
write(Path.join(directory, index == 0 ? 'index' : index.toString() ) + '.html', content)
write(Path.join(directory, index == (feed.length - 1) ? 'index' : index.toString() ) + '.html', content)
)
for(let i = 0; i < pages.length; i++) {

489
lib.js Normal file
View File

@ -0,0 +1,489 @@
// Ascii font used is "Shimrod"
import Path from "path"
import FS from "fs/promises"
import { JSDOM } from "jsdom"
let cache = await FS.readFile('./cache.json', { encoding: 'utf-8' })
.then(json => JSON.parse(json) )
// | o |
// . . |- . | ,-.
// | | | | | `-.
// `-` `-' ' ' `-'
export function PromiseBatch() {
let promises = []
this.add = (promise) =>
promises.push(promise)
this.complete = () =>
Promise.all(promises)
}
export const annotate = annotation =>
data => {
console.log(annotation),
data
}
export const write = async (path, content) => {
let dir = Path.dirname(path)
try {
await FS.access(dir)
} catch(e) {
await FS.mkdir(dir, { recursive: true })
}
return await FS.writeFile(path, content)
}
export const download = async (url, path, courtesyWait) => {
let response = await delayedFetch(url, {}, courtesyWait)
.catch(err => console.error(`Failed download of ${url}:`, err) )
if(response.ok) {
await write(path, response.body)
} else {
throw createNetworkingError(response)
}
}
export const createNetworkingError = response => {
return new Error(`Request failed, ${response.status}: ${response.statusText}`)
}
export const getLinkExtname = link =>
Path.extname(new URL(link).pathname)
export const buildImagePathHandler = (source, id) => (url, i, array) => {
let path = `images/${source.name}-${id}`
if(array.length > 1)
path += `-${i}`
return path + getLinkExtname(url)
}
export const addStylesheet = (path, { viewDir, batch }) =>
batch.add(
FS.readFile(path)
.then(content => write(Path.join(viewDir, 'style.css'), content))
)
export const postIdFromPathname = post => {
let { pathname } = new URL(post.link)
return pathname.slice(pathname.lastIndexOf('/') + 1)
}
export const createLock = async renderer => {
let lockExists = false
try {
await FS.access(renderer.lockPath)
lockExists = true
} catch(err) {
lockExists = false
}
renderer.lock = {
sources: {},
lists: {}
}
if(lockExists) {
let lock = JSON.parse(await FS.readFile(renderer.lockPath, { encoding: 'utf8' }))
Object.assign(renderer.lock, lock)
}
}
export const writeLock = renderer =>
write(renderer.lockPath, JSON.stringify(renderer.lock) )
//
// ;-. ,-. ,-.
// | `-. `-.
// ' `-' `-'
class NoMatchesError extends Error {}
export const processRss = (source, reducerCallback) => {
let { document } = new JSDOM(source.rss, {
contentType: 'text/xml'
}).window
let items = document.querySelectorAll('channel item')
if(items.length == 0) {
throw new NoMatchesError('Got no matches')
}
source.posts = []
for(let item of items) {
let description = new JSDOM(item.querySelector('description').textContent).window.document
let dateString = item.querySelector('pubDate').textContent
let link = item.querySelector('link').textContent
let guid = item.querySelector('guid').textContent
let post = {
source,
item,
description,
dateString,
date: new Date(dateString).valueOf() ?? 0,
link,
guid
}
post = reducerCallback(post)
if(post) {
source.posts.push(post)
}
}
return source
}
let waitingList = new Map()
export const sleep = delay => new Promise(resolve => setTimeout(() => resolve(), delay) )
export const delayedFetch = async (url, options, courtesyWait = 5 * 1000) => {
let [ domain ] = /[\w-]+.[\w-]+$/.exec(url.hostname)
let timeout = waitingList.get(domain) ?? 0
let now = Date.now()
if(timeout == null || timeout <= now) {
waitingList.set(domain, timeout + courtesyWait)
} else {
await sleep(timeout - now)
}
return await fetch(url, options)
}
export async function fetchRss(source) {
let { hostname } = source
let error
let response
try {
response = await delayedFetch(
new URL(source.pathname, 'https://' + hostname),
{},
source.courtesyWait ?? 5 * 1000
)
} catch(err) {
error = err
}
source.errored = error !== undefined || !response.ok
if(source.errored) {
source.error = error ?? createNetworkingError(response)
source.rss = ''
} else {
source.rss = await response.text()
}
return source
}
export const extractImages = (post, cache = true) => {
let images = post.description.querySelectorAll('img')
if(images) {
let imageUrls = []
for(let image of images) {
let { src } = image
if(!src) {
let finalSrc = image.srcset.split(', ').pop()
src = finalSrc.slice(0, finalSrc.indexOf(' ') )
}
// Sending through URL prevents potential XSS
imageUrls.push(new URL(src).href)
}
return imageUrls
}
}
// o
// . , . ,-. , , ,
// |/ | |-' |/|/
// ' ' `-' ' '
export const createPages = (list, { pageSize, header = '', viewDir, batch, getPageFilename, getPageTitle, lock }) => {
let posts = []
let lastPageLink = 'about:blank'
let pageIndex = 0
// let pageIndex = Math.ceil(list.posts.length / pageSize)
// let {
// index: pageIndex = 0,
// lastPostDate
// } = lock.lists[list.name]?.lastPage ?? {}
// let sinceDate = posts[0]?.date ?? 0
// posts = list.posts
// .filter(post => post.date > sinceDate)
// .concat(posts)
// .sort((a, b) => b.date - a.date)
// let firstPageSize =
list.posts.sort((a, b) => a.date - b.date)
for(let i = 0; i < list.posts.length; i++) {
// for(let i = list.posts.length - 1; i >= 0; i--) {
posts.push(list.posts[i])
if(i % pageSize == 0) {
let isLastPage = list.main && i < pageSize
let title = getPageTitle(list, pageIndex)
let html = renderPage(title, posts.reverse(), header, renderNextPageLink(lastPageLink))
let filename = isLastPage ? 'index.html' : getPageFilename(list, pageIndex)
let promise = write(Path.join(viewDir, filename), html)
batch.add(promise.then(annotate(`Created "${title}" (${filename})`)))
posts = []
lastPageLink = filename
pageIndex++
}
}
// lock.lists[list.name] = {
// pageIndex,
// lastPostDate: posts[0]?.date ?? lastPostDate
// }
}
export const renderPage = (title, posts, header, footer) => `\
<html>
<head>
<title>${title}</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="./style.css">
</head>
<body>
<header>
${header}
</header>
${posts.map(renderPost).join('\n')}
<footer>
${footer}
</footer>
</body>
</html>`
export const renderPost = post => {
let date = new Date(post.date)
return `\
${post.images.map(renderImage).join('\n')}
<p><b>${post.source.displayName}</b> ${renderDate(date)} <a href="${post.link}">open</a></p><hr>`
}
export const renderImage = href => {
return `\
<a href="${href}"><img src="${href}" loading="lazy"></img></a>`
}
export const renderDate = date =>
(date.getMonth() + 1) + '.' + date.getDate() + '.' + date.getFullYear()
export const renderNextPageLink = link => `\
<a href="${link}">next</a>`
export const renderNav = (feeds, sources) => `\
<details>
<summary>Feeds</summary>
<section>
<ul>
${feeds.map(renderNavEntry).join('\n')}
</ul>
<hr>
<ul>
${sources.map(renderNavEntry).join('\n')}
</ul>
</section>
</details>
<hr>`
export const renderNavEntry = (list) => {
let extra = ''
if(list.error) {
extra += ' (errored)'
} else if (list.posts.length == 0) {
extra += ' (empty)'
}
return `<li><a href="${list.link}">${list.displayName}</a>${extra}</li>`
}
// | | | o
// ,-. ,-. | | ,-: |- . ,-. ;-.
// | | | | | | | | | | | | |
// `-' `-' ' ' `-` `-' ' `-' ' '
export const downloadImages = (images, getImagePath, courtesyWait, { viewDir, batch }) => {
let out = []
for(let i = 0; i < images.length; i ++) {
let url = images[i]
let relativePath = getImagePath(url, i, images)
let fullPath = Path.join(viewDir, relativePath)
let promise = FS.access(fullPath)
.catch(() =>
download(url, fullPath, courtesyWait)
.then(annotate( `Downloaded ${relativePath}`))
)
out.push(relativePath)
batch.add(promise)
}
return out
}
export const pullImages = (post, renderer, discardPostIfNoImages = false, getPostId = postIdFromPathname) => {
let images = extractImages(post)
if(!discardPostIfNoImages || images.length > 0) {
post.images = downloadImages(
images,
buildImagePathHandler(post.source, getPostId(post)),
post.source.courtesyWait,
renderer
)
return post
}
}
export const createFeed = (name, sources) => {
return {
name,
posts: sources.reduce((posts, source) => posts.concat(source.posts), [])
}
}
export const fetchRssFromInstances = async (source, renderer) => {
let index = 0
let instances = source.instances
let lockHostname = renderer.lock.sources[source.name]?.hostname
if(lockHostname) {
instances.unshift(lockHostname)
}
while(!source.rss && index != instances.length) {
source.hostname = instances[index]
source = await fetchRss(source)
if(source.errored) {
console.error(`Failed to fetch ${source.name} from ${source.hostname}: `, source.error)
index++
} else {
break
}
}
(renderer.lock.sources[source.name] ??= {}).hostname = source.hostname
return source
}
const addPostsToLock = (source, renderer) => {
(renderer.lock.sources[source.name] ??= {}).postData = source.posts.map(post => post.description)
}
// | | ,-
// ;-. | ,-: |- | ,-. ;-. ;-.-. ,-.
// | | | | | | |- | | | | | | `-.
// |-' ' `-` `-' | `-' ' ' ' ' `-'
// ' -'
export const tumblr = {
async createSource(user, courtesyWait, postReducerCallback, renderer) {
let source = {
hostname: user + '.tumblr.com',
pathname: 'rss',
courtesyWait,
name: `tumblr-${user}`,
displayName: user,
user
}
source = await fetchRss(source)
source = processRss(source, postReducerCallback)
addPostsToLock(source, renderer)
return source
},
createSources(users, ...args) {
return Promise.all(users.map(user => tumblr.createSource(user, ...args)))
},
isRepost(post) {
let reblog = post.description.querySelector('p > a.tumblr_blog')
return reblog && reblog.innerHTML !== post.source.user
},
pullImages
}
export const nitter = {
async createSource(user, instances, courtesyWait, postReducerCallback, renderer) {
let source = {
instances,
pathname: user + '/rss',
courtesyWait,
name: `nitter-${user}`,
displayName: user,
user
}
source = await fetchRssFromInstances(source, renderer)
source = processRss(source, postReducerCallback)
return source
},
createSources(users, ...args) {
return Promise.all(users.map(user => nitter.createSource(user, ...args)))
},
isRepost(post) {
let creator = post.item.getElementsByTagName('dc:creator')[0]
return creator.innerHTML.slice(1) === post.source.user
},
pullImages
}
// TODO: Mastodon support
//
// "Turns out Mastodon has built-in RSS; your feed URL is [instance]/@[username].rss, so for example I'm
// https://mastodon.social/@brownpau.rss (note the "@")"
// - https://mastodon.social/@brownpau/100523448408374430

View File

@ -10,8 +10,7 @@
"author": "",
"license": "ISC",
"dependencies": {
"jsdom": "^22.1.0",
"node-fetch": "^3.3.1"
"jsdom": "^22.1.0"
},
"type": "module"
}

13
shell.nix Normal file
View File

@ -0,0 +1,13 @@
{ pkgs ? import <nixpkgs> {} }:
pkgs.stdenv.mkDerivation {
name = "rssssing-dev";
buildInputs = [
pkgs.nodejs_21
pkgs.yarn
];
shellHook = ''
export PATH="$PWD/node_modules/.bin/:$PATH"
export NPM_PACKAGES="$HOME/.npm-packages"
'';
}

43
test.html Normal file
View File

@ -0,0 +1,43 @@
<html>
<head>
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>muses</title>
<link rel="stylesheet" href="style.css">
<style>
</style>
</head>
<body>
<main>
<article>
</article>
</main>
<header><a href="index.html">art regurgitor</a></header>
<nav>
<ul>
<li><a href="index.html">index</a></li>
<li><a href="meta.html">meta</a></li>
<li><a href="muses.html"><i>muses</i></a></li>
<li><a href="musings.html">musings</a></li>
</ul>
<ul>
<li><a href="computing.html">computing</a></li>
</ul>
<hr><p><a href="https://git.sys42.net/dakedres/website/raw/branch/v2/site/muses.md">[source]</a></p>
</nav>
</body>
</html>

View File

@ -38,11 +38,6 @@ cssstyle@^3.0.0:
dependencies:
rrweb-cssom "^0.6.0"
data-uri-to-buffer@^4.0.0:
version "4.0.1"
resolved "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz"
integrity sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==
data-urls@^4.0.0:
version "4.0.0"
resolved "https://registry.npmjs.org/data-urls/-/data-urls-4.0.0.tgz"
@ -81,14 +76,6 @@ entities@^4.4.0:
resolved "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz"
integrity sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==
fetch-blob@^3.1.2, fetch-blob@^3.1.4:
version "3.2.0"
resolved "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz"
integrity sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==
dependencies:
node-domexception "^1.0.0"
web-streams-polyfill "^3.0.3"
form-data@^4.0.0:
version "4.0.0"
resolved "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz"
@ -98,13 +85,6 @@ form-data@^4.0.0:
combined-stream "^1.0.8"
mime-types "^2.1.12"
formdata-polyfill@^4.0.10:
version "4.0.10"
resolved "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz"
integrity sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==
dependencies:
fetch-blob "^3.1.2"
html-encoding-sniffer@^3.0.0:
version "3.0.0"
resolved "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz"
@ -187,20 +167,6 @@ ms@2.1.2:
resolved "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz"
integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==
node-domexception@^1.0.0:
version "1.0.0"
resolved "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz"
integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==
node-fetch@^3.3.1:
version "3.3.2"
resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz"
integrity sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==
dependencies:
data-uri-to-buffer "^4.0.0"
fetch-blob "^3.1.4"
formdata-polyfill "^4.0.10"
nwsapi@^2.2.4:
version "2.2.7"
resolved "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.7.tgz"
@ -292,11 +258,6 @@ w3c-xmlserializer@^4.0.0:
dependencies:
xml-name-validator "^4.0.0"
web-streams-polyfill@^3.0.3:
version "3.2.1"
resolved "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz"
integrity sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==
webidl-conversions@^7.0.0:
version "7.0.0"
resolved "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz"