commit 85afb7798a5e1f65562b2e04fe0df38991d3ff84 Author: dakedres Date: Sat Jul 22 21:05:15 2023 -0400 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..af705ae --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +# Node +node_modules/ + +# Artifacts +out/* +cache.json +config.js \ No newline at end of file diff --git a/README b/README new file mode 100644 index 0000000..a24f8a8 --- /dev/null +++ b/README @@ -0,0 +1,10 @@ +Do not expect quality code. Only works with the Bun[1] runtime + +Usage +- Run `bun run setup`, `npm run setup`, etc. +- Add usernames and such to the "feeds" array in config.js +- `bun .` +- Open `out/index.html` in a browser. Enjoy :) + +References +1. https://bun.sh \ No newline at end of file diff --git a/bun.lockb b/bun.lockb new file mode 100755 index 0000000..371f04e Binary files /dev/null and b/bun.lockb differ diff --git a/index.js b/index.js new file mode 100644 index 0000000..7e4cb12 --- /dev/null +++ b/index.js @@ -0,0 +1,193 @@ +const { fetch } = require('node-fetch') +const config = require('./config.js') + +let cache = require('./cache.json') + +let waitingList = new Map() + +const getMatches = regex => string => { + let match + let matches = [] + + while((match = regex.exec(string)) != null) { + if (match.index === regex.lastIndex) { + regex.lastIndex++; + } + + matches.push(match) + } + + return matches +} + +const handleNitterUser = async user => { + let data + let index = 0 + let sources = cache.nitter[user] ? + [ cache.nitter[user] ].concat(config.sources.nitter) : + config.sources.nitter + + while(!data && index < sources.length) { + let source = sources[index] + + if(waitingList.get(source)) { + console.log('Waiting...') + await sleep(config.courtesyWait) + waitingList.set(source, false) + } + + let rss = await fetch('https://' + source + '/' + user + "/rss") + .catch(console.error) + .then(r => r.text() ) + + waitingList.set(source, true) + + try { + data = processNitter(user, rss) + } catch(err) { + console.log(`Failed to fetch ${user} from ${source}`) + index++ + } + } + + console.log(`Found ${user} at ${sources[index]}`) + cache.nitter[user] = sources[index] + return data +} + +const sleep = delay => new Promise(resolve => setTimeout(() => resolve(), delay) ) + +const processNitter = (user, rss) => { + const descriptionMatches = getMatches( + new RegExp(`\ +.*?\ +@${user}<\/dc:creator>.*?\ +(.+?)<\/description>.*?\ +(.+?).*?\ +(.*?)<\/link>`, 'sg') + )(rss) + + if(descriptionMatches.length == 0) { + throw new Error('Got no matches') + return + } + + const getImageMatches = getMatches(/ 0) { + posts.push({ + user, + images, + date: new Date(date).valueOf(), + link + }) + } + } + + return posts +} + +const print = async feeds => { + // Coalate + let masterFeed = [] + + for(let feed of feeds) { + masterFeed = masterFeed.concat(feed) + } + + masterFeed = masterFeed.sort((a, b) => a.date < b.date) + + // Render + + let pages = [] + + for(let i = 0; i < Math.ceil(masterFeed.length / config.pageSize); i++) { + pages.push(masterFeed.slice(i * config.pageSize, (i + 1) * config.pageSize) ) + } + + // Write + + console.log('Writing...') + for(let i = 0; i < pages.length; i++) { + Bun.write('out/' + (i == 0 ? 'index' : i) + '.html', renderPage(pages[i], i) ) + } + Bun.write('cache.json', JSON.stringify(cache, null, 2)) +} + +const renderPage = (posts, index) => { + let html = `\ + + + +Page ${index + 1} + + + +\n` + + for(let post of posts) { + let date = new Date(post.date) + + html += `\ +${post.images.map(renderImage).join('\n')} +

${post.user} ${date.getMonth()}/${date.getDay()}/${date.getFullYear()} open

\n` + } + + html += ` + + +` + return html +} + +const renderImage = image => `\ +` + +const main = async () => { + const feeds = [] + + console.log('Grabbing posts...') + for(let user of config.feeds.nitter) { + feeds.push(await handleNitterUser(user) ) + } +} + +main() + +// Promise.all([ +// ...config.feeds.nitter.map(handleNitterUser) +// ]) +// .then(print) \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..0b31579 --- /dev/null +++ b/package.json @@ -0,0 +1,15 @@ +{ + "name": "rssssing", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "setup": "mkdir out && cp -r default/* ." + }, + "author": "", + "license": "ISC", + "type": "module", + "dependencies": { + "node-fetch": "^3.3.1" + } +}