Page ${index + 1}

const { fetch } = require('node-fetch') const config = require('./config.js') let cache = require('./cache.json') let waitingList = new Map() const getMatches = regex => string => { let match let matches = [] while((match = regex.exec(string)) != null) { if (match.index === regex.lastIndex) { regex.lastIndex++; } matches.push(match) } return matches } const handleNitterUser = async user => { let data let index = 0 let sources = cache.nitter[user] ? [ cache.nitter[user] ].concat(config.sources.nitter) : config.sources.nitter while(!data && index < sources.length) { let source = sources[index] if(waitingList.get(source)) { console.log('Waiting...') await sleep(config.courtesyWait) waitingList.set(source, false) } let rss = await fetch('https://' + source + '/' + user + "/rss") .catch(console.error) .then(r => r.text() ) waitingList.set(source, true) try { data = processNitter(user, rss) } catch(err) { console.log(`Failed to fetch ${user} from ${source}`) index++ } } console.log(`Found ${user} at ${sources[index]}`) cache.nitter[user] = sources[index] return data } const sleep = delay => new Promise(resolve => setTimeout(() => resolve(), delay) ) const processNitter = (user, rss) => { const descriptionMatches = getMatches( new RegExp(`\ .*?\ @${user}<\/dc:creator>.*?\ (.+?)<\/description>.*?\ (.+?).*?\ (.*?)<\/link>`, 'sg') )(rss) if(descriptionMatches.length == 0) { throw new Error('Got no matches') return } const getImageMatches = getMatches(/

0) { posts.push({ user, images, date: new Date(date).valueOf(), link }) } } return posts } const print = async feeds => { // Coalate let masterFeed = [] for(let feed of feeds) { masterFeed = masterFeed.concat(feed) } masterFeed = masterFeed.sort((a, b) => a.date < b.date) // Render let pages = [] for(let i = 0; i < Math.ceil(masterFeed.length / config.pageSize); i++) { pages.push(masterFeed.slice(i * config.pageSize, (i + 1) * config.pageSize) ) } // Write console.log('Writing...') for(let i = 0; i < pages.length; i++) { Bun.write('out/' + (i == 0 ? 'index' : i) + '.html', renderPage(pages[i], i) ) } Bun.write('cache.json', JSON.stringify(cache, null, 2)) } const renderPage = (posts, index) => { let html = `\ Page ${index + 1} \n` for(let post of posts) { let date = new Date(post.date) html += `\ ${post.images.map(renderImage).join('\n')}

${post.user} ${date.getMonth()}/${date.getDay()}/${date.getFullYear()} open

\n` } html += ` ` return html } const renderImage = image => `\

` const main = async () => { const feeds = [] console.log('Grabbing posts...') for(let user of config.feeds.nitter) { feeds.push(await handleNitterUser(user) ) } } main()