188 lines
3.6 KiB
JavaScript
188 lines
3.6 KiB
JavaScript
const { fetch } = require('node-fetch')
|
|
const config = require('./config.js')
|
|
|
|
let cache = require('./cache.json')
|
|
|
|
let waitingList = new Map()
|
|
|
|
const getMatches = regex => string => {
|
|
let match
|
|
let matches = []
|
|
|
|
while((match = regex.exec(string)) != null) {
|
|
if (match.index === regex.lastIndex) {
|
|
regex.lastIndex++;
|
|
}
|
|
|
|
matches.push(match)
|
|
}
|
|
|
|
return matches
|
|
}
|
|
|
|
const handleNitterUser = async user => {
|
|
let data
|
|
let index = 0
|
|
let sources = cache.nitter[user] ?
|
|
[ cache.nitter[user] ].concat(config.sources.nitter) :
|
|
config.sources.nitter
|
|
|
|
while(!data && index < sources.length) {
|
|
let source = sources[index]
|
|
|
|
if(waitingList.get(source)) {
|
|
console.log('Waiting...')
|
|
await sleep(config.courtesyWait)
|
|
waitingList.set(source, false)
|
|
}
|
|
|
|
let rss = await fetch('https://' + source + '/' + user + "/rss")
|
|
.catch(console.error)
|
|
.then(r => r.text() )
|
|
|
|
waitingList.set(source, true)
|
|
|
|
try {
|
|
data = processNitter(user, rss)
|
|
} catch(err) {
|
|
console.log(`Failed to fetch ${user} from ${source}`)
|
|
index++
|
|
}
|
|
}
|
|
|
|
console.log(`Found ${user} at ${sources[index]}`)
|
|
cache.nitter[user] = sources[index]
|
|
return data
|
|
}
|
|
|
|
const sleep = delay => new Promise(resolve => setTimeout(() => resolve(), delay) )
|
|
|
|
const processNitter = (user, rss) => {
|
|
const descriptionMatches = getMatches(
|
|
new RegExp(`\
|
|
<item>.*?\
|
|
<dc:creator>@${user}<\/dc:creator>.*?\
|
|
<description>(.+?)<\/description>.*?\
|
|
<pubDate>(.+?)</pubDate>.*?\
|
|
<link>(.*?)<\/link>`, 'sg')
|
|
)(rss)
|
|
|
|
if(descriptionMatches.length == 0) {
|
|
throw new Error('Got no matches')
|
|
return
|
|
}
|
|
|
|
const getImageMatches = getMatches(/<img src="([^]*?)"/g)
|
|
|
|
let posts = []
|
|
|
|
for(let [, description, date, link] of descriptionMatches) {
|
|
let images = []
|
|
|
|
for(let [, url] of getImageMatches(description) ) {
|
|
images.push(url)
|
|
}
|
|
|
|
if(images.length > 0) {
|
|
posts.push({
|
|
user,
|
|
images,
|
|
date: new Date(date).valueOf(),
|
|
link
|
|
})
|
|
}
|
|
}
|
|
|
|
return posts
|
|
}
|
|
|
|
const print = async feeds => {
|
|
// Coalate
|
|
let masterFeed = []
|
|
|
|
for(let feed of feeds) {
|
|
masterFeed = masterFeed.concat(feed)
|
|
}
|
|
|
|
masterFeed = masterFeed.sort((a, b) => a.date < b.date)
|
|
|
|
// Render
|
|
|
|
let pages = []
|
|
|
|
for(let i = 0; i < Math.ceil(masterFeed.length / config.pageSize); i++) {
|
|
pages.push(masterFeed.slice(i * config.pageSize, (i + 1) * config.pageSize) )
|
|
}
|
|
|
|
// Write
|
|
|
|
console.log('Writing...')
|
|
for(let i = 0; i < pages.length; i++) {
|
|
Bun.write('out/' + (i == 0 ? 'index' : i) + '.html', renderPage(pages[i], i) )
|
|
}
|
|
Bun.write('cache.json', JSON.stringify(cache, null, 2))
|
|
}
|
|
|
|
const renderPage = (posts, index) => {
|
|
let html = `\
|
|
<html>
|
|
<head>
|
|
|
|
<title>Page ${index + 1}</title>
|
|
<style>
|
|
body {
|
|
max-width: 640px;
|
|
float: right;
|
|
}
|
|
|
|
p {
|
|
padding: 30px;
|
|
}
|
|
|
|
img {
|
|
margin: 10px auto;
|
|
max-width: 100%;
|
|
}
|
|
|
|
p a, footer a {
|
|
float: right
|
|
}
|
|
|
|
hr {
|
|
clear: both
|
|
}
|
|
</style>
|
|
|
|
</head>
|
|
<body>\n`
|
|
|
|
for(let post of posts) {
|
|
let date = new Date(post.date)
|
|
|
|
html += `\
|
|
${post.images.map(renderImage).join('\n')}
|
|
<p><b>${post.user}</b> ${date.getMonth()}/${date.getDay()}/${date.getFullYear()} <a href="${post.link}">open</a></p><hr>\n`
|
|
}
|
|
|
|
html += `
|
|
<footer>
|
|
<a href="${index + 1}.html">next</a>
|
|
</footer>
|
|
</body>
|
|
</html>`
|
|
return html
|
|
}
|
|
|
|
const renderImage = image => `\
|
|
<a href="${image}"><img src="${image}" loading="lazy"></img></a>`
|
|
|
|
const main = async () => {
|
|
const feeds = []
|
|
|
|
console.log('Grabbing posts...')
|
|
for(let user of config.feeds.nitter) {
|
|
feeds.push(await handleNitterUser(user) )
|
|
}
|
|
}
|
|
|
|
main() |