diff --git a/lib.js b/lib.js
index a74baba..8cf349d 100644
--- a/lib.js
+++ b/lib.js
@@ -100,6 +100,7 @@ export async function fetchRss(source) {
let { hostname } = source
let error
let response
+ let rss
try {
response = await delayedFetch(
@@ -114,13 +115,12 @@ export async function fetchRss(source) {
source.errored = error !== undefined || !response.ok
if(source.errored) {
source.error = error ?? createNetworkingError(response)
- source.rss = ''
} else {
- source.rss = await response.text()
+ rss = await response.text()
console.log(`Found ${source.name} at ${hostname}`)
}
- return source
+ return rss
}
let waitingList = new Map()
@@ -160,8 +160,12 @@ export const openCache = async (source, cache) => {
exists = false
}
- if(exists) {
- let rss = await FS.readFile(path, { encoding: 'utf8' })
+ let rss
+
+ if(exists)
+ rss = await FS.readFile(path, { encoding: 'utf8' })
+
+ if(exists & rss) {
let channel = createChannel(rss)
let date = readPubDate(channel.querySelector('pubDate'))
let link = new URL(channel.querySelector('link').textContent)
@@ -200,7 +204,7 @@ export const createCache = (source, cache) => `\
${new Date(source.latestPostDate).toUTCString()}
rssssing
-${source.posts.map(post => post.item.outerHTML.replaceAll(/\n\s*/g, '')).join('\n')}
+${source.items.map(item => item.outerHTML.replaceAll(/\n\s*/g, '')).join('\n')}
@@ -217,7 +221,6 @@ export const readPubDate = (pubDate) =>
class NoMatchesError extends Error {}
export const createPosts = (channel, source, fromDate, reducerCallback) => {
- // let { document } = new JSDOM(rss, { contentType: 'text/xml' }).window
let items = channel.querySelectorAll('item')
// if(items.length === 0) {
@@ -226,6 +229,7 @@ export const createPosts = (channel, source, fromDate, reducerCallback) => {
// }
for(let item of items) {
+ source.items.push(item)
let post = createPost(item, source, reducerCallback)
if(post && post.date > fromDate) {
@@ -504,14 +508,15 @@ export const fetchRssFromInstances = async (source) => {
let index = 0
let instances = source.instances
let cachedLink = source.cache.link
+ let rss
if(cachedLink) {
instances.unshift(cachedLink.hostname)
}
- while(!source.rss && index != instances.length) {
+ while(!rss && index != instances.length) {
source.hostname = instances[index]
- source = await fetchRss(source)
+ rss = await fetchRss(source)
if(source.errored) {
console.error(`Failed to fetch ${source.name} from ${source.hostname}: `, source.error)
@@ -521,11 +526,12 @@ export const fetchRssFromInstances = async (source) => {
}
}
- return source
+ return rss
}
-export const populateSource = (source, postReducerCallback, useCache = true) => {
+export const populateSource = (rss, source, postReducerCallback, useCache = true) => {
let fromDate = 0
+ source.items = []
source.posts = []
if(useCache) {
@@ -542,7 +548,8 @@ export const populateSource = (source, postReducerCallback, useCache = true) =>
return postReducerCallback(post)
}
- source = createPosts(createChannel(source.rss), source, fromDate, remoteReducerCallback)
+ if(rss ?? false)
+ source = createPosts(createChannel(rss), source, fromDate, remoteReducerCallback)
return source
}
@@ -568,8 +575,9 @@ export const writeView = (sources, feeds, view) => {
export const createSource = async (source, getRss, postReducerCallback, cache) => {
source = await openCache(source, cache)
- source = await getRss(source)
- source = populateSource(source, postReducerCallback, cache.populate)
+ source = populateSource(await getRss(source), source, postReducerCallback, cache.populate)
+
+ cache.batch.add(cacheSource(source, cache))
return source
}
@@ -588,7 +596,7 @@ export const tumblr = {
pathname: 'rss',
courtesyWait,
name: `tumblr-${lowercaseUser}`,
- displayName: user,
+ displayName: user + ' (t)',
user: lowercaseUser,
}
@@ -628,7 +636,7 @@ export const nitter = {
pathname: user + '/rss',
courtesyWait,
name: `nitter-${user}`,
- displayName: user,
+ displayName: user + ' (n)',
user
}
@@ -642,7 +650,7 @@ export const nitter = {
isRepost(post) {
let creator = post.item.getElementsByTagName('dc:creator')[0]
- return creator.innerHTML.slice(1) === post.source.user
+ return creator.innerHTML.slice(1) !== post.source.user
},
pullImages