Reordering of codebase
This commit is contained in:
parent
0ee7c8d016
commit
b58680d249
458
lib.js
Executable file → Normal file
458
lib.js
Executable file → Normal file
@ -1,9 +1,9 @@
|
|||||||
// Ascii font used is "Shimrod"
|
|
||||||
|
|
||||||
import Path from "path"
|
import Path from "path"
|
||||||
import FS from "fs/promises"
|
import FS from "fs/promises"
|
||||||
import { JSDOM } from "jsdom"
|
import { JSDOM } from "jsdom"
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// | o |
|
// | o |
|
||||||
// . . |- . | ,-.
|
// . . |- . | ,-.
|
||||||
// | | | | | `-.
|
// | | | | | `-.
|
||||||
@ -89,6 +89,39 @@ export const isUnset = (value) => {
|
|||||||
return typeof value === "undefined" || value === null
|
return typeof value === "undefined" || value === null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let waitingList = new Map()
|
||||||
|
export const sleep = delay => new Promise(resolve => setTimeout(() => resolve(), delay) )
|
||||||
|
|
||||||
|
export const delayedFetch = async (url, options, courtesyWait = 5 * 1000) => {
|
||||||
|
let [ domain ] = /[\w-]+.[\w-]+$/.exec(new URL(url).hostname)
|
||||||
|
let waitFor = waitingList.get(domain) ?? 0
|
||||||
|
|
||||||
|
waitingList.set(domain, waitFor + courtesyWait)
|
||||||
|
if(waitFor !== 0) {
|
||||||
|
console.log(`Waiting ${waitFor}ms to download ${url}`)
|
||||||
|
await sleep(waitFor)
|
||||||
|
}
|
||||||
|
|
||||||
|
return await fetch(url, options)
|
||||||
|
}
|
||||||
|
|
||||||
|
export const retryDelayedFetch = async (url, options, courtesyWait, retryAttempts) => {
|
||||||
|
let attemptsTried = 0
|
||||||
|
let response = undefined
|
||||||
|
|
||||||
|
while(isUnset(response) && attemptsTried <= (retryAttempts ?? 3)) {
|
||||||
|
if(attemptsTried > 0)
|
||||||
|
console.error(`Failed to fetch ${url}, retrying...`)
|
||||||
|
|
||||||
|
response = await delayedFetch(url, options, courtesyWait)
|
||||||
|
attemptsTried++
|
||||||
|
}
|
||||||
|
|
||||||
|
return response
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
//
|
//
|
||||||
// ;-. ,-. ,-.
|
// ;-. ,-. ,-.
|
||||||
// | `-. `-.
|
// | `-. `-.
|
||||||
@ -134,36 +167,113 @@ export async function fetchChannel(source) {
|
|||||||
return channel
|
return channel
|
||||||
}
|
}
|
||||||
|
|
||||||
let waitingList = new Map()
|
export const createChannel = rss => {
|
||||||
export const sleep = delay => new Promise(resolve => setTimeout(() => resolve(), delay) )
|
let { document } = new JSDOM(rss, { contentType: 'text/xml' }).window
|
||||||
export const delayedFetch = async (url, options, courtesyWait = 5 * 1000) => {
|
|
||||||
let [ domain ] = /[\w-]+.[\w-]+$/.exec(new URL(url).hostname)
|
|
||||||
let waitFor = waitingList.get(domain) ?? 0
|
|
||||||
|
|
||||||
waitingList.set(domain, waitFor + courtesyWait)
|
return document.querySelector('channel')
|
||||||
if(waitFor !== 0) {
|
|
||||||
console.log(`Waiting ${waitFor}ms to download ${url}`)
|
|
||||||
await sleep(waitFor)
|
|
||||||
}
|
|
||||||
|
|
||||||
return await fetch(url, options)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export const retryDelayedFetch = async (url, options, courtesyWait, retryAttempts) => {
|
export const readPubDate = (pubDate) =>
|
||||||
let attemptsTried = 0
|
pubDate ? new Date(pubDate.textContent).valueOf() : 0
|
||||||
let response = undefined
|
|
||||||
|
|
||||||
while(isUnset(response) && attemptsTried <= (retryAttempts ?? 3)) {
|
export const createPosts = async (channel, source, fromDate, reducerCallback) => {
|
||||||
if(attemptsTried > 0)
|
let items = channel.querySelectorAll('item')
|
||||||
console.error(`Failed to fetch ${url}, retrying...`)
|
|
||||||
|
|
||||||
response = await delayedFetch(url, options, courtesyWait)
|
let promises = []
|
||||||
attemptsTried++
|
|
||||||
|
for(let item of items) {
|
||||||
|
let post = createPost(item, source)
|
||||||
|
|
||||||
|
if(post.date <= fromDate)
|
||||||
|
continue
|
||||||
|
|
||||||
|
source.items.push(item)
|
||||||
|
|
||||||
|
let postResolvable = reducerCallback(post)
|
||||||
|
|
||||||
|
if(postResolvable instanceof Promise) {
|
||||||
|
postResolvable
|
||||||
|
.then(post => {
|
||||||
|
if(post) {
|
||||||
|
source.posts.push(post)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
if(postResolvable) {
|
||||||
|
source.posts.push(postResolvable)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
promises.push(postResolvable)
|
||||||
}
|
}
|
||||||
|
|
||||||
return response
|
await Promise.all(promises)
|
||||||
|
return source
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export const createPost = (item, source) => {
|
||||||
|
let description = new JSDOM(item.querySelector('description').textContent).window.document
|
||||||
|
let date = readPubDate(item.querySelector('pubDate'))
|
||||||
|
let link = item.querySelector('link').textContent
|
||||||
|
let guid = item.querySelector('guid')?.textContent
|
||||||
|
let title = item.querySelector('title')?.textContent
|
||||||
|
|
||||||
|
let post = {
|
||||||
|
source,
|
||||||
|
item,
|
||||||
|
description,
|
||||||
|
date,
|
||||||
|
link,
|
||||||
|
guid,
|
||||||
|
title,
|
||||||
|
occurances: []
|
||||||
|
}
|
||||||
|
|
||||||
|
return post
|
||||||
|
}
|
||||||
|
|
||||||
|
export const extractImages = (post) => {
|
||||||
|
let images = post.description.querySelectorAll('img')
|
||||||
|
|
||||||
|
if(images) {
|
||||||
|
let imageUrls = []
|
||||||
|
|
||||||
|
for(let image of images) {
|
||||||
|
let { src } = image
|
||||||
|
|
||||||
|
if(isUnset(src)) {
|
||||||
|
let finalSrc = image.srcset.split(', ').pop()
|
||||||
|
|
||||||
|
src = finalSrc.slice(0, finalSrc.indexOf(' ') )
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sending through URL prevents potential XSS
|
||||||
|
imageUrls.push(new URL(src).href)
|
||||||
|
}
|
||||||
|
|
||||||
|
return imageUrls
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export const processCategories = (post) => {
|
||||||
|
let categoryMatches = post.item.querySelectorAll('category')
|
||||||
|
post.categories = []
|
||||||
|
|
||||||
|
for(let category of categoryMatches) {
|
||||||
|
post.categories.push(category.textContent)
|
||||||
|
}
|
||||||
|
|
||||||
|
return post
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
// .
|
||||||
|
// |
|
||||||
|
// ,-. ,-: ,-. |-. ,-.
|
||||||
|
// | | | | | | |-'
|
||||||
|
// `-' `-` `-' ' ' `-'
|
||||||
|
|
||||||
export const createCache = async (cache = {}) => {
|
export const createCache = async (cache = {}) => {
|
||||||
if(isUnset(cache.enabled)) {
|
if(isUnset(cache.enabled)) {
|
||||||
cache.enabled = false
|
cache.enabled = false
|
||||||
@ -247,106 +357,102 @@ ${source.items.map(item => item.outerHTML.replaceAll(/\n\s*/g, '')).join('\n')}
|
|||||||
|
|
||||||
</rss>`
|
</rss>`
|
||||||
|
|
||||||
export const createChannel = rss => {
|
|
||||||
let { document } = new JSDOM(rss, { contentType: 'text/xml' }).window
|
|
||||||
|
|
||||||
return document.querySelector('channel')
|
|
||||||
|
// | | | o
|
||||||
|
// ,-. ,-. | | ,-: |- . ,-. ;-.
|
||||||
|
// | | | | | | | | | | | | |
|
||||||
|
// `-' `-' ' ' `-` `-' ' `-' ' '
|
||||||
|
|
||||||
|
export const createFeed = (name, sources, main = false) => {
|
||||||
|
return {
|
||||||
|
name,
|
||||||
|
displayName: name,
|
||||||
|
main,
|
||||||
|
posts: sources.reduce((posts, source) => posts.concat(source.posts), [])
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export const readPubDate = (pubDate) =>
|
export const downloadImage = async (url, basename, source, view) => {
|
||||||
pubDate ? new Date(pubDate.textContent).valueOf() : 0
|
let response = await retryDelayedFetch(url, {}, source.courtesyWait, source.retryAttempts)
|
||||||
|
.catch(err => console.error(`Failed download of ${url}:`, err, err.errors) )
|
||||||
|
|
||||||
export const createPosts = async (channel, source, fromDate, reducerCallback) => {
|
if(response == undefined) {
|
||||||
let items = channel.querySelectorAll('item')
|
console.error('Could not download image: ' + url)
|
||||||
|
return url
|
||||||
let promises = []
|
|
||||||
|
|
||||||
for(let item of items) {
|
|
||||||
let post = createPost(item, source)
|
|
||||||
|
|
||||||
if(post.date <= fromDate)
|
|
||||||
continue
|
|
||||||
|
|
||||||
source.items.push(item)
|
|
||||||
|
|
||||||
let postResolvable = reducerCallback(post)
|
|
||||||
|
|
||||||
if(postResolvable instanceof Promise) {
|
|
||||||
postResolvable
|
|
||||||
.then(post => {
|
|
||||||
if(post) {
|
|
||||||
source.posts.push(post)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
if(postResolvable) {
|
|
||||||
source.posts.push(postResolvable)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
promises.push(postResolvable)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
await Promise.all(promises)
|
if(response.ok) {
|
||||||
return source
|
let mimetype = response.headers.get('Content-Type').split(';')[0]
|
||||||
|
let extension = imageExtensions[mimetype]
|
||||||
|
|
||||||
|
if(typeof extension !== 'string') {
|
||||||
|
console.error(`Unknown mimetype for ${url}: ${mimetype}. Cannot download`)
|
||||||
|
return url
|
||||||
|
}
|
||||||
|
|
||||||
|
let pathname = Path.join(view.imageStoreDirectory, basename + extension)
|
||||||
|
let path = Path.join(view.path, pathname)
|
||||||
|
|
||||||
|
const download = () => write(path, response.body)
|
||||||
|
.then(annotate( `Downloaded ${pathname}`))
|
||||||
|
|
||||||
|
// TODO: See if the image is downloaded before even trying to download it
|
||||||
|
view.batch.add(FS.access(path).catch(download))
|
||||||
|
return pathname
|
||||||
|
} else {
|
||||||
|
console.error( createNetworkingError(response) )
|
||||||
|
return url
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export const createPost = (item, source) => {
|
export const downloadImages = (images, source, postId, view) => {
|
||||||
let description = new JSDOM(item.querySelector('description').textContent).window.document
|
let basePath = getImageBasePath(source, postId)
|
||||||
let date = readPubDate(item.querySelector('pubDate'))
|
let pathnames = []
|
||||||
let link = item.querySelector('link').textContent
|
|
||||||
let guid = item.querySelector('guid')?.textContent
|
|
||||||
let title = item.querySelector('title')?.textContent
|
|
||||||
|
|
||||||
let post = {
|
for(let i = 0; i < images.length; i++) {
|
||||||
source,
|
let basename = images.length > 1 ? basePath + '-' + i : basePath
|
||||||
item,
|
let pathname = view.imageStore.get(basename)
|
||||||
description,
|
|
||||||
date,
|
if(isUnset(pathname)) {
|
||||||
link,
|
pathname = downloadImage(images[i], basename, source, view)
|
||||||
guid,
|
}
|
||||||
title,
|
|
||||||
occurances: []
|
pathnames.push(pathname)
|
||||||
}
|
}
|
||||||
|
|
||||||
return post
|
return Promise.all(pathnames)
|
||||||
}
|
}
|
||||||
|
|
||||||
export const processCategories = (post) => {
|
export const imageExtensions = {
|
||||||
let categoryMatches = post.item.querySelectorAll('category')
|
'image/apng': '.apng',
|
||||||
post.categories = []
|
'image/avif': '.avif',
|
||||||
|
'image/bmp': '.bmp',
|
||||||
for(let category of categoryMatches) {
|
'image/gif': '.gif',
|
||||||
post.categories.push(category.textContent)
|
'image/vnd.microsoft.icon': '.icon',
|
||||||
}
|
'image/jpeg': '.jpg',
|
||||||
|
'image/png': '.png',
|
||||||
return post
|
'image/svg+xml': '.xml',
|
||||||
|
'image/tiff': '.tif',
|
||||||
|
'image/webp': '.webp'
|
||||||
}
|
}
|
||||||
|
|
||||||
export const extractImages = (post) => {
|
export const pullImages = async (post, view, discardPostIfNoImages = false, getPostId = getPostIdFromPathname) => {
|
||||||
let images = post.description.querySelectorAll('img')
|
let images = extractImages(post)
|
||||||
|
|
||||||
if(images) {
|
if(!discardPostIfNoImages || images.length > 0) {
|
||||||
let imageUrls = []
|
post.images = await downloadImages(
|
||||||
|
images,
|
||||||
for(let image of images) {
|
post.source,
|
||||||
let { src } = image
|
getPostId(post),
|
||||||
|
view
|
||||||
if(isUnset(src)) {
|
)
|
||||||
let finalSrc = image.srcset.split(', ').pop()
|
return post
|
||||||
|
|
||||||
src = finalSrc.slice(0, finalSrc.indexOf(' ') )
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sending through URL prevents potential XSS
|
|
||||||
imageUrls.push(new URL(src).href)
|
|
||||||
}
|
|
||||||
|
|
||||||
return imageUrls
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// o
|
// o
|
||||||
// . , . ,-. , , ,
|
// . , . ,-. , , ,
|
||||||
// |/ | |-' |/|/
|
// |/ | |-' |/|/
|
||||||
@ -588,122 +694,11 @@ export const renderNavEntry = (list) => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// | | | o
|
|
||||||
// ,-. ,-. | | ,-: |- . ,-. ;-.
|
|
||||||
// | | | | | | | | | | | | |
|
|
||||||
// `-' `-' ' ' `-` `-' ' `-' ' '
|
|
||||||
|
|
||||||
export const downloadImage = async (url, basename, source, view) => {
|
//
|
||||||
let response = await retryDelayedFetch(url, {}, source.courtesyWait, source.retryAttempts)
|
// ,-. ,-. . . ;-. ,-. ,-.
|
||||||
.catch(err => console.error(`Failed download of ${url}:`, err, err.errors) )
|
// `-. | | | | | | |-'
|
||||||
|
// `-' `-' `-` ' `-' `-'
|
||||||
if(response == undefined) {
|
|
||||||
console.error('Could not download image: ' + url)
|
|
||||||
return url
|
|
||||||
}
|
|
||||||
|
|
||||||
if(response.ok) {
|
|
||||||
let mimetype = response.headers.get('Content-Type').split(';')[0]
|
|
||||||
let extension = imageExtensions[mimetype]
|
|
||||||
|
|
||||||
if(typeof extension !== 'string') {
|
|
||||||
console.error(`Unknown mimetype for ${url}: ${mimetype}. Cannot download`)
|
|
||||||
return url
|
|
||||||
}
|
|
||||||
|
|
||||||
let pathname = Path.join(view.imageStoreDirectory, basename + extension)
|
|
||||||
let path = Path.join(view.path, pathname)
|
|
||||||
|
|
||||||
const download = () => write(path, response.body)
|
|
||||||
.then(annotate( `Downloaded ${pathname}`))
|
|
||||||
|
|
||||||
// TODO: See if the image is downloaded before even trying to download it
|
|
||||||
view.batch.add(FS.access(path).catch(download))
|
|
||||||
return pathname
|
|
||||||
} else {
|
|
||||||
console.error( createNetworkingError(response) )
|
|
||||||
return url
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export const downloadImages = (images, source, postId, view) => {
|
|
||||||
let basePath = getImageBasePath(source, postId)
|
|
||||||
let pathnames = []
|
|
||||||
|
|
||||||
for(let i = 0; i < images.length; i++) {
|
|
||||||
let basename = images.length > 1 ? basePath + '-' + i : basePath
|
|
||||||
let pathname = view.imageStore.get(basename)
|
|
||||||
|
|
||||||
if(isUnset(pathname)) {
|
|
||||||
pathname = downloadImage(images[i], basename, source, view)
|
|
||||||
}
|
|
||||||
|
|
||||||
pathnames.push(pathname)
|
|
||||||
}
|
|
||||||
|
|
||||||
return Promise.all(pathnames)
|
|
||||||
}
|
|
||||||
|
|
||||||
export const imageExtensions = {
|
|
||||||
'image/apng': '.apng',
|
|
||||||
'image/avif': '.avif',
|
|
||||||
'image/bmp': '.bmp',
|
|
||||||
'image/gif': '.gif',
|
|
||||||
'image/vnd.microsoft.icon': '.icon',
|
|
||||||
'image/jpeg': '.jpg',
|
|
||||||
'image/png': '.png',
|
|
||||||
'image/svg+xml': '.xml',
|
|
||||||
'image/tiff': '.tif',
|
|
||||||
'image/webp': '.webp'
|
|
||||||
}
|
|
||||||
|
|
||||||
export const pullImages = async (post, view, discardPostIfNoImages = false, getPostId = getPostIdFromPathname) => {
|
|
||||||
let images = extractImages(post)
|
|
||||||
|
|
||||||
if(!discardPostIfNoImages || images.length > 0) {
|
|
||||||
post.images = await downloadImages(
|
|
||||||
images,
|
|
||||||
post.source,
|
|
||||||
getPostId(post),
|
|
||||||
view
|
|
||||||
)
|
|
||||||
return post
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export const createFeed = (name, sources, main = false) => {
|
|
||||||
return {
|
|
||||||
name,
|
|
||||||
displayName: name,
|
|
||||||
main,
|
|
||||||
posts: sources.reduce((posts, source) => posts.concat(source.posts), [])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export const fetchChannelFromInstances = async (source) => {
|
|
||||||
let index = 0
|
|
||||||
let instances = source.instances
|
|
||||||
let cachedLink = source.cache.link
|
|
||||||
let channel
|
|
||||||
|
|
||||||
if(cachedLink) {
|
|
||||||
instances.unshift(cachedLink.hostname)
|
|
||||||
}
|
|
||||||
|
|
||||||
while(!channel && index != instances.length) {
|
|
||||||
source.hostname = instances[index]
|
|
||||||
channel = await fetchChannel(source)
|
|
||||||
|
|
||||||
if(source.errored) {
|
|
||||||
console.error(`Failed to fetch ${source.name} from ${source.hostname}: `, source.error)
|
|
||||||
index++
|
|
||||||
} else {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return channel
|
|
||||||
}
|
|
||||||
|
|
||||||
export const populateSource = async (channel, source, postReducerCallback, cache) => {
|
export const populateSource = async (channel, source, postReducerCallback, cache) => {
|
||||||
let fromDate = 0
|
let fromDate = 0
|
||||||
@ -750,6 +745,8 @@ export const createSourceOptions = (options, view) => {
|
|||||||
return options
|
return options
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// | | ,-
|
// | | ,-
|
||||||
// ;-. | ,-: |- | ,-. ;-. ;-.-. ,-.
|
// ;-. | ,-: |- | ,-. ;-. ;-.-. ,-.
|
||||||
// | | | | | | |- | | | | | | `-.
|
// | | | | | | |- | | | | | | `-.
|
||||||
@ -798,6 +795,31 @@ export const tumblr = {
|
|||||||
pullImages
|
pullImages
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export const fetchChannelFromInstances = async (source) => {
|
||||||
|
let index = 0
|
||||||
|
let instances = source.instances
|
||||||
|
let cachedLink = source.cache.link
|
||||||
|
let channel
|
||||||
|
|
||||||
|
if(cachedLink) {
|
||||||
|
instances.unshift(cachedLink.hostname)
|
||||||
|
}
|
||||||
|
|
||||||
|
while(!channel && index != instances.length) {
|
||||||
|
source.hostname = instances[index]
|
||||||
|
channel = await fetchChannel(source)
|
||||||
|
|
||||||
|
if(source.errored) {
|
||||||
|
console.error(`Failed to fetch ${source.name} from ${source.hostname}: `, source.error)
|
||||||
|
index++
|
||||||
|
} else {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return channel
|
||||||
|
}
|
||||||
|
|
||||||
export const nitter = {
|
export const nitter = {
|
||||||
createSource(user, options, instances, postReducerCallback, cache) {
|
createSource(user, options, instances, postReducerCallback, cache) {
|
||||||
let source = {
|
let source = {
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user