diff --git a/lib.js b/lib.js index 6750178..f43ae0a 100644 --- a/lib.js +++ b/lib.js @@ -58,12 +58,6 @@ export const getPostIdFromPathname = post => { return pathname.slice(pathname.lastIndexOf('/') + 1) } -export const testWhitelist = (array, whitelist) => - whitelist.find(tag => !array.includes(tag)) !== undefined - -export const testBlacklist = (array, blacklist) => - blacklist.find(tag => array.includes(tag)) !== undefined - export const doesExist = async (path) => { let exists @@ -743,175 +737,3 @@ export const createSourceOptions = (options, view) => { return options } - - - -// | | ,- -// ;-. | ,-: |- | ,-. ;-. ;-.-. ,-. -// | | | | | | |- | | | | | | `-. -// |-' ' `-` `-' | `-' ' ' ' ' `-' -// ' -' - -export const tumblr = { - createSource(user, options, postReducerCallback, cache) { - let lowercaseUser = user.toLowerCase() - let source = { - type: 'tumblr', - description: `Aggregate feed for @${lowercaseUser} on tumblr.com`, - hostname: lowercaseUser + '.tumblr.com', - pathname: 'rss', - name: `tumblr-${lowercaseUser}`, - displayName: user, - user: lowercaseUser, - ...createSourceOptions(options) - } - - return createSource(source, fetchChannel, postReducerCallback, cache) - }, - - createSources(users, ...args) { - return Promise.all(users.map(user => tumblr.createSource(user, ...args))) - }, - - isRepost(post) { - let reblog = post.description.querySelector('p > a.tumblr_blog') - - return reblog && reblog.innerHTML !== post.source.user - }, - - matchesTags(post, whitelist, blacklist) { - if(whitelist && testWhitelist(post.categories, whitelist)) { - return false - } - - if(blacklist && testBlacklist(post.categories, blacklist)) { - return false - } - - return true - }, - - pullImages -} - -export const fetchChannelFromInstances = async (source) => { - let index = 0 - let instances = source.instances - let cachedLink = source.cache.link - let channel - - if(cachedLink) { - instances.unshift(cachedLink.hostname) - } - - while(!channel && index != instances.length) { - source.hostname = instances[index] - channel = await fetchChannel(source) - - if(source.errored) { - console.error(`Failed to fetch ${source.name} from ${source.hostname}: `, source.error) - index++ - } else { - break - } - } - - return channel -} - -export const nitter = { - createSource(user, options, instances, postReducerCallback, cache) { - let source = { - type: 'nitter', - description: `Aggregate feed for @${user} on twitter.com`, - instances, - pathname: user + '/rss', - name: `nitter-${user}`, - displayName: user, - user, - ...createSourceOptions(options) - } - - return createSource(source, fetchChannelFromInstances, postReducerCallback, cache) - }, - - createSources(users, ...args) { - return Promise.all(users.map(user => nitter.createSource(user, ...args))) - }, - - isRepost(post) { - let creator = post.item.getElementsByTagName('dc:creator')[0] - - return creator.innerHTML.slice(1) !== post.source.user - }, - - async pullImages (post, view, imageMirrorDomain, discardPostIfNoImages = false, getPostId = getPostIdFromPathname) { - let images = extractImages(post) - let mirroredImages = [] - const mirrorImage = nitter.createImageMirrorer(post, imageMirrorDomain) - - if(!discardPostIfNoImages || images.length > 0) { - post.images = await downloadImages( - images.map(mirrorImage), - post.source, - getPostId(post), - view - ) - return post - } - }, - - createImageMirrorer(post, imageMirrorDomain) { - let mirrorUrl = new URL(imageMirrorDomain) - let basePathname = new URL(post.guid).pathname - - return (image, index, images) => { - mirrorUrl.pathname = Path.join(basePathname, 'photo', (index + 1).toString()) - - return mirrorUrl.href - } - } -} - -export const mastodon = { - createSource(usertag, options, postReducerCallback, cache) { - let [ user, hostname ] = usertag.toLowerCase().split('@') - - let source = { - type: 'mastodon', - description: `Aggregate feed for @${user} at ${hostname}`, - hostname, - pathname: '@' + user + ".rss", - name: `${hostname}-${user}`, - displayName: user, - user, - ...createSourceOptions(options) - } - - return createSource(source, fetchChannel, postReducerCallback, cache) - }, - - isRepost(post) { - // Mastodon's rss does not provide retweets/retoots - return false - }, - - async pullImages(post, view, discardPostIfNoImages) { - let media = post.item.getElementsByTagName('media:content') - let images = [] - - for(let image of media) { - images.push(image.getAttribute('url')) - } - - if(!discardPostIfNoImages || media.length > 0) { - post.images = await downloadImages( - images, - post.source, - getPostIdFromPathname(post), - view - ) - return post - } - } -} diff --git a/platforms/mastodon.js b/platforms/mastodon.js new file mode 100644 index 0000000..81bffe3 --- /dev/null +++ b/platforms/mastodon.js @@ -0,0 +1,46 @@ +import { createSource, createSourceOptions, downloadImages, fetchChannel, getPostIdFromPathname } from "../lib.js" + +let mastodon = {} + +mastodon.createSource = (usertag, options, postReducerCallback, cache) => { + let [ user, hostname ] = usertag.toLowerCase().split('@') + + let source = { + type: 'mastodon', + description: `Aggregate feed for @${user} at ${hostname}`, + hostname, + pathname: '@' + user + ".rss", + name: `${hostname}-${user}`, + displayName: user, + user, + ...createSourceOptions(options) + } + + return createSource(source, fetchChannel, postReducerCallback, cache) +} + +mastodon.isRepost = (post) => { + // Mastodon's rss does not provide retweets/retoots + return false +} + +mastodon.pullImages = async (post, view, discardPostIfNoImages) => { + let media = post.item.getElementsByTagName('media:content') + let images = [] + + for(let image of media) { + images.push(image.getAttribute('url')) + } + + if(!discardPostIfNoImages || media.length > 0) { + post.images = await downloadImages( + images, + post.source, + getPostIdFromPathname(post), + view + ) + return post + } +} + +export default mastodon diff --git a/platforms/nitter.js b/platforms/nitter.js new file mode 100644 index 0000000..2bba11e --- /dev/null +++ b/platforms/nitter.js @@ -0,0 +1,78 @@ +import Path from "path" +import { createSource, createSourceOptions, extractImages, downloadImages, getPostIdFromPathname, fetchChannel } from "../lib.js" + +let nitter = {} + +export const fetchChannelFromInstances = async (source) => { + let index = 0 + let instances = source.instances + let cachedLink = source.cache.link + let channel + + if(cachedLink) { + instances.unshift(cachedLink.hostname) + } + + while(!channel && index != instances.length) { + source.hostname = instances[index] + channel = await fetchChannel(source) + + if(source.errored) { + console.error(`Failed to fetch ${source.name} from ${source.hostname}: `, source.error) + index++ + } else { + break + } + } + + return channel +} + +nitter.createSource = (user, options, instances, postReducerCallback, cache) => { + let source = { + type: 'nitter', + description: `Aggregate feed for @${user} on twitter.com`, + instances, + pathname: user + '/rss', + name: `nitter-${user}`, + displayName: user, + user, + ...createSourceOptions(options) + } + + return createSource(source, fetchChannelFromInstances, postReducerCallback, cache) +} + +nitter.isRepost = (post) => { + let creator = post.item.getElementsByTagName('dc:creator')[0] + + return creator.innerHTML.slice(1) !== post.source.user +} + +nitter.pullImages = async (post, view, imageMirrorDomain, discardPostIfNoImages = false, getPostId = getPostIdFromPathname) => { + let images = extractImages(post) + const mirrorImage = nitter.createImageMirrorer(post, imageMirrorDomain) + + if(!discardPostIfNoImages || images.length > 0) { + post.images = await downloadImages( + images.map(mirrorImage), + post.source, + getPostId(post), + view + ) + return post + } +} + +nitter.createImageMirrorer =(post, imageMirrorDomain) => { + let mirrorUrl = new URL(imageMirrorDomain) + let basePathname = new URL(post.guid).pathname + + return (image, index, images) => { + mirrorUrl.pathname = Path.join(basePathname, 'photo', (index + 1).toString()) + + return mirrorUrl.href + } +} + +export default nitter diff --git a/platforms/tumblr.js b/platforms/tumblr.js new file mode 100644 index 0000000..4812117 --- /dev/null +++ b/platforms/tumblr.js @@ -0,0 +1,47 @@ +import { createSource, createSourceOptions, fetchChannel, pullImages } from "../lib.js" + +let tumblr = {} + +export const testWhitelist = (array, whitelist) => + whitelist.find(tag => !array.includes(tag)) !== undefined + +export const testBlacklist = (array, blacklist) => + blacklist.find(tag => array.includes(tag)) !== undefined + +tumblr.createSource = (user, options, postReducerCallback, cache) => { + let lowercaseUser = user.toLowerCase() + let source = { + type: 'tumblr', + description: `Aggregate feed for @${lowercaseUser} on tumblr.com`, + hostname: lowercaseUser + '.tumblr.com', + pathname: 'rss', + name: `tumblr-${lowercaseUser}`, + displayName: user, + user: lowercaseUser, + ...createSourceOptions(options) + } + + return createSource(source, fetchChannel, postReducerCallback, cache) +} + +tumblr.isRepost = (post) => { + let reblog = post.description.querySelector('p > a.tumblr_blog') + + return reblog && reblog.innerHTML !== post.source.user +} + +tumblr.matchesTags = (post, whitelist, blacklist) => { + if(whitelist && testWhitelist(post.categories, whitelist)) { + return false + } + + if(blacklist && testBlacklist(post.categories, blacklist)) { + return false + } + + return true +} + +tumblr.pullImages = pullImages + +export default tumblr