From 9339748675d301ede32f9faf58792ae3433a75ed Mon Sep 17 00:00:00 2001 From: Zoe Roux Date: Sun, 8 Oct 2023 22:33:44 +0200 Subject: [PATCH] Add custom scripts for rss download --- .../modals/feeds-modal/DownloadRuleForm.tsx | 3 + .../modals/feeds-modal/DownloadRulesTab.tsx | 3 + client/src/javascript/i18n/strings/en.json | 1 + server/routes/api/feed-monitor.test.ts | 1 + server/services/feedService.ts | 74 ++++++++--------- server/util/feedUtil.ts | 81 ++++++++++++------- shared/types/Feed.ts | 2 + 7 files changed, 100 insertions(+), 65 deletions(-) diff --git a/client/src/javascript/components/modals/feeds-modal/DownloadRuleForm.tsx b/client/src/javascript/components/modals/feeds-modal/DownloadRuleForm.tsx index c5508ac11..7edca4936 100644 --- a/client/src/javascript/components/modals/feeds-modal/DownloadRuleForm.tsx +++ b/client/src/javascript/components/modals/feeds-modal/DownloadRuleForm.tsx @@ -101,6 +101,9 @@ const DownloadRuleForm: FC = ({ )} + + + { field: formData.field, match: formData.match ?? initialRule.match, exclude: formData.exclude ?? initialRule.exclude, + script: formData.script ?? initialRule.script, destination: formData.destination ?? initialRule.destination, tags: formData.tags?.split(',') ?? initialRule.tags, startOnLoad: formData.startOnLoad ?? initialRule.startOnLoad, diff --git a/client/src/javascript/i18n/strings/en.json b/client/src/javascript/i18n/strings/en.json index 9b7c4220c..46f5d1d91 100644 --- a/client/src/javascript/i18n/strings/en.json +++ b/client/src/javascript/i18n/strings/en.json @@ -102,6 +102,7 @@ "feeds.no.items.matching": "No items matching search term.", "feeds.no.rules.defined": "No rules defined.", "feeds.regEx": "RegEx", + "feeds.script": "Script", "feeds.search": "Search term", "feeds.search.term": "Search term", "feeds.select.feed": "Select Feed", diff --git a/server/routes/api/feed-monitor.test.ts b/server/routes/api/feed-monitor.test.ts index fc06f4431..f0d22ca78 100644 --- a/server/routes/api/feed-monitor.test.ts +++ b/server/routes/api/feed-monitor.test.ts @@ -220,6 +220,7 @@ describe('PUT /api/feed-monitor/rules', () => { feedIDs: [''], match: '', exclude: '.*', + script: '', destination: tempDirectory, tags: ['FeedItem'], startOnLoad: false, diff --git a/server/services/feedService.ts b/server/services/feedService.ts index a2b144e51..823848552 100644 --- a/server/services/feedService.ts +++ b/server/services/feedService.ts @@ -70,6 +70,7 @@ class FeedService extends BaseService> { field: rule.field, match: rule.match, exclude: rule.exclude, + script: rule.script, startOnLoad: rule.startOnLoad, isBasePath: rule.isBasePath, }); @@ -259,22 +260,22 @@ class FeedService extends BaseService> { } handleNewItems = (feedReaderOptions: FeedReaderOptions, feedItems: Array): void => { - this.getPreviouslyMatchedUrls() - .then((previouslyMatchedUrls) => { - const {feedID, feedLabel} = feedReaderOptions; - const applicableRules = this.rules[feedID]; - if (!applicableRules) return; - - const itemsMatchingRules = getFeedItemsMatchingRules(feedItems, applicableRules); - const itemsToDownload = itemsMatchingRules.filter((item) => - item.urls.some((url) => !previouslyMatchedUrls.includes(url)), - ); + this.getPreviouslyMatchedUrls().then(async (previouslyMatchedUrls) => { + const {feedID, feedLabel} = feedReaderOptions; + const applicableRules = this.rules[feedID]; + if (!applicableRules) return; + + const itemsMatchingRules = await getFeedItemsMatchingRules(feedItems, applicableRules); + const itemsToDownload = itemsMatchingRules.filter((item) => + item.urls.some((url) => !previouslyMatchedUrls.includes(url)), + ); - if (itemsToDownload.length === 0) { - return; - } + if (itemsToDownload.length === 0) { + return; + } - Promise.all( + try { + const ArrayOfURLArrays = await Promise.all( itemsToDownload.map(async (item): Promise> => { const {urls, destination, start, tags, ruleID} = item; @@ -298,28 +299,29 @@ class FeedService extends BaseService> { return urls; }), - ).then((ArrayOfURLArrays) => { - const addedURLs = ArrayOfURLArrays.reduce( - (URLArray: Array, urls: Array) => URLArray.concat(urls), - [], - ); - - this.db.update({type: 'matchedTorrents'}, {$push: {urls: {$each: addedURLs}}}, {upsert: true}); - - this.services?.notificationService.addNotification( - itemsToDownload.map((item) => ({ - id: 'notification.feed.torrent.added', - data: { - title: item.matchTitle, - feedLabel, - ruleLabel: item.ruleLabel, - }, - })), - ); - this.services?.torrentService.fetchTorrentList(); - }); - }) - .catch(console.error); + ); + const addedURLs = ArrayOfURLArrays.reduce( + (URLArray: Array, urls: Array) => URLArray.concat(urls), + [], + ); + + this.db.update({type: 'matchedTorrents'}, {$push: {urls: {$each: addedURLs}}}, {upsert: true}); + + this.services?.notificationService.addNotification( + itemsToDownload.map((item) => ({ + id: 'notification.feed.torrent.added', + data: { + title: item.matchTitle, + feedLabel, + ruleLabel: item.ruleLabel, + }, + })), + ); + this.services?.torrentService.fetchTorrentList(); + } catch (e) { + console.error(e); + } + }); }; async removeItem(id: string): Promise { diff --git a/server/util/feedUtil.ts b/server/util/feedUtil.ts index c11b36bb7..64b2fe085 100644 --- a/server/util/feedUtil.ts +++ b/server/util/feedUtil.ts @@ -4,6 +4,7 @@ import {cdata as matchCDATA} from '../../shared/util/regEx'; import type {AddTorrentByURLOptions} from '../../shared/schema/api/torrents'; import type {Rule} from '../../shared/types/Feed'; +import {spawn} from 'child_process'; interface PendingDownloadItems extends Required> { @@ -54,36 +55,58 @@ export const getTorrentUrlsFromFeedItem = (feedItem: FeedItem): Array => return []; }; -export const getFeedItemsMatchingRules = ( +const execAsync = (...command: string[]) => { + const p = spawn(command[0], command.slice(1)); + return new Promise((resolveFunc) => { + p.stdout.on('data', (x) => { + process.stdout.write(x.toString()); + }); + p.stderr.on('data', (x) => { + process.stderr.write(x.toString()); + }); + p.on('exit', (code) => { + resolveFunc(code); + }); + }); +}; + +export const getFeedItemsMatchingRules = async ( feedItems: Array, rules: Array, -): Array => { - return feedItems.reduce((matchedItems: Array, feedItem) => { - rules.forEach((rule) => { - const matchField = rule.field ? (feedItem[rule.field] as string) : (feedItem.title as string); - const isMatched = new RegExp(rule.match, 'gi').test(matchField); - const isExcluded = rule.exclude !== '' && new RegExp(rule.exclude, 'gi').test(matchField); - - if (isMatched && !isExcluded) { - const torrentUrls = getTorrentUrlsFromFeedItem(feedItem); - const isAlreadyDownloaded = matchedItems.some((matchedItem) => - torrentUrls.every((url) => matchedItem.urls.includes(url)), - ); - - if (!isAlreadyDownloaded && torrentUrls[0] != null) { - matchedItems.push({ - urls: torrentUrls as [string, ...string[]], - tags: rule.tags, - matchTitle: feedItem.title as string, - ruleID: rule._id, - ruleLabel: rule.label, - destination: rule.destination, - start: rule.startOnLoad, - }); - } - } - }); +): Promise> => { + const matchedItems: Array = []; + + await Promise.all( + feedItems.map(async (feedItem) => { + await Promise.all( + rules.map(async (rule) => { + const matchField = rule.field ? (feedItem[rule.field] as string) : (feedItem.title as string); + const isMatched = rule.match === '' || new RegExp(rule.match, 'gi').test(matchField); + const isExcluded = rule.exclude !== '' && new RegExp(rule.exclude, 'gi').test(matchField); + const scriptMatch = rule.script === '' || (await execAsync(rule.script)) === 80; + + if (isMatched && !isExcluded && scriptMatch) { + const torrentUrls = getTorrentUrlsFromFeedItem(feedItem); + const isAlreadyDownloaded = matchedItems.some((matchedItem) => + torrentUrls.every((url) => matchedItem.urls.includes(url)), + ); + + if (!isAlreadyDownloaded && torrentUrls[0] != null) { + matchedItems.push({ + urls: torrentUrls as [string, ...string[]], + tags: rule.tags, + matchTitle: feedItem.title as string, + ruleID: rule._id, + ruleLabel: rule.label, + destination: rule.destination, + start: rule.startOnLoad, + }); + } + } + }), + ); + }), + ); - return matchedItems; - }, []); + return matchedItems; }; diff --git a/shared/types/Feed.ts b/shared/types/Feed.ts index 287d1931c..ea738b689 100644 --- a/shared/types/Feed.ts +++ b/shared/types/Feed.ts @@ -27,6 +27,8 @@ export interface Rule { match: string; // Regular expression to exclude items. exclude: string; + // Custom script to select if the item should be downloaded (exit with status 80 to download). + script: string; // Destination path where matched items are downloaded to. destination: string; // Tags to be added when items are queued for download.