Add custom scripts for rss download

client/src/javascript/components/modals/feeds-modal/DownloadRuleForm.tsx

@@ -101,6 +101,9 @@ const DownloadRuleForm: FC<DownloadRuleFormProps> = ({
           )}
         </Textbox>
       </FormRow>
+      <FormRow>
+        <Textbox id="script" label={i18n._('feeds.script')} defaultValue={rule.script} />
+      </FormRow>
       <FormRow>
         <FormRowItem>
           <FilesystemBrowserTextbox

client/src/javascript/components/modals/feeds-modal/DownloadRulesTab.tsx

@@ -17,6 +17,7 @@ const initialRule: AddRuleOptions = {
   feedIDs: [],
   match: '',
   exclude: '',
+  script: '',
   tags: [],
   destination: '',
   startOnLoad: false,
@@ -64,6 +65,7 @@ interface RuleFormData {
   feedID: string;
   label: string;
   match: string;
+  script: string;
   tags: string;
   isBasePath: boolean;
   startOnLoad: boolean;
@@ -140,6 +142,7 @@ const DownloadRulesTab: FC = () => {
             field: formData.field,
             match: formData.match ?? initialRule.match,
             exclude: formData.exclude ?? initialRule.exclude,
+            script: formData.script ?? initialRule.script,
             destination: formData.destination ?? initialRule.destination,
             tags: formData.tags?.split(',') ?? initialRule.tags,
             startOnLoad: formData.startOnLoad ?? initialRule.startOnLoad,

client/src/javascript/i18n/strings/en.json

@@ -102,6 +102,7 @@
   "feeds.no.items.matching": "No items matching search term.",
   "feeds.no.rules.defined": "No rules defined.",
   "feeds.regEx": "RegEx",
+  "feeds.script": "Script",
   "feeds.search": "Search term",
   "feeds.search.term": "Search term",
   "feeds.select.feed": "Select Feed",

server/routes/api/feed-monitor.test.ts

@@ -220,6 +220,7 @@ describe('PUT /api/feed-monitor/rules', () => {
     feedIDs: [''],
     match: '',
     exclude: '.*',
+    script: '',
     destination: tempDirectory,
     tags: ['FeedItem'],
     startOnLoad: false,

server/services/feedService.ts

@@ -70,6 +70,7 @@ class FeedService extends BaseService<Record<string, never>> {
                 field: rule.field,
                 match: rule.match,
                 exclude: rule.exclude,
+                script: rule.script,
                 startOnLoad: rule.startOnLoad,
                 isBasePath: rule.isBasePath,
               });
@@ -259,22 +260,22 @@ class FeedService extends BaseService<Record<string, never>> {
   }
 
   handleNewItems = (feedReaderOptions: FeedReaderOptions, feedItems: Array<FeedItem>): void => {
-    this.getPreviouslyMatchedUrls()
-      .then((previouslyMatchedUrls) => {
-        const {feedID, feedLabel} = feedReaderOptions;
-        const applicableRules = this.rules[feedID];
-        if (!applicableRules) return;
-
-        const itemsMatchingRules = getFeedItemsMatchingRules(feedItems, applicableRules);
-        const itemsToDownload = itemsMatchingRules.filter((item) =>
-          item.urls.some((url) => !previouslyMatchedUrls.includes(url)),
-        );
+    this.getPreviouslyMatchedUrls().then(async (previouslyMatchedUrls) => {
+      const {feedID, feedLabel} = feedReaderOptions;
+      const applicableRules = this.rules[feedID];
+      if (!applicableRules) return;
+
+      const itemsMatchingRules = await getFeedItemsMatchingRules(feedItems, applicableRules);
+      const itemsToDownload = itemsMatchingRules.filter((item) =>
+        item.urls.some((url) => !previouslyMatchedUrls.includes(url)),
+      );
 
-        if (itemsToDownload.length === 0) {
-          return;
-        }
+      if (itemsToDownload.length === 0) {
+        return;
+      }
 
-        Promise.all(
+      try {
+        const ArrayOfURLArrays = await Promise.all(
           itemsToDownload.map(async (item): Promise<Array<string>> => {
             const {urls, destination, start, tags, ruleID} = item;
 
@@ -298,28 +299,29 @@ class FeedService extends BaseService<Record<string, never>> {
 
             return urls;
           }),
-        ).then((ArrayOfURLArrays) => {
-          const addedURLs = ArrayOfURLArrays.reduce(
-            (URLArray: Array<string>, urls: Array<string>) => URLArray.concat(urls),
-            [],
-          );
-
-          this.db.update({type: 'matchedTorrents'}, {$push: {urls: {$each: addedURLs}}}, {upsert: true});
-
-          this.services?.notificationService.addNotification(
-            itemsToDownload.map((item) => ({
-              id: 'notification.feed.torrent.added',
-              data: {
-                title: item.matchTitle,
-                feedLabel,
-                ruleLabel: item.ruleLabel,
-              },
-            })),
-          );
-          this.services?.torrentService.fetchTorrentList();
-        });
-      })
-      .catch(console.error);
+        );
+        const addedURLs = ArrayOfURLArrays.reduce(
+          (URLArray: Array<string>, urls: Array<string>) => URLArray.concat(urls),
+          [],
+        );
+
+        this.db.update({type: 'matchedTorrents'}, {$push: {urls: {$each: addedURLs}}}, {upsert: true});
+
+        this.services?.notificationService.addNotification(
+          itemsToDownload.map((item) => ({
+            id: 'notification.feed.torrent.added',
+            data: {
+              title: item.matchTitle,
+              feedLabel,
+              ruleLabel: item.ruleLabel,
+            },
+          })),
+        );
+        this.services?.torrentService.fetchTorrentList();
+      } catch (e) {
+        console.error(e);
+      }
+    });
   };
 
   async removeItem(id: string): Promise<void> {

server/util/feedUtil.ts

@@ -4,6 +4,7 @@ import {cdata as matchCDATA} from '../../shared/util/regEx';
 
 import type {AddTorrentByURLOptions} from '../../shared/schema/api/torrents';
 import type {Rule} from '../../shared/types/Feed';
+import {spawn} from 'child_process';
 
 interface PendingDownloadItems
   extends Required<Pick<AddTorrentByURLOptions, 'urls' | 'destination' | 'tags' | 'start'>> {
@@ -54,36 +55,58 @@ export const getTorrentUrlsFromFeedItem = (feedItem: FeedItem): Array<string> =>
   return [];
 };
 
-export const getFeedItemsMatchingRules = (
+const execAsync = (...command: string[]) => {
+  const p = spawn(command[0], command.slice(1));
+  return new Promise((resolveFunc) => {
+    p.stdout.on('data', (x) => {
+      process.stdout.write(x.toString());
+    });
+    p.stderr.on('data', (x) => {
+      process.stderr.write(x.toString());
+    });
+    p.on('exit', (code) => {
+      resolveFunc(code);
+    });
+  });
+};
+
+export const getFeedItemsMatchingRules = async (
   feedItems: Array<FeedItem>,
   rules: Array<Rule>,
-): Array<PendingDownloadItems> => {
-  return feedItems.reduce((matchedItems: Array<PendingDownloadItems>, feedItem) => {
-    rules.forEach((rule) => {
-      const matchField = rule.field ? (feedItem[rule.field] as string) : (feedItem.title as string);
-      const isMatched = new RegExp(rule.match, 'gi').test(matchField);
-      const isExcluded = rule.exclude !== '' && new RegExp(rule.exclude, 'gi').test(matchField);
-
-      if (isMatched && !isExcluded) {
-        const torrentUrls = getTorrentUrlsFromFeedItem(feedItem);
-        const isAlreadyDownloaded = matchedItems.some((matchedItem) =>
-          torrentUrls.every((url) => matchedItem.urls.includes(url)),
-        );
-
-        if (!isAlreadyDownloaded && torrentUrls[0] != null) {
-          matchedItems.push({
-            urls: torrentUrls as [string, ...string[]],
-            tags: rule.tags,
-            matchTitle: feedItem.title as string,
-            ruleID: rule._id,
-            ruleLabel: rule.label,
-            destination: rule.destination,
-            start: rule.startOnLoad,
-          });
-        }
-      }
-    });
+): Promise<Array<PendingDownloadItems>> => {
+  const matchedItems: Array<PendingDownloadItems> = [];
+
+  await Promise.all(
+    feedItems.map(async (feedItem) => {
+      await Promise.all(
+        rules.map(async (rule) => {
+          const matchField = rule.field ? (feedItem[rule.field] as string) : (feedItem.title as string);
+          const isMatched = rule.match === '' || new RegExp(rule.match, 'gi').test(matchField);
+          const isExcluded = rule.exclude !== '' && new RegExp(rule.exclude, 'gi').test(matchField);
+          const scriptMatch = rule.script === '' || (await execAsync(rule.script)) === 80;
+
+          if (isMatched && !isExcluded && scriptMatch) {
+            const torrentUrls = getTorrentUrlsFromFeedItem(feedItem);
+            const isAlreadyDownloaded = matchedItems.some((matchedItem) =>
+              torrentUrls.every((url) => matchedItem.urls.includes(url)),
+            );
+
+            if (!isAlreadyDownloaded && torrentUrls[0] != null) {
+              matchedItems.push({
+                urls: torrentUrls as [string, ...string[]],
+                tags: rule.tags,
+                matchTitle: feedItem.title as string,
+                ruleID: rule._id,
+                ruleLabel: rule.label,
+                destination: rule.destination,
+                start: rule.startOnLoad,
+              });
+            }
+          }
+        }),
+      );
+    }),
+  );
 
-    return matchedItems;
-  }, []);
+  return matchedItems;
 };

shared/types/Feed.ts

@@ -27,6 +27,8 @@ export interface Rule {
   match: string;
   // Regular expression to exclude items.
   exclude: string;
+  // Custom script to select if the item should be downloaded (exit with status 80 to download).
+  script: string;
   // Destination path where matched items are downloaded to.
   destination: string;
   // Tags to be added when items are queued for download.