Merge pull request #383 from bensteinberg/option-video-size

Limit video attachment size

README.md

@@ -196,6 +196,7 @@ Options:
   --capture-window-x <number>                            Width of the browser window Scoop will open to capture, in pixels. (default: 1600)
   --capture-window-y <number>                            Height of the browser window Scoop will open to capture, in pixels. (default: 900)
   --max-capture-size <number>                            Size limit for the capture's exchanges list, in bytes. (default: 209715200)
+  --max-video-capture-size <number>                      Size limit for the video attachment, in bytes. Scoop will not capture video attachments larger than this. (default: 209715200)
   --auto-scroll <bool>                                   Should Scoop try to scroll through the page? (choices: "true", "false", default: "true")
   --auto-play-media <bool>                               Should Scoop try to autoplay `<audio>` and `<video>` tags? (choices: "true", "false", default: "true")
   --grab-secondary-resources <bool>                      Should Scoop try to download img srcsets and secondary stylesheets? (choices: "true", "false", default: "true")

Scoop.js

@@ -920,6 +920,7 @@ export class Scoop {
         '--output', `"${videoFilename}"`,
         '--no-check-certificate',
         '--proxy', `'http://${this.options.proxyHost}:${this.options.proxyPort}'`,
+        '--max-filesize', `"${this.options.maxVideoCaptureSize}"`,
         this.url
       ]
 
@@ -948,6 +949,9 @@ export class Scoop {
           const body = await readFile(`${this.captureTmpFolderPath}${file}`)
           const isEntryPoint = false // TODO: Reconsider whether this should be an entry point.
 
+          if (!body.length) {
+            continue
+          }
           this.addGeneratedExchange(url, httpHeaders, body, isEntryPoint)
           videoSaved = true
 
@@ -995,6 +999,11 @@ export class Scoop {
       }
     }
 
+    if (videoSaved === false) {
+      this.log.warn('yt-dlp reported success (returned 0), but produced no output.')
+      return
+    }
+
     //
     // Try to add metadata to exchanges
     //
@@ -1032,10 +1041,6 @@ export class Scoop {
     //
     // Generate summary page
     //
-    if ((videoSaved || metadataSaved || subtitlesSaved) === false) {
-      this.log.warn('yt-dlp reported success (returned 0), but produced no output.')
-      return
-    }
     try {
       const html = nunjucks.render('video-extracted-summary.njk', {
         url: this.url,

Scoop.test.js

@@ -34,6 +34,8 @@ await test('Scoop - capture of a web page.', async (t) => {
   app.get('/redirect', (req, res) => res.redirect(parseInt(req.query.statusCode), req.query.path))
   app.get('/:path', (req, res) => res.sendFile(FIXTURES_PATH + req.params.path))
 
+  const testVideoFixture = await readFile(`${FIXTURES_PATH}video.mp4`)
+
   /*
    * TESTS
    */
@@ -72,6 +74,21 @@ await test('Scoop - capture of a web page.', async (t) => {
       'file:///video-extracted-summary.html'
     ]
     assert.deepEqual(expected.filter(url => urls.includes(url)), expected)
+    const attachment = exchanges.filter(
+      ex => ex.url === 'file:///video-extracted-1.mp4'
+    )[0]
+    assert.deepEqual(attachment.response.body, testVideoFixture)
+  })
+
+  await t.test('Scoop observes maxVideoCaptureSize', async (_t) => {
+    const { exchanges } = await Scoop.capture(`${URL}/test.html`, { ...options, captureVideoAsAttachment: true, maxVideoCaptureSize: 50000 })
+    const urls = exchanges.map(ex => ex.url)
+    const expected = [
+      'file:///video-extracted-1.mp4',
+      'file:///video-extracted-metadata.json',
+      'file:///video-extracted-summary.html'
+    ]
+    assert.deepEqual(expected.filter(url => urls.includes(url)), [])
   })
 
   await t.test('Scoop can be configured for different window dimensions', async (_t) => {

bin/cli.js

@@ -100,7 +100,7 @@ program.addOption(
 program.addOption(
   new Option('--provenance-summary <bool>', 'Add provenance summary to capture?')
     .choices(['true', 'false'])
-    .default(String(defaults.captureVideoAsAttachment))
+    .default(String(defaults.provenanceSummary))
 )
 
 program.addOption(
@@ -181,6 +181,13 @@ program.addOption(
     .default(defaults.maxCaptureSize)
 )
 
+program.addOption(
+  new Option(
+    '--max-video-capture-size <number>',
+    'Size limit for the video attachment, in bytes. Scoop will not capture video attachments larger than this.')
+    .default(defaults.maxVideoCaptureSize)
+)
+
 //
 // Behaviors
 //

options.js

@@ -26,6 +26,7 @@ export const defaults = {
   captureWindowY: 900,
 
   maxCaptureSize: 200 * 1024 * 1024,
+  maxVideoCaptureSize: 200 * 1024 * 1024,
 
   autoScroll: true,
   autoPlayMedia: true,

options.types.js

@@ -21,7 +21,9 @@
  * @property {number} captureWindowX=1600 - Browser window resolution in pixels: X axis.
  * @property {number} captureWindowY=900 - Browser window resolution in pixels: Y axis.
  *
- * @property {number} maxCaptureSize=209715200 - Maximum size, in bytes, for the exchanges list. Scoop stop intercepting exchanges at this threshold.
+ * @property {number} maxCaptureSize=209715200 - Maximum size, in bytes, for the exchanges list. Scoop stops intercepting exchanges at this threshold.
+ *
+ * @property {number} maxVideoCaptureSize=209715200 - Maximum size, in bytes, for video attachments. Scoop will not capture video attachments larger than this.
  *
  * @property {boolean} autoScroll=true - Should Scoop try to scroll through the page?
  * @property {boolean} autoPlayMedia=true - Should Scoop try to autoplay `<audio>` and `<video>` tags?