Custom options handler - fixed merging options. (#12)

* Custom options handler - fixed merging options. * Improved custom options handler.
apify · Apr 15, 2021 · b463b0a · b463b0a
1 parent dfc206a
commit b463b0a
Show file tree

Hide file tree

Showing 4 changed files with 87 additions and 24 deletions.
diff --git a/src/handlers/custom-options.js b/src/handlers/custom-options.js
@@ -1,3 +1,5 @@
+const got = require('got');
+
 /**
  * @param {object} options
  * @param {function} next
@@ -9,17 +11,23 @@ function customOptionsHandler(options, next) {
         headerGeneratorOptions,
         useHeaderGenerator,
         context,
-        ...gotOptions } = options;
+    } = options;
 
     // Got expects custom properties inside the context option.
-    gotOptions.context = {
+    const newContext = {
         ...context,
         proxyUrl,
         headerGeneratorOptions,
         useHeaderGenerator,
     };
 
-    return next(gotOptions);
+    delete options.proxyUrl;
+    delete options.headerGeneratorOptions;
+    delete options.useHeaderGenerator;
+
+    const finalOptions = got.mergeOptions(options, { context: newContext });
+
+    return next(finalOptions);
 }
 
 module.exports = {

diff --git a/test/custom-options.test.js b/test/custom-options.test.js
@@ -1,13 +1,26 @@
-const { customOptionsHandler } = require('../src/handlers/custom-options');
+const gotScraping = require('../src');
+const { startDummyServer } = require('./helpers/dummy-server');
 
 describe('Custom options', () => {
-    test('should move custom options to context', () => {
-        const nextHolder = {
-            next() {},
-        };
+    let server;
+    let port;
+
+    beforeAll(async () => {
+        server = await startDummyServer();
+        port = server.address().port; //eslint-disable-line
+    });
+
+    afterAll(() => {
+        server.close();
+    });
+
+    test('should move custom options to context', async () => {
+        expect.assertions(2);
+
         const options = {
-            url: 'testUrl',
-            proxyUrl: 'test',
+            url: `http://localhost:${port}/html`,
+            proxyUrl: 'http://localhost:8080',
+            http2: false,
             headerGeneratorOptions: {
                 browsers: [
                     {
@@ -16,18 +29,24 @@ describe('Custom options', () => {
                 ],
             },
             useHeaderGenerator: false,
-        };
-        jest.spyOn(nextHolder, 'next');
-
-        customOptionsHandler(options, nextHolder.next);
+            hooks: {
+                beforeRequest: [
+                    (opts) => {
+                        expect(opts.context).toMatchObject({
+                            proxyUrl: options.proxyUrl,
+                            headerGeneratorOptions: options.headerGeneratorOptions,
+                            useHeaderGenerator: false,
+                        });
 
-        expect(nextHolder.next).toBeCalledWith({
-            url: options.url,
-            context: {
-                proxyUrl: options.proxyUrl,
-                headerGeneratorOptions: options.headerGeneratorOptions,
-                useHeaderGenerator: false,
+                        throw new Error('request aborted');
+                    },
+                ],
             },
-        });
+        };
+        try {
+            await gotScraping(options);
+        } catch (e) {
+            expect(e.message).toBe('request aborted');
+        }
     });
 });
diff --git a/test/helpers/dummy-server.js b/test/helpers/dummy-server.js
@@ -18,6 +18,10 @@ const startDummyServer = async (port) => {
         res.json({ test: 123 });
     });
 
+    app.post('/jsonPost', (req, res) => {
+        res.json(req.body);
+    });
+
     app.get('/html', (req, res) => {
         res.setHeader('content-type', 'text/html');
         res.send('<html></html>');

diff --git a/test/main.test.js b/test/main.test.js
@@ -91,6 +91,37 @@ describe('GotScraping', () => {
         expect(response.statusCode).toBe(200);
     });
 
+    test('should post json', async () => {
+        const body = { foo: 'bar' };
+
+        const response = await gotScraping({
+            responseType: 'json',
+            url: `http://localhost:${port}/jsonPost`,
+            json: body,
+            method: 'POST',
+        });
+
+        expect(response.statusCode).toBe(200);
+        expect(response.body).toEqual(body);
+    });
+
+    test('should post body', async () => {
+        const body = { foo: 'bar' };
+
+        const response = await gotScraping({
+            url: `http://localhost:${port}/jsonPost`,
+            body: JSON.stringify(body),
+            responseType: 'json',
+            method: 'POST',
+            headers: {
+                'content-type': 'application/json; charset=UTF-8',
+            },
+        });
+
+        expect(response.statusCode).toBe(200);
+        expect(response.body).toEqual(body);
+    });
+
     describe('Integration', () => {
         test('should use http2 first', async () => {
             const response = await gotScraping({ url: 'https://apify.com/' });
@@ -106,10 +137,11 @@ describe('GotScraping', () => {
             });
 
             const responseProxy = await gotScraping({
-                json: true,
-                url: 'https://apify.com',
+                responseType: 'json',
+                url: 'https://api.apify.com/v2/browser-info',
                 proxyUrl: `http://groups-SHADER,session-123:${process.env.APIFY_PROXY_PASSWORD}@proxy.apify.com:8000`,
                 http2: false,
+                ciphers: undefined,
 
             });
             expect(response.statusCode).toBe(200);
@@ -134,7 +166,7 @@ describe('GotScraping', () => {
             const nodeVersion = parseFloat(process.versions.node);
 
             const proxyPromise = gotScraping({
-                json: true,
+                responseType: 'json',
                 url: 'https://api.apify.com/v2/browser-info',
                 proxyUrl: `http://groups-SHADER,session-123:${process.env.APIFY_PROXY_PASSWORD}@proxy.apify.com:8000`,
                 ciphers: undefined,