Feature: custom ALPN - better header generation and proxy handling (#20)

* Feature: custom ALPN - better header generation and proxy handling

* chore: Bumped version. Updated changelog.

CHANGELOG.md

@@ -1,3 +1,7 @@
+1.0.4 / 2021/05/17
+====================
+- HTTP2 protocol resolving fix
+
 1.0.3 / 2021/04/27
 ====================
 - HTTP2 wrapper fix

jest.config.js

@@ -6,5 +6,5 @@ module.exports = {
     rootDir: path.join(__dirname, './'),
     testMatch: ['**/?(*.)+(spec|test).[tj]s?(x)'],
     setupFilesAfterEnv: ['jest-extended'],
-    timeout: 10e3,
+    testTimeout: 10e3,
 };

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "got-scraping",
-	"version": "1.0.3",
+	"version": "1.0.4",
 	"description": "HTTP client made for scraping based on got.",
 	"main": "src/index.js",
 	"files": [

src/handlers/alpn.js

@@ -0,0 +1,29 @@
+const httpResolver = require('../http-resolver');
+
+/**
+ * @param {object} options
+ * @param {function} next
+ * @returns {import('got').GotReturn}
+ */
+async function alpnHandler(options, next) {
+    const { url, http2 } = options;
+
+    if (http2) {
+        const parsedUrl = new URL(url);
+
+        if (parsedUrl.protocol === 'https:') {
+            const protocol = await httpResolver.resolveHttpVersion(parsedUrl);
+
+            options.http2 = protocol === 'h2';
+        } else {
+            // http2 is https
+            options.http2 = false;
+        }
+    }
+
+    return next(options);
+}
+
+module.exports = {
+    alpnHandler,
+};

src/handlers/browser-headers.js

@@ -59,7 +59,9 @@ function createOptionsWithBeforeRequestHook(generatedHeaders, headerOverrides) {
         hooks: {
             beforeRequest: [
                 (gotOptions) => {
-                    gotOptions.headers = mergeHeaders(generatedHeaders, headerOverrides);
+                    const mergedOriginalHeaders = mergeHeaders(generatedHeaders, gotOptions.headers);
+
+                    gotOptions.headers = mergeHeaders(mergedOriginalHeaders, headerOverrides);
                 },
             ],
         },

src/http-resolver.js

@@ -24,7 +24,7 @@ class HttpResolver {
             const result = await http2.auto.resolveProtocol({
                 host: hostname,
                 servername: hostname,
-                port,
+                port: port || 443,
                 ALPNProtocols: ['h2', 'http/1.1'],
                 rejectUnauthorized,
             });

src/index.js

@@ -7,6 +7,7 @@ const { optionsValidationHandler } = require('./handlers/options-validation');
 const { customOptionsHandler } = require('./handlers/custom-options');
 const { browserHeadersHandler } = require('./handlers/browser-headers');
 const { proxyHandler } = require('./handlers/proxy');
+const { alpnHandler } = require('./handlers/alpn');
 
 const isResponseOk = (response) => {
     const { statusCode } = response;
@@ -46,6 +47,9 @@ const gotScraping = got.extend(
         handlers: [
             optionsValidationHandler,
             customOptionsHandler,
+            // ALPN negotiation is handled by got (http2-wrapper) by default.
+            // However, its caching is causing problems with http proxies and https targets on http 1.1
+            alpnHandler,
             proxyHandler,
             browserHeadersHandler,
         ],

test/alpn.test.js

@@ -0,0 +1,43 @@
+const { alpnHandler } = require('../src/handlers/alpn');
+const httpResolver = require('../src/http-resolver');
+
+describe('ALPN', () => {
+    let next;
+    let options;
+
+    beforeEach(() => {
+        options = {
+            context: {},
+            https: {},
+        };
+        next = () => { };
+    });
+
+    afterEach(() => {
+        jest.clearAllMocks();
+    });
+
+    test('should start alpn only if http2 and https', async () => {
+        options.url = 'https://test.com';
+        options.http2 = true;
+        jest.spyOn(httpResolver, 'resolveHttpVersion').mockResolvedValue('h2');
+
+        await alpnHandler(options, next);
+        expect(options.http2).toBe(true);
+
+        jest.spyOn(httpResolver, 'resolveHttpVersion').mockResolvedValue('http/1.1');
+
+        await alpnHandler(options, next);
+        expect(options.http2).toBe(false);
+    });
+
+    test('should skip alpn and assume http/1.1 if not https', async () => {
+        options.url = 'http://test.com';
+        options.http2 = true;
+        jest.spyOn(httpResolver, 'resolveHttpVersion');
+
+        await alpnHandler(options, next);
+        expect(httpResolver.resolveHttpVersion).toBeCalledTimes(0);
+        expect(options.http2).toBe(false);
+    });
+});

test/main.test.js

@@ -25,7 +25,7 @@ describe('GotScraping', () => {
     });
 
     test('should use all handlers', async () => {
-        expect(gotScraping.defaults.handlers).toHaveLength(4);
+        expect(gotScraping.defaults.handlers).toHaveLength(5);
     });
 
     test('should allow passing custom properties', async () => {
@@ -41,7 +41,7 @@ describe('GotScraping', () => {
         expect(options.context.headerGeneratorOptions).toMatchObject(requestOptions.headerGeneratorOptions);
     });
 
-    test('should allow overrding generated options using handlers', async () => {
+    test('should allow overriding generated options using handlers', async () => {
         const requestOptions = {
             url: `http://localhost:${port}/html`,
         };
@@ -75,9 +75,9 @@ describe('GotScraping', () => {
 
         expect(response.statusCode).toBe(200);
         expect(response.request.options).toMatchObject({
-            http2: true,
+            http2: false,
             headers: {
-                'user-agent': 'test',
+                'User-Agent': 'test',
             },
         });
     });
@@ -128,6 +128,27 @@ describe('GotScraping', () => {
             expect(response.statusCode).toBe(200);
             expect(response.httpVersion).toBe('2.0');
         });
+
+        test('Should auto downgrade protocol', async () => {
+            const response = await gotScraping({ url: 'https://eshop.coop-box.cz/' });
+            expect(response.statusCode).toBe(200);
+            expect(response.httpVersion).toBe('1.1');
+            expect(response.request.options.headers.Accept).toBeDefined(); // capitalized headers are proof
+        });
+
+        if (nodeVersion >= 12) {
+            test('Should allow https target via http proxy when auto downgrading', async () => {
+                const response = await gotScraping({
+                    url: 'https://eshop.coop-box.cz/',
+                    proxyUrl: `http://groups-SHADER,session-123:${process.env.APIFY_PROXY_PASSWORD}@proxy.apify.com:8000`,
+
+                });
+                expect(response.statusCode).toBe(200);
+                expect(response.httpVersion).toBe('1.1');
+                expect(response.request.options.headers.Accept).toBeDefined(); // capitalized headers are proof
+            });
+        }
+
         test('should work with proxyUrl and http1', async () => {
             const response = await gotScraping({
                 responseType: 'json',

test/proxy.test.js

@@ -21,7 +21,11 @@ describe('Proxy', () => {
             context: {},
             https: {},
         };
-        next = () => {};
+        next = () => { };
+    });
+
+    afterEach(() => {
+        jest.clearAllMocks();
     });
 
     test('should modify agents only if proxyUrl provided', async () => {

test/scraping-defaults.test.js

@@ -19,7 +19,7 @@ describe('Scraping defaults', () => {
         const { useHeaderGenerator, timeout, ...gotDefaults } = SCRAPING_DEFAULT_OPTIONS;
 
         const response = await gotScraping.get(`http://localhost:${port}/html`);
-        expect(response.request.options).toMatchObject({ ...gotDefaults, timeout: { request: timeout } });
+        expect(response.request.options).toMatchObject({ ...gotDefaults, http2: false, timeout: { request: timeout } });
         expect(response.request.options.context).toMatchObject({ useHeaderGenerator });
     });