From 59209c19c735cacb90d46b7d4569471d726d8206 Mon Sep 17 00:00:00 2001 From: lwthiker Date: Fri, 22 Apr 2022 19:57:46 +0300 Subject: [PATCH 1/9] Add support for linking NSS statically on macOS This is an attmept to link with NSS statically on macOS (both Intel and Apple M1). Statically linking with NSS is a total mess and completely undocumented. There are 20+ .a files to link with, and their linking order matters. The main reference for this commit is a Mozilla Rust code responsible for statically linking NSS: https://github.com/mozilla/application-services/blob/b2690fd2e4cc3e8e10b6868ab0de8b79c89d3a93/components/support/rc_crypto/nss/nss_b uild_common/src/lib.rs#L94 Unfortunately, even with that in hand, a lot of hacking is needed to make it all work. --- Dockerfile.template | 6 +- INSTALL.md | 2 +- Makefile.in | 5 +- firefox/Dockerfile | 2 +- firefox/Dockerfile.alpine | 4 +- firefox/patches/curl-impersonate.patch | 86 ++++++++++++++++++++++++-- 6 files changed, 90 insertions(+), 15 deletions(-) diff --git a/Dockerfile.template b/Dockerfile.template index ed93b0c3..f4b05c09 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -82,11 +82,7 @@ ARG NSS_URL=https://ftp.mozilla.org/pub/security/nss/releases/NSS_3_75_RTM/src/n RUN curl -o ${NSS_VERSION}.tar.gz ${NSS_URL} RUN tar xf ${NSS_VERSION}.tar.gz && \ cd ${NSS_VERSION}/nss && \ -{{#alpine}} - # Hack to make nss compile on alpine with python3 - ln -sf python3 /usr/bin/python && \ -{{/alpine}} - ./build.sh -o --disable-tests --static + ./build.sh -o --disable-tests --static --python=python3 {{/firefox}} {{#chrome}} # BoringSSL doesn't have versions. Choose a commit that is used in a stable diff --git a/INSTALL.md b/INSTALL.md index cac475b2..23362dd6 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -16,7 +16,7 @@ Install dependencies for building all the components: ``` sudo apt install build-essential pkg-config cmake ninja-build curl autoconf automake libtool # For the Firefox version only -sudo apt install python3-pip python-is-python3 +sudo apt install python3-pip pip install gyp-next export PATH="$PATH:~/.local/bin" # Add gyp to PATH # For the Chrome version only diff --git a/Makefile.in b/Makefile.in index ef4c92d3..420ecd38 100644 --- a/Makefile.in +++ b/Makefile.in @@ -126,7 +126,10 @@ $(NSS_VERSION).tar.gz: $(nss_static_libs): $(NSS_VERSION).tar.gz tar xf $(NSS_VERSION).tar.gz cd $(NSS_VERSION)/nss - ./build.sh -o --disable-tests --static + ./build.sh -o --disable-tests --static --python=python3 + # Hack for macOS: Remove dynamic libraries to force the linker to use the + # static ones when linking curl. + rm -Rf $(nss_install_dir)/lib/*.dylib boringssl.zip: diff --git a/firefox/Dockerfile b/firefox/Dockerfile index ee426b2e..3ade9dab 100644 --- a/firefox/Dockerfile +++ b/firefox/Dockerfile @@ -48,7 +48,7 @@ ARG NSS_URL=https://ftp.mozilla.org/pub/security/nss/releases/NSS_3_75_RTM/src/n RUN curl -o ${NSS_VERSION}.tar.gz ${NSS_URL} RUN tar xf ${NSS_VERSION}.tar.gz && \ cd ${NSS_VERSION}/nss && \ - ./build.sh -o --disable-tests --static + ./build.sh -o --disable-tests --static --python=python3 ARG NGHTTP2_VERSION=nghttp2-1.46.0 ARG NGHTTP2_URL=https://github.com/nghttp2/nghttp2/releases/download/v1.46.0/nghttp2-1.46.0.tar.bz2 diff --git a/firefox/Dockerfile.alpine b/firefox/Dockerfile.alpine index e8563044..ed9764c8 100644 --- a/firefox/Dockerfile.alpine +++ b/firefox/Dockerfile.alpine @@ -41,9 +41,7 @@ ARG NSS_URL=https://ftp.mozilla.org/pub/security/nss/releases/NSS_3_75_RTM/src/n RUN curl -o ${NSS_VERSION}.tar.gz ${NSS_URL} RUN tar xf ${NSS_VERSION}.tar.gz && \ cd ${NSS_VERSION}/nss && \ - # Hack to make nss compile on alpine with python3 - ln -sf python3 /usr/bin/python && \ - ./build.sh -o --disable-tests --static + ./build.sh -o --disable-tests --static --python=python3 ARG NGHTTP2_VERSION=nghttp2-1.46.0 ARG NGHTTP2_URL=https://github.com/nghttp2/nghttp2/releases/download/v1.46.0/nghttp2-1.46.0.tar.bz2 diff --git a/firefox/patches/curl-impersonate.patch b/firefox/patches/curl-impersonate.patch index d89fc1ce..3496184d 100644 --- a/firefox/patches/curl-impersonate.patch +++ b/firefox/patches/curl-impersonate.patch @@ -954,19 +954,86 @@ index 8ac15d407..68d01b219 100644 Libs.private: @LIBCURL_LIBS@ Cflags: -I${includedir} @CPPFLAG_CURL_STATICLIB@ diff --git a/m4/curl-nss.m4 b/m4/curl-nss.m4 -index 397ba71b1..abc09a91c 100644 +index 397ba71b1..922cb9a07 100644 --- a/m4/curl-nss.m4 +++ b/m4/curl-nss.m4 -@@ -74,7 +74,7 @@ if test "x$OPT_NSS" != xno; then +@@ -74,7 +74,74 @@ if test "x$OPT_NSS" != xno; then # Without pkg-config, we'll kludge in some defaults AC_MSG_WARN([Using hard-wired libraries and compilation flags for NSS.]) addld="-L$OPT_NSS/lib" - addlib="-lssl3 -lsmime3 -lnss3 -lplds4 -lplc4 -lnspr4" -+ addlib="-Wl,-Bstatic -Wl,--start-group -lssl -lnss_static -lpk11wrap_static -lcertdb -lcerthi -lsmime -lnsspki -lnssdev -lsoftokn_static -lfreebl_static -lsha-x86_c_lib -lgcm-aes-x86_c_lib -lhw-acc-crypto-avx -lhw-acc-crypto-avx2 -lnssutil -lnssb -lcryptohi -l:libplc4.a -l:libplds4.a -l:libnspr4.a -lsqlite -Wl,--end-group -Wl,-Bdynamic -pthread -ldl" ++ ++ # curl-impersonate: Link NSS statically. ++ # NSS is poorly documented in this regard and a lot of trial and error ++ # was made to come up with the correct list of linking flags. The ++ # libraries have circular dependencies which makes their order extremely ++ # difficult to find out. ++ ++ # Some references: ++ # https://github.com/mozilla/application-services/blob/b2690fd2e4cc3e8e10b6868ab0de8b79c89d3a93/components/support/rc_crypto/nss/nss_build_common/src/lib.rs#L94 ++ # and ++ # https://hg.mozilla.org/mozilla-central/file/tip/security/nss/lib/freebl/freebl.gyp ++ ++ # On Linux we can use special linker flags to force static linking ++ # (-l:libplc4.a etc.), otherwise the linker will prefer to use ++ # libplc4.so. On other systems the dynamic libraries would have to be ++ # removed manually from the NSS directory before building curl. ++ case $host_os in ++ linux*) ++ addlib="-lssl -lnss_static -lpk11wrap_static -lcertdb -lcerthi -lnsspki -lnssdev -lsoftokn_static -lfreebl_static -lnssutil -lnssb -lcryptohi -l:libplc4.a -l:libplds4.a -l:libnspr4.a -lsqlite" ++ ;; ++ darwin*) ++ addlib="-lssl -lnss_static -lpk11wrap_static -lcertdb -lcerthi -lnsspki -lnssdev -lsoftokn_static -lfreebl_static -lnssutil -lnssb -lcryptohi -lplc4 -lplds4 -lnspr4" ++ ;; ++ *) ++ addlib="-lssl -lnss_static -lpk11wrap_static -lcertdb -lcerthi -lnsspki -lnssdev -lsoftokn_static -lfreebl_static -lnssutil -lnssb -lcryptohi -lplc4 -lplds4 -lnspr4 -lsqlite" ++ ;; ++ esac ++ ++ case $host_cpu in ++ arm) ++ addlib="$addlib -larmv8_c_lib" ++ ;; ++ aarch64) ++ addlib="$addlib -larmv8_c_lib -lgcm-aes-aarch64_c_lib" ++ ;; ++ x86) ++ addlib="$addlib -lgcm-aes-x86_c_lib" ++ ;; ++ x86_64) ++ addlib="$addlib -lgcm-aes-x86_c_lib -lhw-acc-crypto-avx -lhw-acc-crypto-avx2 -lsha-x86_c_lib" ++ case $host_os in ++ linux*) ++ addlib="$addlib -lintel-gcm-wrap_c_lib -lintel-gcm-s_lib" ++ ;; ++ esac ++ ;; ++ esac ++ ++ # curl-impersonate: ++ # On Linux these linker flags are necessary to resolve ++ # the symbol mess and circular dependencies of NSS .a libraries ++ # to make the AC_CHECK_LIB test below pass. ++ case $host_os in ++ linux*) ++ addlib="-Wl,--start-group $addlib -Wl,--end-group" ++ ;; ++ esac ++ ++ # External dependencies for nss ++ case $host_os in ++ linux*) ++ addlib="$addlib -pthread -ldl" ++ ;; ++ darwin*) ++ addlib="$addlib -lsqlite3" ++ ;; ++ esac ++ addcflags="-I$OPT_NSS/include" version="unknown" nssprefix=$OPT_NSS -@@ -91,7 +91,7 @@ if test "x$OPT_NSS" != xno; then +@@ -91,7 +158,7 @@ if test "x$OPT_NSS" != xno; then fi dnl The function SSL_VersionRangeSet() is needed to enable TLS > 1.0 @@ -975,6 +1042,17 @@ index 397ba71b1..abc09a91c 100644 [ AC_DEFINE(USE_NSS, 1, [if NSS is enabled]) AC_SUBST(USE_NSS, [1]) +@@ -101,9 +168,7 @@ if test "x$OPT_NSS" != xno; then + test nss != "$DEFAULT_SSL_BACKEND" || VALID_DEFAULT_SSL_BACKEND=yes + ], + [ +- LDFLAGS="$CLEANLDFLAGS" +- LIBS="$CLEANLIBS" +- CPPFLAGS="$CLEANCPPFLAGS" ++ AC_MSG_ERROR([Failed linking NSS statically]) + ]) + + if test "x$USE_NSS" = "xyes"; then diff --git a/src/Makefile.am b/src/Makefile.am index c8abc93b1..fcecb10d0 100644 --- a/src/Makefile.am From 8019a00959263932ea934f3d09e4f15f5fefba0f Mon Sep 17 00:00:00 2001 From: lwthiker Date: Fri, 22 Apr 2022 20:47:05 +0300 Subject: [PATCH 2/9] Run the automatic tests on a macOS runner as well --- .github/workflows/build-and-test-make.yml | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index 8a15e688..15f22299 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -14,7 +14,12 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04] + os: [ubuntu-20.04, macos-11] + include: + - os: ubuntu-20.04 + capture_interface: eth0 + - os: macos-11 + capture_interface: en0 steps: - name: Install Ubuntu dependencies if: matrix.os == 'ubuntu-20.04' @@ -30,6 +35,19 @@ jobs: # More dependencies for the tests sudo apt install tcpdump nghttp2-server libnss3 + - name: Install macOS dependencies + if: matrix.os == 'macos-11' + run: | + brew install pkg-config make cmake ninja autoconf automake libtool + # Firefox version dependencies + pip3 install gyp-next + # Chrome version dependencies + brew install go + # Needed to compile 'minicurl' + brew install curl + # More dependencies for the tests + brew install tcpdump nghttp2 nss + - name: Check out the repo uses: actions/checkout@v2 @@ -65,4 +83,4 @@ jobs: run: | cd tests # sudo is needed for capturing packets - sudo pytest . --log-cli-level DEBUG --install-dir ${{ runner.temp}}/install + sudo pytest . --log-cli-level DEBUG --install-dir ${{ runner.temp}}/install --capture-interface ${{ matrix.capture_interface }} From ad77483b70e6ef672f6501246636d875df832773 Mon Sep 17 00:00:00 2001 From: lwthiker Date: Fri, 22 Apr 2022 20:52:33 +0300 Subject: [PATCH 3/9] Don't install Go on macOS GitHub Action runner Go is already installed and conflicts with 'brew install go'. --- .github/workflows/build-and-test-make.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index 15f22299..ca1723bc 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -42,7 +42,8 @@ jobs: # Firefox version dependencies pip3 install gyp-next # Chrome version dependencies - brew install go + # (Go is already installed) + # brew install go # Needed to compile 'minicurl' brew install curl # More dependencies for the tests From 90012e78de356c2921cc7c2f25868f734dbee5ad Mon Sep 17 00:00:00 2001 From: lwthiker Date: Fri, 22 Apr 2022 20:54:00 +0300 Subject: [PATCH 4/9] Fix various issues with automatic tests on macOS --- .github/workflows/build-and-test-docker.yml | 2 +- .github/workflows/build-and-test-make.yml | 32 ++++++++++++--------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-and-test-docker.yml b/.github/workflows/build-and-test-docker.yml index accfc1a0..89a93096 100644 --- a/.github/workflows/build-and-test-docker.yml +++ b/.github/workflows/build-and-test-docker.yml @@ -9,7 +9,7 @@ on: - main jobs: - build-and-test: + build-docker-and-test: name: Build curl-impersonate Docker images and run the tests runs-on: ubuntu-latest services: diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index ca1723bc..deaddfef 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -18,16 +18,17 @@ jobs: include: - os: ubuntu-20.04 capture_interface: eth0 + make: make - os: macos-11 capture_interface: en0 + make: gmake steps: + - uses: actions/setup-python@v3 + - name: Install Ubuntu dependencies if: matrix.os == 'ubuntu-20.04' run: | sudo apt install build-essential pkg-config cmake ninja-build curl autoconf automake libtool - # Firefox version dependencies - sudo apt install python3-pip python-is-python3 - pip install gyp-next # Chrome version dependencies sudo apt install golang-go # Needed to compile 'minicurl' @@ -39,8 +40,6 @@ jobs: if: matrix.os == 'macos-11' run: | brew install pkg-config make cmake ninja autoconf automake libtool - # Firefox version dependencies - pip3 install gyp-next # Chrome version dependencies # (Go is already installed) # brew install go @@ -49,13 +48,17 @@ jobs: # More dependencies for the tests brew install tcpdump nghttp2 nss + - name: Install common dependencies + run: | + # Firefox version dependencies + pip3 install gyp-next + - name: Check out the repo uses: actions/checkout@v2 - name: Install dependencies for the tests script run: | - # Install globally so that we can run 'pytest' with 'sudo' - sudo pip install -r tests/requirements.txt + pip3 install -r tests/requirements.txt - name: Run configure script run: | @@ -65,15 +68,15 @@ jobs: - name: Build the Chrome version of curl-impersonate run: | - make chrome-build - make chrome-checkbuild - make chrome-install + ${{ matrix.make }} chrome-build + ${{ matrix.make }} chrome-checkbuild + ${{ matrix.make }} chrome-install - name: Build the Firefox version of curl-impersonate run: | - make firefox-build - make firefox-checkbuild - make firefox-install + ${{ matrix.make }} firefox-build + ${{ matrix.make }} firefox-checkbuild + ${{ matrix.make }} firefox-install - name: Prepare the tests run: | @@ -84,4 +87,5 @@ jobs: run: | cd tests # sudo is needed for capturing packets - sudo pytest . --log-cli-level DEBUG --install-dir ${{ runner.temp}}/install --capture-interface ${{ matrix.capture_interface }} + python_bin=$(which python3) + sudo $python_bin -m pytest . --log-cli-level DEBUG --install-dir ${{ runner.temp}}/install --capture-interface ${{ matrix.capture_interface }} From d0c5777420aae23fa1cceb2b218e2133f15e2801 Mon Sep 17 00:00:00 2001 From: lwthiker Date: Sat, 23 Apr 2022 10:43:57 +0300 Subject: [PATCH 5/9] Run the LD_PRELOAD injection tests on Linux only Injecting libcurl-impersonate with LD_PRELOAD is supported on Linux only. On Mac there is DYLD_INSERT_LIBRARIES but it reuqires more work to be fully functional. --- tests/test_impersonate.py | 56 ++++++++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/tests/test_impersonate.py b/tests/test_impersonate.py index 75c84a16..1719a2bf 100644 --- a/tests/test_impersonate.py +++ b/tests/test_impersonate.py @@ -1,6 +1,7 @@ import os import io import re +import sys import logging import subprocess import tempfile @@ -145,7 +146,7 @@ class TestImpersonation: { "CURL_IMPERSONATE": "chrome98" }, - "libcurl-impersonate-chrome.so", + "libcurl-impersonate-chrome", "chrome_98.0.4758.102_win10" ), ( @@ -153,7 +154,7 @@ class TestImpersonation: { "CURL_IMPERSONATE": "chrome99" }, - "libcurl-impersonate-chrome.so", + "libcurl-impersonate-chrome", "chrome_99.0.4844.51_win10" ), ( @@ -161,7 +162,7 @@ class TestImpersonation: { "CURL_IMPERSONATE": "chrome99_android" }, - "libcurl-impersonate-chrome.so", + "libcurl-impersonate-chrome", "chrome_99.0.4844.73_android12-pixel6" ), ( @@ -169,7 +170,7 @@ class TestImpersonation: { "CURL_IMPERSONATE": "edge98" }, - "libcurl-impersonate-chrome.so", + "libcurl-impersonate-chrome", "edge_98.0.1108.62_win10" ), ( @@ -177,7 +178,7 @@ class TestImpersonation: { "CURL_IMPERSONATE": "edge99" }, - "libcurl-impersonate-chrome.so", + "libcurl-impersonate-chrome", "edge_99.0.1150.30_win10" ), ( @@ -185,7 +186,7 @@ class TestImpersonation: { "CURL_IMPERSONATE": "safari15_3" }, - "libcurl-impersonate-chrome.so", + "libcurl-impersonate-chrome", "safari_15.3_macos11.6.4" ), ( @@ -193,7 +194,7 @@ class TestImpersonation: { "CURL_IMPERSONATE": "ff91esr" }, - "libcurl-impersonate-ff.so", + "libcurl-impersonate-ff", "firefox_91.6.0esr_win10" ), ( @@ -201,7 +202,7 @@ class TestImpersonation: { "CURL_IMPERSONATE": "ff95" }, - "libcurl-impersonate-ff.so", + "libcurl-impersonate-ff", "firefox_95.0.2_win10" ), ( @@ -209,7 +210,7 @@ class TestImpersonation: { "CURL_IMPERSONATE": "ff98" }, - "libcurl-impersonate-ff.so", + "libcurl-impersonate-ff", "firefox_98.0_win10" ) ] @@ -255,6 +256,12 @@ def nghttpd(self): p.terminate() p.wait(timeout=10) + def _set_ld_preload(self, env_vars, lib): + if sys.platform.startswith("linux"): + env_vars["LD_PRELOAD"] = lib + ".so" + elif sys.platform.startswith("darwin"): + env_vars["DYLD_INSERT_LIBRARIES"] = lib + ".dylib" + def _run_curl(self, curl_binary, env_vars, extra_args, url, output="/dev/null"): env = os.environ.copy() @@ -368,9 +375,15 @@ def test_tls_client_hello(self, pytestconfig.getoption("install_dir"), "bin", curl_binary ) if ld_preload: - env_vars["LD_PRELOAD"] = os.path.join( + # Injecting libcurl-impersonate with LD_PRELOAD is supported on + # Linux only. On Mac there is DYLD_INSERT_LIBRARIES but it + # reuqires more work to be functional. + if not sys.platform.startswith("linux"): + pytest.skip() + + self._set_ld_preload(env_vars, os.path.join( pytestconfig.getoption("install_dir"), "lib", ld_preload - ) + )) ret = self._run_curl(curl_binary, env_vars=env_vars, @@ -425,9 +438,16 @@ def test_http2_headers(self, pytestconfig.getoption("install_dir"), "bin", curl_binary ) if ld_preload: - env_vars["LD_PRELOAD"] = os.path.join( + # Injecting libcurl-impersonate with LD_PRELOAD is supported on + # Linux only. On Mac there is DYLD_INSERT_LIBRARIES but it + # reuqires more work to be functional. + if not sys.platform.startswith("linux"): + pytest.skip() + + self._set_ld_preload(env_vars, os.path.join( pytestconfig.getoption("install_dir"), "lib", ld_preload - ) + )) + ret = self._run_curl(curl_binary, env_vars=env_vars, extra_args=["-k"], @@ -481,9 +501,15 @@ def test_content_encoding(self, pytestconfig.getoption("install_dir"), "bin", curl_binary ) if ld_preload: - env_vars["LD_PRELOAD"] = os.path.join( + # Injecting libcurl-impersonate with LD_PRELOAD is supported on + # Linux only. On Mac there is DYLD_INSERT_LIBRARIES but it + # reuqires more work to be functional. + if not sys.platform.startswith("linux"): + pytest.skip() + + self._set_ld_preload(env_vars, os.path.join( pytestconfig.getoption("install_dir"), "lib", ld_preload - ) + )) output = tempfile.mkstemp()[1] ret = self._run_curl(curl_binary, From ea639732568ef537009b77106a17dd163875d772 Mon Sep 17 00:00:00 2001 From: lwthiker Date: Sat, 30 Apr 2022 11:33:19 +0300 Subject: [PATCH 6/9] tests: Use asyncio to wait for nghttpd to start Seems like the tests may fail because we don't wait for nghttpd (the HTTP2 server used for testing) to actually start listening. This commit uses asyncio to launch nghttpd and read its stdout until it starts listening. --- tests/pytest.ini | 2 + tests/requirements.txt | 1 + tests/test_impersonate.py | 94 ++++++++++++++++++++++++++++----------- 3 files changed, 71 insertions(+), 26 deletions(-) create mode 100644 tests/pytest.ini diff --git a/tests/pytest.ini b/tests/pytest.ini new file mode 100644 index 00000000..2f4c80e3 --- /dev/null +++ b/tests/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +asyncio_mode = auto diff --git a/tests/requirements.txt b/tests/requirements.txt index fa2223ae..f05a4938 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,3 +1,4 @@ pyyaml pytest +pytest-asyncio dpkt diff --git a/tests/test_impersonate.py b/tests/test_impersonate.py index 1719a2bf..83821a5b 100644 --- a/tests/test_impersonate.py +++ b/tests/test_impersonate.py @@ -2,6 +2,7 @@ import io import re import sys +import asyncio import logging import subprocess import tempfile @@ -241,20 +242,69 @@ def tcpdump(self, pytestconfig): p.terminate() p.wait(timeout=10) + async def _read_proc_output(self, proc, timeout): + """Read an async process' output until timeout is reached""" + data = bytes() + loop = asyncio.get_running_loop() + start_time = loop.time() + passed = loop.time() - start_time + while passed < timeout: + try: + data += await asyncio.wait_for( + proc.stdout.readline(), timeout=timeout - passed + ) + except asyncio.TimeoutError: + pass + passed = loop.time() - start_time + return data + + async def _wait_nghttpd(self, proc): + """Wait for nghttpd to start listening on its designated port""" + data = bytes() + while data is not None: + data = await proc.stdout.readline() + if not data: + # Process terminated + return False + + line = data.decode("utf-8").rstrip() + if "listen 0.0.0.0:8443" in line: + return True + + return False + @pytest.fixture - def nghttpd(self): - """Initiailize an HTTP/2 server""" + async def nghttpd(self): + """Initiailize an HTTP/2 server. + The returned object is an asyncio.subprocess.Process object, + so async methods must be used with it. + """ logging.debug(f"Running nghttpd on :8443") - p = subprocess.Popen([ + # Launch nghttpd and wait for it to start listening. + + proc = await asyncio.create_subprocess_exec( "nghttpd", "-v", - "8443", "ssl/server.key", "ssl/server.crt" - ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + "8443", "ssl/server.key", "ssl/server.crt", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE + ) - yield p + try: + # Wait up to 3 seconds for nghttpd to start. + # Otherwise fail. + started = await asyncio.wait_for( + self._wait_nghttpd(proc), timeout=3 + ) + if not started: + raise Exception("nghttpd failed to start on time") + except asyncio.TimeoutError: + raise Exception("nghttpd failed to start on time") - p.terminate() - p.wait(timeout=10) + yield proc + + proc.terminate() + await proc.wait() def _set_ld_preload(self, env_vars, lib): if sys.platform.startswith("linux"): @@ -422,18 +472,19 @@ def test_tls_client_hello(self, equals, msg = sig.equals(expected_sig, reason=True) assert equals, msg + @pytest.mark.asyncio @pytest.mark.parametrize( "curl_binary, env_vars, ld_preload, expected_signature", CURL_BINARIES_AND_SIGNATURES ) - def test_http2_headers(self, - pytestconfig, - nghttpd, - curl_binary, - env_vars, - ld_preload, - browser_signatures, - expected_signature): + async def test_http2_headers(self, + pytestconfig, + nghttpd, + curl_binary, + env_vars, + ld_preload, + browser_signatures, + expected_signature): curl_binary = os.path.join( pytestconfig.getoption("install_dir"), "bin", curl_binary ) @@ -453,17 +504,8 @@ def test_http2_headers(self, extra_args=["-k"], url="https://localhost:8443") assert ret == 0 - try: - output, stderr = nghttpd.communicate(timeout=2) - # If nghttpd finished running before timeout, it's likely it failed - # with an error. - assert nghttpd.returncode == 0, \ - (f"nghttpd failed with error code {nghttpd.returncode}, " - f"stderr: {stderr}") - except subprocess.TimeoutExpired: - nghttpd.kill() - output, stderr = nghttpd.communicate(timeout=3) + output = await self._read_proc_output(nghttpd, timeout=2) assert len(output) > 0 pseudo_headers, headers = self._parse_nghttpd2_output(output) From 2b4b284cc803fec40c181956f1313b30f74878ff Mon Sep 17 00:00:00 2001 From: lwthiker Date: Sat, 30 Apr 2022 11:42:37 +0300 Subject: [PATCH 7/9] Add 'apt-get update' in GitHub action workflow And change 'apt' to 'apt-get' to prevent the warning message: 'WARNING: apt does not have a stable CLI interface. Use with caution in scripts.' --- .github/workflows/build-and-test-make.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-and-test-make.yml b/.github/workflows/build-and-test-make.yml index deaddfef..2d4c89d1 100644 --- a/.github/workflows/build-and-test-make.yml +++ b/.github/workflows/build-and-test-make.yml @@ -28,13 +28,14 @@ jobs: - name: Install Ubuntu dependencies if: matrix.os == 'ubuntu-20.04' run: | - sudo apt install build-essential pkg-config cmake ninja-build curl autoconf automake libtool + sudo apt-get update + sudo apt-get install build-essential pkg-config cmake ninja-build curl autoconf automake libtool # Chrome version dependencies - sudo apt install golang-go + sudo apt-get install golang-go # Needed to compile 'minicurl' - sudo apt install libcurl4-openssl-dev + sudo apt-get install libcurl4-openssl-dev # More dependencies for the tests - sudo apt install tcpdump nghttp2-server libnss3 + sudo apt-get install tcpdump nghttp2-server libnss3 - name: Install macOS dependencies if: matrix.os == 'macos-11' From 31e61775a30ce9dd91ca99ab3776caf7ff5293e9 Mon Sep 17 00:00:00 2001 From: lwthiker Date: Sat, 30 Apr 2022 13:23:52 +0300 Subject: [PATCH 8/9] Search for libnssckbi in curl's configure script libnssckbi is loaded at runtime by NSS. On some systems it is located in a non-standard location that dlopen() can't find. For example, in Ubuntu it may be in /usr/lib/x86_64-linux-gnu and on Mac M1 in /opt/homebrew/nss. This becomes a problem when you static link NSS. Search for libnssckbi in the configure script and add the relevant path using '-rpath' linker flag. In addition, drop the previous hack for Ubuntu that searched libnssckbi in a hardcoded location. --- firefox/patches/curl-impersonate.patch | 70 ++++++++++++++++---------- 1 file changed, 43 insertions(+), 27 deletions(-) diff --git a/firefox/patches/curl-impersonate.patch b/firefox/patches/curl-impersonate.patch index 3496184d..ba6b6045 100644 --- a/firefox/patches/curl-impersonate.patch +++ b/firefox/patches/curl-impersonate.patch @@ -760,7 +760,7 @@ index cc9c88870..a35a20e10 100644 killed. */ struct dynamically_allocated_data { diff --git a/lib/vtls/nss.c b/lib/vtls/nss.c -index 2b44f0512..4c60797c7 100644 +index 2b44f0512..eec2bf76f 100644 --- a/lib/vtls/nss.c +++ b/lib/vtls/nss.c @@ -143,6 +143,7 @@ static const struct cipher_s cipherlist[] = { @@ -867,32 +867,15 @@ index 2b44f0512..4c60797c7 100644 /* * Return true if at least one cipher-suite is enabled. Used to determine * if we need to call NSS_SetDomesticPolicy() to enable the default ciphers. -@@ -1320,6 +1410,24 @@ static CURLcode nss_load_module(SECMODModule **pmod, const char *library, +@@ -1320,6 +1410,7 @@ static CURLcode nss_load_module(SECMODModule **pmod, const char *library, if(module) SECMOD_DestroyModule(module); -+ -+ /* Patch for Ubuntu - add a "nss/" suffix to the library name */ -+ config_string = aprintf("library=/usr/lib/x86_64-linux-gnu/nss/%s name=%s", library, name); -+ if(!config_string) -+ return CURLE_OUT_OF_MEMORY; -+ -+ module = SECMOD_LoadUserModule(config_string, NULL, PR_FALSE); -+ free(config_string); -+ -+ if(module && module->loaded) { -+ /* loaded successfully */ -+ *pmod = module; -+ return CURLE_OK; -+ } -+ -+ if(module) -+ SECMOD_DestroyModule(module); + return CURLE_FAILED_INIT; } -@@ -1921,6 +2029,12 @@ static CURLcode nss_setup_connect(struct Curl_easy *data, +@@ -1921,6 +2012,12 @@ static CURLcode nss_setup_connect(struct Curl_easy *data, if(SSL_OptionSet(model, SSL_NO_CACHE, ssl_no_cache) != SECSuccess) goto error; @@ -905,7 +888,7 @@ index 2b44f0512..4c60797c7 100644 /* enable/disable the requested SSL version(s) */ if(nss_init_sslver(&sslver, data, conn) != CURLE_OK) goto error; -@@ -1960,6 +2074,14 @@ static CURLcode nss_setup_connect(struct Curl_easy *data, +@@ -1960,6 +2057,14 @@ static CURLcode nss_setup_connect(struct Curl_easy *data, } } @@ -920,7 +903,7 @@ index 2b44f0512..4c60797c7 100644 if(!SSL_CONN_CONFIG(verifypeer) && SSL_CONN_CONFIG(verifyhost)) infof(data, "warning: ignoring value of ssl.verifyhost"); -@@ -2113,6 +2235,10 @@ static CURLcode nss_setup_connect(struct Curl_easy *data, +@@ -2113,6 +2218,10 @@ static CURLcode nss_setup_connect(struct Curl_easy *data, int cur = 0; unsigned char protocols[128]; @@ -931,7 +914,7 @@ index 2b44f0512..4c60797c7 100644 #ifdef USE_HTTP2 if(data->state.httpwant >= CURL_HTTP_VERSION_2 #ifndef CURL_DISABLE_PROXY -@@ -2124,9 +2250,6 @@ static CURLcode nss_setup_connect(struct Curl_easy *data, +@@ -2124,9 +2233,6 @@ static CURLcode nss_setup_connect(struct Curl_easy *data, cur += ALPN_H2_LENGTH; } #endif @@ -954,10 +937,10 @@ index 8ac15d407..68d01b219 100644 Libs.private: @LIBCURL_LIBS@ Cflags: -I${includedir} @CPPFLAG_CURL_STATICLIB@ diff --git a/m4/curl-nss.m4 b/m4/curl-nss.m4 -index 397ba71b1..922cb9a07 100644 +index 397ba71b1..d2a8fc1f2 100644 --- a/m4/curl-nss.m4 +++ b/m4/curl-nss.m4 -@@ -74,7 +74,74 @@ if test "x$OPT_NSS" != xno; then +@@ -74,7 +74,107 @@ if test "x$OPT_NSS" != xno; then # Without pkg-config, we'll kludge in some defaults AC_MSG_WARN([Using hard-wired libraries and compilation flags for NSS.]) addld="-L$OPT_NSS/lib" @@ -1029,11 +1012,44 @@ index 397ba71b1..922cb9a07 100644 + addlib="$addlib -lsqlite3" + ;; + esac ++ ++ # Attempt to locate libnssckbi. ++ # This library file contains the trusted certificates and nss loads it ++ # at runtime using dlopen. If it's not in a path findable by dlopen ++ # we have to add that path explicitly using -rpath so it may find it. ++ # On Ubuntu and Mac M1 it is in a non-standard location. ++ AC_MSG_CHECKING([if libnssckbi is in a non-standard location]) ++ case $host_os in ++ linux*) ++ search_paths="/usr/lib/$host /usr/lib/$host/nss" ++ search_paths="$search_paths /usr/lib/$host_cpu-$host_os" ++ search_paths="$search_paths /usr/lib/$host_cpu-$host_os/nss" ++ search_ext="so" ++ ;; ++ darwin*) ++ search_paths="/opt/homebrew/lib" ++ search_ext="dylib" ++ ;; ++ esac ++ ++ found="no" ++ for path in $search_paths; do ++ if test -f "$path/libnssckbi.$search_ext"; then ++ AC_MSG_RESULT([$path]) ++ addld="$addld -Wl,-rpath,$path" ++ found="yes" ++ break ++ fi ++ done ++ ++ if test "$found" = "no"; then ++ AC_MSG_RESULT([no]) ++ fi + addcflags="-I$OPT_NSS/include" version="unknown" nssprefix=$OPT_NSS -@@ -91,7 +158,7 @@ if test "x$OPT_NSS" != xno; then +@@ -91,7 +191,7 @@ if test "x$OPT_NSS" != xno; then fi dnl The function SSL_VersionRangeSet() is needed to enable TLS > 1.0 @@ -1042,7 +1058,7 @@ index 397ba71b1..922cb9a07 100644 [ AC_DEFINE(USE_NSS, 1, [if NSS is enabled]) AC_SUBST(USE_NSS, [1]) -@@ -101,9 +168,7 @@ if test "x$OPT_NSS" != xno; then +@@ -101,9 +201,7 @@ if test "x$OPT_NSS" != xno; then test nss != "$DEFAULT_SSL_BACKEND" || VALID_DEFAULT_SSL_BACKEND=yes ], [ From e3a552f90cb7eb4d1ea1d7621ce7c051073ebe0e Mon Sep 17 00:00:00 2001 From: lwthiker Date: Sat, 30 Apr 2022 13:31:41 +0300 Subject: [PATCH 9/9] Update README.md and INSTALL.md for macOS --- INSTALL.md | 21 ++++++++++++++++++--- README.md | 2 +- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 23362dd6..59186299 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -16,7 +16,7 @@ Install dependencies for building all the components: ``` sudo apt install build-essential pkg-config cmake ninja-build curl autoconf automake libtool # For the Firefox version only -sudo apt install python3-pip +sudo apt install python3-pip libnss3 pip install gyp-next export PATH="$PATH:~/.local/bin" # Add gyp to PATH # For the Chrome version only @@ -65,11 +65,12 @@ curl-impersonate-chrome https://www.wikipedia.org ``` ### macOS -*macOS support is still a work in progress and currently supports the Chrome version only.* - Install dependencies for building all the components: ``` brew install pkg-config make cmake ninja autoconf automake libtool +# For the Firefox version only +brew install sqlite nss +pip3 install gyp-next # For the Chrome version only brew install go ``` @@ -83,6 +84,9 @@ Configure and compile: ``` mkdir build && cd build ../configure +# Build and install the Firefox version +gmake firefox-build +sudo gmake firefox-install # Build and install the Chrome version gmake chrome-build sudo gmake chrome-install @@ -93,6 +97,17 @@ cd ../ && rm -Rf build ### Static compilation To compile curl-impersonate statically with libcurl-impersonate, pass `--enable-static` to the `configure` script. +### A note about the Firefox version +The Firefox version compiles a static version of nss, Firefox's TLS library. +For NSS to have a list of root certificates, curl attempts to load at runtime `libnssckbi`, one of the NSS libraries. +If you get the error: +``` +curl: (60) Peer's Certificate issuer is not recognized +``` +Make sure that NSS is installed (see above). +If the issue persists it might be that NSS is installed in a non-standard location on your system. +Please open an issue in that case. + ## Docker build The Docker build is a bit more reproducible and serves as the reference implementation. It creates a Debian-based Docker image with the binaries. diff --git a/README.md b/README.md index 7c5e7b5d..d6e1485f 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ You can call it with the target names, e.g. `chrome98`, and it will internally s Note that if you call `curl_easy_setopt()` later with one of the above it will override the options set by `curl_easy_impersonate()`. ### Using CURL_IMPERSONATE env var -*Experimental*: If your application uses `libcurl` already, you can replace the existing library at runtime with `LD_PRELOAD`. You can then set the `CURL_IMPERSONATE` env var. For example: +*Experimental*: If your application uses `libcurl` already, you can replace the existing library at runtime with `LD_PRELOAD` (Linux only). You can then set the `CURL_IMPERSONATE` env var. For example: ```bash LD_PRELOAD=/path/to/libcurl-impersonate.so CURL_IMPERSONATE=chrome98 my_app ```