Fix NSS KeyLog cannot decrypt TLS1.3 traffic.

doc/notebooks/tls/notebook3_tls_compromised.ipynb

@@ -12,89 +12,89 @@
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
    "metadata": {},
-   "outputs": [],
+   "cell_type": "code",
    "source": [
     "from scapy.all import *\n",
     "load_layer('tls')"
-   ]
+   ],
+   "outputs": [],
+   "execution_count": null
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
    "metadata": {},
-   "outputs": [],
+   "cell_type": "code",
    "source": [
     "record1_str = open('raw_data/tls_session_compromised/01_cli.raw', 'rb').read()\n",
     "record1 = TLS(record1_str)\n",
     "record1.msg[0].show()"
-   ]
+   ],
+   "outputs": [],
+   "execution_count": null
   },
   {
    "cell_type": "code",
-   "execution_count": null,
    "metadata": {
     "scrolled": true
    },
-   "outputs": [],
    "source": [
     "record2_str = open('raw_data/tls_session_compromised/02_srv.raw', 'rb').read()\n",
     "record2 = TLS(record2_str, tls_session=record1.tls_session.mirror())\n",
     "record2.msg[0].show()"
-   ]
+   ],
+   "outputs": [],
+   "execution_count": null
   },
   {
    "cell_type": "code",
-   "execution_count": null,
    "metadata": {},
-   "outputs": [],
    "source": [
     "# Supposing that the private key of the server was stolen,\n",
     "# the traffic can be decoded by registering it to the Scapy TLS session\n",
     "key = PrivKey('raw_data/pki/srv_key.pem')\n",
     "record2.tls_session.server_rsa_key = key"
-   ]
+   ],
+   "outputs": [],
+   "execution_count": null
   },
   {
    "cell_type": "code",
-   "execution_count": null,
    "metadata": {},
-   "outputs": [],
    "source": [
     "record3_str = open('raw_data/tls_session_compromised/03_cli.raw', 'rb').read()\n",
     "record3 = TLS(record3_str, tls_session=record2.tls_session.mirror())\n",
     "record3.show()"
-   ]
+   ],
+   "outputs": [],
+   "execution_count": null
   },
   {
    "cell_type": "code",
-   "execution_count": null,
    "metadata": {},
-   "outputs": [],
    "source": [
     "record4_str = open('raw_data/tls_session_compromised/04_srv.raw', 'rb').read()\n",
     "record4 = TLS(record4_str, tls_session=record3.tls_session.mirror())\n",
     "record4.show()"
-   ]
+   ],
+   "outputs": [],
+   "execution_count": null
   },
   {
    "cell_type": "code",
-   "execution_count": null,
    "metadata": {},
-   "outputs": [],
    "source": [
     "# This is the first TLS Record containing user data. If decryption works,\n",
     "# you should see the string \"To boldly go where no man has gone before...\" in plaintext.\n",
     "record5_str = open('raw_data/tls_session_compromised/05_cli.raw', 'rb').read()\n",
     "record5 = TLS(record5_str, tls_session=record4.tls_session.mirror())\n",
     "record5.show()"
-   ]
+   ],
+   "outputs": [],
+   "execution_count": null
   },
   {
-   "cell_type": "markdown",
    "metadata": {},
+   "cell_type": "markdown",
    "source": [
     "# Decrypting TLS Traffic Protected with PFS\n",
     "\n",
@@ -104,14 +104,15 @@
     "```\n",
     "cd doc/notebooks/tls/raw_data/\n",
     "\n",
-    "# Start a TLS 1.12 Server using the s_server\n",
-    "sudo openssl s_server -accept localhost:443 -cert pki/srv_cert.pem -key pki/srv_key.pem -WWW -tls1_2\n",
+    "# Start a TLS Server using the s_server\n",
+    "sudo openssl s_server -accept localhost:443 -cert pki/srv_cert.pem -key pki/srv_key.pem -WWW\n",
     "\n",
     "# Sniff the network and write packets to a file\n",
     "sudo tcpdump -i lo -w tls_nss_example.pcap port 443\n",
     "\n",
-    "# Connect to the server using s_client and retrieve the secrets.txt file\n",
-    "openssl s_client -connect localhost:443 -keylogfile tls_nss_example.keys.txt\n",
+    "# Connect to the server using TLS 1.2 and TLS 1.3, and write the keys to a file\n",
+    "echo -e \"GET /pki/srv_key.pem HTTP/1.0\\r\\n\" | openssl s_client -connect localhost:443 -keylogfile tls_nss_example.keys.txt -tls1_2 -ign_eof\n",
+    "echo -e \"GET /pki/srv_key.pem HTTP/1.0\\r\\n\" | openssl s_client -connect localhost:443 -keylogfile tls_nss_example.keys.txt -tls1_3 -ign_eof\n",
     "```\n",
     "\n",
     "## Decrypt a PCAP files\n",
@@ -120,38 +121,58 @@
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
    "metadata": {},
-   "outputs": [],
+   "cell_type": "code",
    "source": [
     "load_layer(\"tls\")\n",
     "\n",
     "conf.tls_session_enable = True\n",
     "conf.tls_nss_filename = \"raw_data/tls_nss_example.keys.txt\"\n",
     "\n",
-    "packets = rdpcap(\"raw_data/tls_nss_example.pcap\")"
-   ]
+    "packets = sniff(offline=\"raw_data/tls_nss_example.pcap\", session=TCPSession)"
+   ],
+   "outputs": [],
+   "execution_count": null
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
    "metadata": {},
-   "outputs": [],
+   "cell_type": "code",
    "source": [
-    "# Display the HTTP GET query\n",
-    "packets[11][TLS].show()"
-   ]
+    "# Display the TLS1.2 HTTP GET query\n",
+    "packets[9][TLS].show()"
+   ],
+   "outputs": [],
+   "execution_count": null
   },
   {
+   "metadata": {},
    "cell_type": "code",
-   "execution_count": null,
+   "source": [
+    "# Display the answer containing the secret\n",
+    "packets[10][TLS].show()"
+   ],
+   "outputs": [],
+   "execution_count": null
+  },
+  {
    "metadata": {},
+   "cell_type": "code",
+   "source": [
+    "# Display the TLS1.3 HTTP GET query\n",
+    "packets[27][TLS13].show()"
+   ],
    "outputs": [],
+   "execution_count": null
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
    "source": [
     "# Display the answer containing the secret\n",
-    "packets[13][TLS].show()"
-   ]
+    "packets[28][TLS13].show()"
+   ],
+   "outputs": [],
+   "execution_count": null
   },
   {
    "cell_type": "markdown",
@@ -166,24 +187,23 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
    "metadata": {},
-   "outputs": [],
    "source": [
     "# Read packets from a pcap\n",
     "load_layer(\"tls\")\n",
     "\n",
+    "conf.tls_session_enable = False\n",
     "packets = rdpcap(\"raw_data/tls_nss_example.pcap\")\n",
     "\n",
     "# Load the keys from a NSS Key Log\n",
     "nss_keys = load_nss_keys(\"raw_data/tls_nss_example.keys.txt\")"
-   ]
+   ],
+   "outputs": [],
+   "execution_count": null
   },
   {
    "cell_type": "code",
-   "execution_count": null,
    "metadata": {},
-   "outputs": [],
    "source": [
     "# Parse the Client Hello message from its raw bytes. This configures a new tlsSession object\n",
     "client_hello = TLS(raw(packets[3][TLS]))\n",
@@ -192,34 +212,37 @@
     "server_hello = TLS(raw(packets[5][TLS]), tls_session=client_hello.tls_session.mirror())\n",
     "\n",
     "# Configure the TLS master secret retrieved from the NSS Key Log\n",
-    "server_hello.tls_session.master_secret = nss_keys[\"CLIENT_RANDOM\"][\"Secret\"]\n",
+    "server_hello.tls_session.master_secret = nss_keys[\"CLIENT_RANDOM\"][client_hello.tls_session.client_random]\n",
+    "server_hello.tls_session.compute_ms_and_derive_keys()\n",
     "\n",
     "# Parse remaining TLS messages\n",
     "client_finished = TLS(raw(packets[7][TLS]), tls_session=server_hello.tls_session.mirror())\n",
-    "server_finished = TLS(raw(packets[9][TLS]), tls_session=client_finished.tls_session.mirror())"
-   ]
+    "server_finished = TLS(raw(packets[8][TLS]), tls_session=client_finished.tls_session.mirror())"
+   ],
+   "outputs": [],
+   "execution_count": null
   },
   {
    "cell_type": "code",
-   "execution_count": null,
    "metadata": {},
-   "outputs": [],
    "source": [
     "# Display the HTTP GET query\n",
-    "http_query = TLS(raw(packets[11][TLS]), tls_session=server_finished.tls_session.mirror())\n",
+    "http_query = TLS(raw(packets[9][TLS]), tls_session=server_finished.tls_session.mirror())\n",
     "http_query.show()"
-   ]
+   ],
+   "outputs": [],
+   "execution_count": null
   },
   {
    "cell_type": "code",
-   "execution_count": null,
    "metadata": {},
-   "outputs": [],
    "source": [
     "# Display the answer containing the secret\n",
-    "http_response = TLS(raw(packets[13][TLS]), tls_session=http_query.tls_session.mirror())\n",
+    "http_response = TLS(raw(packets[10][TLS]), tls_session=http_query.tls_session.mirror())\n",
     "http_response.show()"
-   ]
+   ],
+   "outputs": [],
+   "execution_count": null
   }
  ],
  "metadata": {

doc/notebooks/tls/raw_data/tls_nss_example.keys.txt

@@ -1,2 +1,7 @@
 # SSL/TLS secrets log file, generated by OpenSSL
-CLIENT_RANDOM c43c799f04ad31e397ee4fe14c8819a19bf5951bbc545cada407c6c7589e60ab b599798159244555ddd10d80b5552a37d327fd6e661f3520194c28ef6e8bb0af6e3fb4d4f9945a61e83a41f2345fa27a
+CLIENT_RANDOM 216e876ea1a480c60145c4c80eb8d05c85b6806043105c391236cd4e88f79a21 54a828bfc25edf47070cd48b8253e8137e88082face8d7e96960756653b57f41bc6df3f45a5746bc9c6305ccd9b35ab8
+SERVER_HANDSHAKE_TRAFFIC_SECRET 74ef95570af6a305910ee6cb0f98fc5bcec0c5d5dffe5f293ae9a4d7ba2110f2 5f2fd60aecc80ee54d17d48ec58fcfccf6fe229e08055dba1a6a09297bea98fd1268bdd6fe19e15c76d7c152d17f7237
+EXPORTER_SECRET 74ef95570af6a305910ee6cb0f98fc5bcec0c5d5dffe5f293ae9a4d7ba2110f2 02aa67e90b524002f7eb00fcda23365ca6bfea5ad179d965264b5c1f6ff93483465b3c147c5070a90e47a406bd431152
+SERVER_TRAFFIC_SECRET_0 74ef95570af6a305910ee6cb0f98fc5bcec0c5d5dffe5f293ae9a4d7ba2110f2 c5f265aee5d17472c71fa889cfa351b12b9280bf74d16477161fd495c87432632908cae923e390d5d52a4719c2f896de
+CLIENT_HANDSHAKE_TRAFFIC_SECRET 74ef95570af6a305910ee6cb0f98fc5bcec0c5d5dffe5f293ae9a4d7ba2110f2 bf58ee2a720cb26a594c0c7b714783a406f4daad18fbf7b7b3437bfe944d840cbc0e1843096e1c4ec92b68f230b22fa9
+CLIENT_TRAFFIC_SECRET_0 74ef95570af6a305910ee6cb0f98fc5bcec0c5d5dffe5f293ae9a4d7ba2110f2 7f3ac59f48dbe7f0fa66f92a0e691cf6ad4b84062e66b303f3149107c723ffb8424f8a3488072a8938d842b403e43229

scapy/layers/tls/session.py

@@ -731,6 +731,13 @@ def compute_tls13_early_secrets(self, external=False):
                                  b"".join(self.handshake_messages))
         self.tls13_derived_secrets["early_exporter_secret"] = ees
 
+        if self.nss_keys:
+            cets = self.nss_keys.get('CLIENT_EARLY_TRAFFIC_SECRET', {}).get(self.client_random, cets)
+            self.tls13_derived_secrets["client_early_traffic_secret"] = cets
+
+            ees = self.nss_keys.get('EARLY_EXPORTER_SECRET', {}).get(self.client_random, ees)
+            self.tls13_derived_secrets["early_exporter_secret"] = ees
+
         if self.connection_end == "server":
             if self.prcs:
                 self.prcs.tls13_derive_keys(cets)
@@ -768,6 +775,13 @@ def compute_tls13_handshake_secrets(self):
                                   b"".join(self.handshake_messages))
         self.tls13_derived_secrets["server_handshake_traffic_secret"] = shts
 
+        if self.nss_keys:
+            chts = self.nss_keys.get('CLIENT_HANDSHAKE_TRAFFIC_SECRET', {}).get(self.client_random, chts)
+            self.tls13_derived_secrets["client_handshake_traffic_secret"] = chts
+
+            shts = self.nss_keys.get('SERVER_HANDSHAKE_TRAFFIC_SECRET', {}).get(self.client_random, shts)
+            self.tls13_derived_secrets["server_handshake_traffic_secret"] = shts
+
     def compute_tls13_traffic_secrets(self):
         """
         Ciphers key and IV are updated accordingly for Application data.
@@ -801,6 +815,16 @@ def compute_tls13_traffic_secrets(self):
                                 b"".join(self.handshake_messages))
         self.tls13_derived_secrets["exporter_secret"] = es
 
+        if self.nss_keys:
+            cts0 = self.nss_keys.get('CLIENT_TRAFFIC_SECRET_0', {}).get(self.client_random, cts0)
+            self.tls13_derived_secrets["client_traffic_secrets"] = [cts0]
+
+            sts0 = self.nss_keys.get('SERVER_TRAFFIC_SECRET_0', {}).get(self.client_random, sts0)
+            self.tls13_derived_secrets["server_traffic_secrets"] = [sts0]
+
+            es = self.nss_keys.get('EXPORTER_SECRET', {}).get(self.client_random, es)
+            self.tls13_derived_secrets["exporter_secret"] = es
+
         if self.connection_end == "server":
             # self.prcs.tls13_derive_keys(cts0)
             self.pwcs.tls13_derive_keys(sts0)

test/scapy/layers/tls/tls.uts

@@ -1585,9 +1585,11 @@ bck_conf = conf
 conf.tls_session_enable = True
 conf.tls_nss_filename = scapy_path("doc/notebooks/tls/raw_data/tls_nss_example.keys.txt")
 
-packets = rdpcap(scapy_path("doc/notebooks/tls/raw_data/tls_nss_example.pcap"))
-assert b"GET /secret.txt HTTP/1.0\n" in packets[11].msg[0].data
-assert b"z2|gxarIKOxt,G1d>.Q2MzGY[k@" in packets[13].msg[0].data
+packets = sniff(offline=scapy_path("doc/notebooks/tls/raw_data/tls_nss_example.pcap"), session=TCPSession)
+assert b"GET /pki/srv_key.pem HTTP/1.0\r\n" in packets[9].msg[0].data
+assert b"BEGIN PRIVATE KEY" in packets[10].msg[0].data
+assert b"GET /pki/srv_key.pem HTTP/1.0\r\n" in packets[27].inner.msg[0].data
+assert b"BEGIN PRIVATE KEY" in packets[28].inner.msg[0].data
 
 conf = bck_conf
 
@@ -1602,9 +1604,11 @@ if shutil.which("editcap"):
     pcapng_path = get_temp_file()
     exit_status = os.system("editcap --inject-secrets tls,%s %s %s" % (key_log_path, pcap_path, pcapng_path))
     assert exit_status == 0
-    packets = rdpcap(pcapng_path)
-    assert b"GET /secret.txt HTTP/1.0\n" in packets[11].msg[0].data
-    assert b"z2|gxarIKOxt,G1d>.Q2MzGY[k@" in packets[13].msg[0].data
+    packets = sniff(offline=pcapng_path, session=TCPSession)
+    assert b"GET /pki/srv_key.pem HTTP/1.0\r\n" in packets[9].msg[0].data
+    assert b"BEGIN PRIVATE KEY" in packets[10].msg[0].data
+    assert b"GET /pki/srv_key.pem HTTP/1.0\r\n" in packets[27].inner.msg[0].data
+    assert b"BEGIN PRIVATE KEY" in packets[28].inner.msg[0].data
     conf = bck_conf
 
 = pcapng file with a non-UTF-8 Decryption Secrets Block