Disable aiohttp requoting of redirection URL (#5459)

* Fix requote redirect url * Add explanatory comment Co-authored-by: Quentin Lhoest <[email protected]> --------- Co-authored-by: Quentin Lhoest <[email protected]>
huggingface · Jan 31, 2023 · c4a4f96 · c4a4f96
1 parent b262d41
commit c4a4f96
Showing 1 changed file with 6 additions and 1 deletion.
diff --git a/src/datasets/filesystems/compression.py b/src/datasets/filesystems/compression.py
@@ -35,7 +35,12 @@ def __init__(
         super().__init__(self, **kwargs)
         # always open as "rb" since fsspec can then use the TextIOWrapper to make it work for "r" mode
         self.file = fsspec.open(
-            fo, mode="rb", protocol=target_protocol, compression=self.compression, **(target_options or {})
+            fo,
+            mode="rb",
+            protocol=target_protocol,
+            compression=self.compression,
+            client_kwargs={"requote_redirect_url": False},  # see https://github.com/huggingface/datasets/pull/5459
+            **(target_options or {}),
         )
         self.compressed_name = os.path.basename(self.file.path.split("::")[0])
         self.uncompressed_name = (