fix: use always utf-8 encoding when opening files(jina-ai#5821)

Author: florian-hoenicke

docs/conf.py

@@ -18,7 +18,7 @@
     if 'JINA_VERSION' not in os.environ:
         pkg_name = 'jina'
         libinfo_py = path.join(repo_dir, pkg_name, '__init__.py')
-        libinfo_content = open(libinfo_py, 'r').readlines()
+        libinfo_content = open(libinfo_py, 'r', encoding='utf-8').readlines()
         version_line = [
             l.strip() for l in libinfo_content if l.startswith('__version__')
         ][0]

docs/pin_requirements.py

@@ -2,7 +2,7 @@
 import sys
 
 file_name = sys.argv[1]
-with open(file_name, 'r') as f:
+with open(file_name, 'r', encoding='utf-8') as f:
     input = f.read()
 
 
@@ -16,5 +16,5 @@
 )
 
 
-with open(file_name, 'w') as f:
+with open(file_name, 'w', encoding='utf-8') as f:
     f.write(output)

fastentrypoints.py

@@ -98,7 +98,7 @@ def main():
         setup_path = os.path.join(dst, 'setup.py')
 
         # Insert the include statement to MANIFEST.in if not present
-        with open(manifest_path, 'a+') as manifest:
+        with open(manifest_path, 'a+', encoding='utf-8') as manifest:
             manifest.seek(0)
             manifest_content = manifest.read()
             if 'include fastentrypoints.py' not in manifest_content:
@@ -107,7 +107,7 @@ def main():
                 )
 
         # Insert the import statement to setup.py if not present
-        with open(setup_path, 'a+') as setup:
+        with open(setup_path, 'a+', encoding='utf-8') as setup:
             setup.seek(0)
             setup_content = setup.read()
             if 'import fastentrypoints' not in setup_content:

jina/exporter.py

@@ -63,22 +63,22 @@ def export_schema(args):
         dump_api = api_to_dict()
         for yp in args.yaml_path:
             f_name = (yp % __version__) if '%s' in yp else yp
-            with open(f_name, 'w', encoding='utf8') as fp:
+            with open(f_name, 'w', encoding='utf-8') as fp:
                 JAML.dump(dump_api, fp)
             default_logger.info(f'API is exported to {f_name}')
 
     if args.json_path:
         dump_api = api_to_dict()
         for jp in args.json_path:
             f_name = (jp % __version__) if '%s' in jp else jp
-            with open(f_name, 'w', encoding='utf8') as fp:
+            with open(f_name, 'w', encoding='utf-8') as fp:
                 json.dump(dump_api, fp, sort_keys=True)
             default_logger.info(f'API is exported to {f_name}')
 
     if args.schema_path:
         dump_api = get_full_schema()
         for jp in args.schema_path:
             f_name = (jp % __version__) if '%s' in jp else jp
-            with open(f_name, 'w', encoding='utf8') as fp:
+            with open(f_name, 'w', encoding='utf-8') as fp:
                 json.dump(dump_api, fp, sort_keys=True)
             default_logger.info(f'API is exported to {f_name}')

jina/helper.py

@@ -1482,7 +1482,7 @@ async def root():
 def get_ci_vendor() -> Optional[str]:
     from jina.constants import __resources_path__
 
-    with open(os.path.join(__resources_path__, 'ci-vendors.json')) as fp:
+    with open(os.path.join(__resources_path__, 'ci-vendors.json'), encoding='utf-8') as fp:
         all_cis = json.load(fp)
         for c in all_cis:
             if isinstance(c['env'], str) and c['env'] in os.environ:

jina/importer.py

@@ -43,7 +43,7 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_val, traceback):
         if exc_type == ModuleNotFoundError:
             missing_module = self._pkg_name or exc_val.name
-            with open(os.path.join(__resources_path__, 'extra-requirements.txt')) as fp:
+            with open(os.path.join(__resources_path__, 'extra-requirements.txt'), encoding='utf-8') as fp:
                 for v in fp:
                     if (
                         v.strip()

jina/jaml/__init__.py

@@ -598,7 +598,7 @@ def save_config(self, filename: Optional[str] = None):
             warnings.warn(
                 f'no "filename" is given, {self!r}\'s config will be saved to: {f}'
             )
-        with open(f, 'w', encoding='utf8') as fp:
+        with open(f, 'w', encoding='utf-8') as fp:
             JAML.dump(self, fp)
 
     @classmethod

jina/jaml/helper.py

@@ -169,7 +169,7 @@ def parse_config_source(
         return path, None
     elif allow_yaml_file and is_yaml_filepath(path):
         comp_path = complete_path(path, extra_search_paths)
-        return open(comp_path, encoding='utf8'), comp_path
+        return open(comp_path, encoding='utf-8'), comp_path
     elif allow_url and urllib.parse.urlparse(path).scheme in {'http', 'https'}:
         req = urllib.request.Request(path, headers={'User-Agent': 'Mozilla/5.0'})
         with urllib.request.urlopen(req) as fp:

jina/logging/logger.py

@@ -199,7 +199,7 @@ def add_handlers(self, config_path: Optional[str] = None, **kwargs):
             if not os.path.exists(config_path):
                 config_path = old_config_path
 
-        with open(config_path) as fp:
+        with open(config_path, encoding='utf-8') as fp:
             config = JAML.load(fp)
 
         for h in config['handlers']:

jina/orchestrate/deployments/__init__.py

@@ -1651,7 +1651,7 @@ def to_docker_compose_yaml(
                 services[service_name] = service
 
         docker_compose_dict['services'] = services
-        with open(output_path, 'w+') as fp:
+        with open(output_path, 'w+', encoding='utf-8') as fp:
             yaml.dump(docker_compose_dict, fp, sort_keys=False)
 
         command = (
@@ -1705,7 +1705,7 @@ def _to_kubernetes_yaml(
         for name, k8s_objects in configs:
             filename = os.path.join(output_base_path, f'{name}.yml')
             os.makedirs(output_base_path, exist_ok=True)
-            with open(filename, 'w+') as fp:
+            with open(filename, 'w+', encoding='utf-8') as fp:
                 for i, k8s_object in enumerate(k8s_objects):
                     yaml.dump(k8s_object, fp)
                     if i < len(k8s_objects) - 1:

jina/orchestrate/deployments/config/k8slib/kubernetes_tools.py

@@ -42,7 +42,7 @@ def _get_yaml(template: str, params: Dict) -> Dict:
 
     path = os.path.join(DEFAULT_RESOURCE_DIR, f'{template}.yml')
 
-    with open(path) as f:
+    with open(path, encoding='utf-8') as f:
         content = f.read()
         for k, v in params.items():
             content = content.replace(f'{{{k}}}', str(v))
@@ -56,7 +56,7 @@ def _get_configmap_yaml(template: str, params: Dict):
 
     path = os.path.join(DEFAULT_RESOURCE_DIR, f'{template}.yml')
 
-    with open(path) as f:
+    with open(path, encoding='utf-8') as f:
         config_map = yaml.safe_load(f)
 
     config_map['metadata']['name'] = params.get('name') + '-' + 'configmap'

jina/orchestrate/flow/base.py

@@ -2824,7 +2824,7 @@ def to_docker_compose_yaml(
                 services[service_name] = service
 
         docker_compose_dict['services'] = services
-        with open(output_path, 'w+') as fp:
+        with open(output_path, 'w+', encoding='utf-8') as fp:
             yaml.dump(docker_compose_dict, fp, sort_keys=False)
 
         command = (

jina/orchestrate/pods/container_helper.py

@@ -24,7 +24,7 @@ def get_docker_network(client) -> Optional[str]:
     except docker.errors.NotFound:
         try:
             # https://stackoverflow.com/a/52988227/15683245
-            with open('/proc/1/cpuset') as f:
+            with open('/proc/1/cpuset', encoding='utf-8') as f:
                 hostname = os.path.basename(f.read().rstrip())
             container = client.containers.get(hostname)
         except Exception:

jina/serve/networking/utils.py

@@ -285,7 +285,7 @@ def in_docker():
     if os.path.exists('/.dockerenv'):
         return True
     if os.path.isfile(path):
-        with open(path) as file:
+        with open(path, encoding='utf-8') as file:
             return any('docker' in line for line in file)
     return False
 

jina_cli/__init__.py

@@ -37,7 +37,7 @@ def _get_run_args(print_args: bool = True):
                 if isinstance(a, (_StoreAction, _StoreTrueAction))
             }
 
-            with open(os.path.join(__resources_path__, 'jina.logo')) as fp:
+            with open(os.path.join(__resources_path__, 'jina.logo'), encoding='utf-8') as fp:
                 logo_str = fp.read()
 
             param_str = Table(

scripts/announcement.py

@@ -8,7 +8,7 @@
 
 def rm_announce():
     # remove all announcement
-    with open(readme_md) as fp:
+    with open(readme_md, encoding='utf-8') as fp:
         _old = fp.read()
         _new = re.sub(
             r'(<!--startmsg-->\s*?\n).*(\n\s*?<!--endmsg-->)',
@@ -17,18 +17,18 @@ def rm_announce():
             flags=re.DOTALL,
         )
 
-    with open(readme_md, 'w') as fp:
+    with open(readme_md, 'w', encoding='utf-8') as fp:
         fp.write(_new)
 
-    with open(conf_py) as fp:
+    with open(conf_py, encoding='utf-8') as fp:
         _old = fp.read()
         _new = re.sub(
             r'(# start-announce\s*?\n).*(\n\s*?# end-announce)',
             rf'\g<1>\g<2>',
             _old,
             flags=re.DOTALL,
         )
-    with open(conf_py, 'w') as fp:
+    with open(conf_py, 'w', encoding='utf-8') as fp:
         fp.write(_new)
 
 
@@ -49,7 +49,7 @@ def rm_announce():
         meetup_svg_url = f'<a href="{url}"><img src="https://github.com/jina-ai/jina/blob/master/{meetup_svg}?raw=true"></a>'
 
         # update meetup_svg
-        with open(meetup_svg) as fp:
+        with open(meetup_svg, encoding='utf-8') as fp:
             _old = fp.read()
             _new = re.sub(r'(<a href=").*(")', rf'\g<1>{url}\g<2>', _old)
             _new = re.sub(
@@ -59,11 +59,11 @@ def rm_announce():
                 flags=re.DOTALL,
             )
 
-        with open(meetup_svg, 'w') as fp:
+        with open(meetup_svg, 'w', encoding='utf-8') as fp:
             fp.write(_new)
 
         # update readme_md
-        with open(readme_md) as fp:
+        with open(readme_md, encoding='utf-8') as fp:
             _old = fp.read()
             _new = re.sub(
                 r'(<!--startmsg-->\s*?\n).*(\n\s*?<!--endmsg-->)',
@@ -72,11 +72,11 @@ def rm_announce():
                 flags=re.DOTALL,
             )
 
-        with open(readme_md, 'w') as fp:
+        with open(readme_md, 'w', encoding='utf-8') as fp:
             fp.write(_new)
 
         # update conf
-        with open(conf_py) as fp:
+        with open(conf_py, encoding='utf-8') as fp:
             _old = fp.read()
             _new = re.sub(
                 r'(# start-announce\s*?\n).*(\n\s*?# end-announce)',
@@ -85,5 +85,5 @@ def rm_announce():
                 flags=re.DOTALL,
             )
 
-        with open(conf_py, 'w') as fp:
+        with open(conf_py, 'w', encoding='utf-8') as fp:
             fp.write(_new)

scripts/create-conda-recipe.py

@@ -9,7 +9,7 @@
 
 def get_extra_requires(path):
     try:
-        with open(path) as fp:
+        with open(path, encoding='utf-8') as fp:
             extra_deps = defaultdict(set)
             for k in fp:
                 if k.strip() and not k.startswith('#'):
@@ -211,5 +211,5 @@ def increase_indent(self, flow=False, *args, **kwargs):
 '''
 
 recipe = recipe_header + recipe
-with open('conda/meta.yaml', 'w+') as fp:
+with open('conda/meta.yaml', 'w+', encoding='utf-8') as fp:
     fp.write(recipe)

scripts/generate-list-args.py

@@ -15,5 +15,5 @@
         type = None if v['type'] == 'null' else v['type']
         table.append(f'| `{k}` | {desc} | `{type}` | `{v["default"]}` |')
 
-    with open(f'../docs/concepts/flow/{s}-args.md', 'w') as fp:
+    with open(f'../docs/concepts/flow/{s}-args.md', 'w', encoding='utf-8') as fp:
         fp.write('\n'.join(table))

scripts/get-last-release-note.py

@@ -2,12 +2,12 @@
 # python scripts/get-last-release-note.py
 ## result in root/tmp.md
 
-with open('CHANGELOG.md') as fp:
+with open('CHANGELOG.md', encoding='utf-8') as fp:
     n = []
     for v in fp:
         if v.startswith('## Release Note'):
             n.clear()
         n.append(v)
 
-with open('tmp.md', 'w') as fp:
+with open('tmp.md', 'w', encoding='utf-8') as fp:
     fp.writelines(n)

scripts/get-openapi-schemas.py

@@ -47,5 +47,5 @@
 gateway_schema['servers'].append(
     {'url': f'http://localhost:{args.port}', 'description': 'Local Jina gateway'}
 )
-with open(GATEWAY_SCHEMA_FILENAME, 'w') as f:
+with open(GATEWAY_SCHEMA_FILENAME, 'w', encoding='utf-8') as f:
     json.dump(gateway_schema, f)

scripts/inject-cli-as-overload.py

@@ -123,7 +123,7 @@ def fill_overload(
             re.DOTALL,
         )
 
-    with open(filepath, 'w') as fp:
+    with open(filepath, 'w', encoding='utf-8') as fp:
         fp.write(final_code)
     return {regex_tag or cli_entrypoint: doc_str}
 
@@ -229,7 +229,7 @@ def fill_implementation_stub(
             re.DOTALL,
         )
 
-    with open(filepath, 'w') as fp:
+    with open(filepath, 'w', encoding='utf-8') as fp:
         fp.write(final_code)
 
 

scripts/inject-document-props-as-overload.py

@@ -76,7 +76,7 @@ def write_signature(
         0,
         re.DOTALL,
     )
-    with open(filepath, 'w') as fp:
+    with open(filepath, 'w', encoding='utf-8') as fp:
         fp.write(final_code)
 
 

scripts/prepend_version_json.py

@@ -11,11 +11,11 @@
 )
 args = parser.parse_args()
 
-with open("./docs/_versions.json") as f:
+with open("./docs/_versions.json", encoding='utf-8') as f:
     versions: List[dict] = json.load(f)
     element = {k: v for k, v in args._get_kwargs()}
     if element != versions[0]:
         versions.insert(0, element)
 
-with open("./docs/_versions.json", "w") as f:
+with open("./docs/_versions.json", "w", encoding='utf-8') as f:
     json.dump(versions, f)

scripts/update-autocomplete-cli.py

@@ -24,7 +24,7 @@ def _gaa(key, parser):
     cmd = compl.pop('')
     compl = {'commands': cmd, 'completions': compl}
 
-    with open(ac_file, 'w') as fp:
+    with open(ac_file, 'w', encoding='utf-8') as fp:
         fp.write(f'ac_table = {compl}\n')
 
 

scripts/update-fluent-interface.py

@@ -42,7 +42,7 @@
 doc_md = '../docs/fundamentals/document/fluent-interface.md'
 text = '\n'.join(all_s)
 
-with open(doc_md) as fp:
+with open(doc_md, encoding='utf-8') as fp:
     _old = fp.read()
     _new = re.sub(
         r'(<!-- fluent-interface-start -->\s*?\n).*(\n\s*?<!-- fluent-interface-end -->)',
@@ -51,5 +51,5 @@
         flags=re.DOTALL,
     )
 
-with open(doc_md, 'w') as fp:
+with open(doc_md, 'w', encoding='utf-8') as fp:
     fp.write(_new)