Move to Poetry for Python

.flake8

@@ -0,0 +1,8 @@
+[flake8]
+exclude =
+    .git,
+    __pycache__,
+    venv/
+# E203: Whitespace before ':'. Addresses a discrepancy with Black slice formatting.
+extend-ignore = E203
+max-line-length = 99

.gitignore

@@ -37,3 +37,6 @@ tools/tokenserver/loadtests/*.pem
 tools/tokenserver/loadtests/*.pub
 venv
 .vscode/settings.json
+
+# For poetry install
+.install.stamp

Makefile

@@ -10,9 +10,54 @@ PATH_TO_SYNC_SPANNER_KEYS = `pwd`/service-account.json
 # https://github.com/mozilla-services/server-syncstorage
 PATH_TO_GRPC_CERT = ../server-syncstorage/local/lib/python2.7/site-packages/grpc/_cython/_credentials/roots.pem
 
+POETRY := $(shell command -v poetry 2> /dev/null)
+INSTALL_STAMP := .install.stamp
+TOOLS_DIR := tools
+FLAKE8_CONFIG := .flake8
+PROJECT_ROOT_DIR := ./
+ROOT_PYPROJECT_TOML := pyproject.toml
+HAWK_DIR := $(TOOLS_DIR)/hawk
+INTEGRATION_TEST_DIR := $(TOOLS_DIR)/integration_tests
+INTEGRATION_TEST_DIR_TOKENSERVER := $(TOOLS_DIR)/integration_tests/tokenserver
+SPANNER_DIR := $(TOOLS_DIR)/spanner
+TOKENSERVER_UTIL_DIR := $(TOOLS_DIR)/tokenserver
+LOAD_TEST_DIR := $(TOOLS_DIR)/tokenserver/loadtests
+
 SRC_ROOT = $(shell pwd)
 PYTHON_SITE_PACKGES = $(shell $(SRC_ROOT)/venv/bin/python -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")
 
+.PHONY: install
+install: $(INSTALL_STAMP)  ##  Install dependencies with poetry
+$(INSTALL_STAMP): pyproject.toml poetry.lock
+	@if [ -z $(POETRY) ]; then echo "Poetry could not be found. See https://python-poetry.org/docs/"; exit 2; fi
+	$(POETRY) install
+	touch $(INSTALL_STAMP)
+
+hawk:
+	# install dependencies for hawk token utility.
+	$(POETRY) -V
+	$(POETRY) install --directory=$(HAWK_DIR) --no-root
+
+integration-test:
+	# install dependencies for integration tests.
+	$(POETRY) -V
+	$(POETRY) install --directory=$(INTEGRATION_TEST_DIR) --no-root
+
+spanner:
+	# install dependencies for spanner utilities.
+	$(POETRY) -V
+	$(POETRY) install --directory=$(SPANNER_DIR) --no-root
+
+tokenserver:
+	# install dependencies for tokenserver utilities.
+	$(POETRY) -V
+	$(POETRY) install --directory=$(TOKENSERVER_UTIL_DIR) --no-root
+
+tokenserver-load:
+	# install dependencies for tokenserver utilities.
+	$(POETRY) -V
+	$(POETRY) install --directory=$(LOAD_TEST_DIR) --no-root
+
 clippy_mysql:
 	# Matches what's run in circleci
 	cargo clippy --workspace --all-targets --no-default-features --features=syncstorage-db/mysql --features=py_verifier -- -D warnings
@@ -73,3 +118,32 @@ test:
 		SYNC_TOKENSERVER__DATABASE_URL=mysql://sample_user:sample_password@localhost/tokenserver_rs \
 		RUST_TEST_THREADS=1 \
 		cargo test --workspace
+
+.PHONY: isort
+isort: $(INSTALL_STAMP)  ##  Run isort
+	$(POETRY) run isort --check-only $(ROOT_PYPROJECT_TOML) $(PROJECT_ROOT_DIR)
+
+.PHONY: black
+black: $(INSTALL_STAMP)  ##  Run black
+	$(POETRY) run black --quiet --diff --check $(ROOT_PYPROJECT_TOML) $(PROJECT_ROOT_DIR)
+
+.PHONY: format
+format: $(INSTALL_STAMP)  ##  Sort imports and reformats code
+	$(POETRY) run isort $(ROOT_PYPROJECT_TOML) $(PROJECT_ROOT_DIR)
+	$(POETRY) run black $(ROOT_PYPROJECT_TOML) $(PROJECT_ROOT_DIR)
+
+.PHONY: flake8
+flake8: $(INSTALL_STAMP)  ##  Run flake8
+	$(POETRY) run flake8 --config $(FLAKE8_CONFIG) $(PROJECT_ROOT_DIR)
+
+.PHONY: bandit
+bandit: $(INSTALL_STAMP)  ##  Run bandit
+	$(POETRY) run bandit --quiet -r -c $(ROOT_PYPROJECT_TOML) $(PROJECT_ROOT_DIR)
+
+.PHONY: mypy
+mypy: $(INSTALL_STAMP)  ##  Run mypy
+	$(POETRY) run mypy --config-file=$(ROOT_PYPROJECT_TOML) $(PROJECT_ROOT_DIR)
+
+.PHONY: pydocstyle
+pydocstyle: $(INSTALL_STAMP)  ##  Run pydocstyle
+	$(POETRY) run pydocstyle -es --count --config=$(ROOT_PYPROJECT_TOML) $(PROJECT_ROOT_DIR)

pyproject.toml

@@ -0,0 +1,69 @@
+[tool.poetry]
+name = "syncstorage-rs"
+version = "0.1.0"
+description = "sync storage server in rust"
+authors = ["Mozilla Services Engineering <[email protected]>"]
+license = "Mozilla Public License Version 2.0"
+readme = "README.md"
+packages = [{include = "tools"}]
+
+[tool.poetry.dependencies]
+python = "^3.12"
+tokenlib = "2.0.0"
+pyfxa = "0.7.7"
+cryptography = "42.0.8"
+
+[tool.poetry.group.dev.dependencies]
+black = "^24.4.2"
+isort = "^5.13.2"
+flake8 = "^7.1.0"
+mypy = "^1.10.1"
+pydocstyle = "^6.3.0"
+bandit = "^1.7.9"
+
+[tool.black]
+line-length = 99
+
+[tool.isort]
+profile = "black"
+skip_gitignore = true
+
+[tool.bandit]
+# skips asserts
+# B101: https://bandit.readthedocs.io/en/latest/plugins/b101_assert_used.html#
+# B104: https://bandit.readthedocs.io/en/latest/plugins/b104_hardcoded_bind_all_interfaces.html
+# B311: https://bandit.readthedocs.io/en/latest/blacklists/blacklist_calls.html#b311-random 
+# B404: https://bandit.readthedocs.io/en/latest/blacklists/blacklist_imports.html#b404-import-subprocess
+skips = ["B101", "B104", "B311", "B404"]
+
+[tool.mypy]
+python_version = "3.12"
+disable_error_code = "attr-defined"
+disallow_untyped_calls = false
+follow_imports = "normal"
+ignore_missing_imports = true
+pretty = true
+show_error_codes = true
+strict_optional = true
+warn_no_return = true
+warn_redundant_casts = true
+warn_return_any = true
+warn_unused_ignores = true
+warn_unreachable = true
+
+[tool.pydocstyle]
+match = ".*\\.py"
+convention = "pep257"
+# Error Code Ref: https://www.pydocstyle.org/en/stable/error_codes.html
+# D212 Multi-line docstring summary should start at the first line
+add-select = ["D212"]
+# D105 Docstrings for magic methods
+# D107 Docstrings for __init__
+# D203 as it conflicts with D211 https://github.com/PyCQA/pydocstyle/issues/141
+# D205 1 blank line required between summary line and description, awkward spacing
+# D400 First line should end with a period, doesn't work when sentence spans 2 lines
+add-ignore = ["D105","D107","D203", "D205", "D400"]
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

tokenserver-auth/src/oauth/verify.py

@@ -1,8 +1,9 @@
-from fxa.oauth import Client
-from fxa.errors import ClientError, TrustError
 import json
 
-DEFAULT_OAUTH_SCOPE = 'https://identity.mozilla.com/apps/oldsync'
+from fxa.errors import ClientError, TrustError
+from fxa.oauth import Client
+
+DEFAULT_OAUTH_SCOPE = "https://identity.mozilla.com/apps/oldsync"
 
 
 class FxaOAuthClient:

tools/hawk/make_hawk_token.py

@@ -36,60 +36,44 @@
 # 10 years
 DURATION = timedelta(days=10 * 365).total_seconds()
 
-SALT = hexlify(os.urandom(3)).decode('ascii')
+SALT = hexlify(os.urandom(3)).decode("ascii")
 
 
 def get_args():
-    parser = argparse.ArgumentParser(
-        description="Create a hawk header for use in testing"
-        )
+    parser = argparse.ArgumentParser(description="Create a hawk header for use in testing")
     parser.add_argument(
-        '--uid', type=int, default=LEGACY_UID,
-        help="Legacy UID ({})".format(LEGACY_UID))
+        "--uid", type=int, default=LEGACY_UID, help="Legacy UID ({})".format(LEGACY_UID)
+    )
+    parser.add_argument("--uri", default=URI, help="URI path ({})".format(URI))
+    parser.add_argument("--method", default=METHOD, help="The HTTP Method ({})".format(METHOD))
+    parser.add_argument("--fxa_uid", default=FXA_UID, help="FxA User ID ({})".format(FXA_UID))
+    parser.add_argument("--fxa_kid", default=FXA_KID, help="FxA K ID ({})".format(FXA_KID))
     parser.add_argument(
-        '--uri', default=URI,
-        help="URI path ({})".format(URI))
+        "--device_id", default=DEVICE_ID, help="FxA Device ID ({})".format(DEVICE_ID)
+    )
+    parser.add_argument("--node", default=NODE, help="HTTP Host URI for node ({})".format(NODE))
     parser.add_argument(
-        '--method', default=METHOD,
-        help="The HTTP Method ({})".format(METHOD))
+        "--duration", type=int, default=DURATION, help="Hawk TTL ({})".format(DURATION)
+    )
+    parser.add_argument("--secret", default=SECRET, help="Shared HAWK secret ({})".format(SECRET))
+    parser.add_argument("--hmac_key", default=HMAC_KEY, help="HAWK HMAC key ({})".format(HMAC_KEY))
     parser.add_argument(
-        '--fxa_uid', default=FXA_UID,
-        help="FxA User ID ({})".format(FXA_UID))
-    parser.add_argument(
-        '--fxa_kid', default=FXA_KID,
-        help="FxA K ID ({})".format(FXA_KID))
-    parser.add_argument(
-        '--device_id', default=DEVICE_ID,
-        help="FxA Device ID ({})".format(DEVICE_ID))
-    parser.add_argument(
-        '--node', default=NODE,
-        help="HTTP Host URI for node ({})".format(NODE))
-    parser.add_argument(
-        '--duration', type=int, default=DURATION,
-        help="Hawk TTL ({})".format(DURATION))
-    parser.add_argument(
-        '--secret', default=SECRET,
-        help="Shared HAWK secret ({})".format(SECRET))
-    parser.add_argument(
-        '--hmac_key', default=HMAC_KEY,
-        help="HAWK HMAC key ({})".format(HMAC_KEY))
-    parser.add_argument(
-        '--as_header', action="store_true", default=False,
-        help="return only header (False)")
+        "--as_header", action="store_true", default=False, help="return only header (False)"
+    )
     return parser.parse_args()
 
 
 def create_token(args):
     expires = int(time.time()) + args.duration
     token_data = {
-        'uid': args.uid,
-        'node': args.node,
-        'expires': expires,
-        'fxa_uid': args.fxa_uid,
-        'fxa_kid': args.fxa_kid,
-        'hashed_fxa_uid': metrics_hash(args, args.fxa_uid),
-        'hashed_device_id': metrics_hash(args, args.device_id),
-        'salt': SALT,
+        "uid": args.uid,
+        "node": args.node,
+        "expires": expires,
+        "fxa_uid": args.fxa_uid,
+        "fxa_kid": args.fxa_kid,
+        "hashed_fxa_uid": metrics_hash(args, args.fxa_uid),
+        "hashed_device_id": metrics_hash(args, args.device_id),
+        "salt": SALT,
     }
     token = tokenlib.make_token(token_data, secret=args.secret)
     key = tokenlib.get_derived_secret(token, secret=args.secret)
@@ -99,18 +83,16 @@ def create_token(args):
 def metrics_hash(args, value):
     if isinstance(args.hmac_key, str):
         args.hmac_key = args.hmac_key.encode()
-    hasher = hmac.new(args.hmac_key, b'', sha256)
+    hasher = hmac.new(args.hmac_key, b"", sha256)
     # value may be an email address, in which case we only want the first part
-    hasher.update(value.encode('utf-8').split(b"@", 1)[0])
+    hasher.update(value.encode("utf-8").split(b"@", 1)[0])
     return hasher.hexdigest()
 
 
 def main():
     args = get_args()
     token, key, expires, salt = create_token(args)
-    path = "{node}{uri}".format(
-            node=args.node,
-            uri=args.uri)
+    path = "{node}{uri}".format(node=args.node, uri=args.uri)
     req = Request.blank(path)
     req.method = args.method
     header = hawkauthlib.sign_request(req, token, key)
@@ -123,5 +105,5 @@ def main():
         print("Authorization:", header)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

tools/hawk/pyproject.toml

@@ -0,0 +1,18 @@
+[tool.poetry]
+name = "hawk"
+version = "0.1.0"
+description = "Hawk Auth Header Generation Tool"
+authors = ["Mozilla Services Engineering <[email protected]>"]
+license = "Mozilla Public License Version 2.0"
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.12"
+hawkauthlib = "^2.0.0"
+tokenlib = "^2.0.0"
+WebOb = "^1.8.7"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"