feat: add s3pypi delete command

CHANGELOG.md

@@ -6,15 +6,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/).
 
 
-## 2.0.0rc0 - 2024-01-XX
+## 2.0.0rc1 - 2024-01-XX
 
 ### Added
 
 - `s3pypi delete` command to delete packages from S3.
 
 ### Changed
 
-- Moved `s3pypi` command to `s3pypi upload`.
+- Moved default command to `s3pypi upload`.
 
 ### Removed
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "s3pypi"
-version = "2.0.0rc0"
+version = "2.0.0rc1"
 description = "CLI for creating a Python Package Repository in an S3 bucket"
 authors = [
     "Matteo De Wint <[email protected]>",

s3pypi/__init__.py

@@ -1,2 +1,2 @@
 __prog__ = "s3pypi"
-__version__ = "2.0.0rc0"
+__version__ = "2.0.0rc1"

s3pypi/__main__.py

@@ -21,9 +21,7 @@ def build_arg_parser() -> ArgumentParser:
     p.add_argument("-V", "--version", action="version", version=__version__)
     p.add_argument("-v", "--verbose", action="store_true", help="Verbose output.")
 
-    commands = p.add_subparsers(help="Commands")
-    commands.required = True
-    p.set_defaults(func=None)
+    commands = p.add_subparsers(help="Commands", required=True)
 
     def add_command(
         func: Callable[[core.Config, Namespace], None], help: str
@@ -41,6 +39,11 @@ def add_command(
         help="The distribution files to upload to S3. Usually `dist/*`.",
     )
     build_s3_args(up)
+    up.add_argument(
+        "--put-root-index",
+        action="store_true",
+        help="Write a root index that lists all available package names.",
+    )
     g = up.add_mutually_exclusive_group()
     g.add_argument(
         "--strict",
@@ -98,15 +101,16 @@ def build_s3_args(p: ArgumentParser) -> None:
             "This ensures that concurrent invocations of s3pypi do not overwrite each other's changes."
         ),
     )
-    p.add_argument(
-        "--put-root-index",
-        action="store_true",
-        help="Write a root index that lists all available package names.",
-    )
 
 
 def upload(cfg: core.Config, args: Namespace) -> None:
-    core.upload_packages(cfg, args.dist, strict=args.strict, force=args.force)
+    core.upload_packages(
+        cfg,
+        args.dist,
+        put_root_index=args.put_root_index,
+        strict=args.strict,
+        force=args.force,
+    )
 
 
 def delete(cfg: core.Config, args: Namespace) -> None:
@@ -127,7 +131,6 @@ def main(*raw_args: str) -> None:
             no_sign_request=args.no_sign_request,
         ),
         lock_indexes=args.lock_indexes,
-        put_root_index=args.put_root_index,
         profile=args.profile,
         region=args.region,
     )

s3pypi/core.py

@@ -6,15 +6,15 @@
 from itertools import groupby
 from operator import attrgetter
 from pathlib import Path
-from typing import List, Optional
+from typing import List, Optional, Tuple
 from zipfile import ZipFile
 
 import boto3
 
 from s3pypi import __prog__
 from s3pypi.exceptions import S3PyPiError
 from s3pypi.index import Hash
-from s3pypi.locking import DummyLocker, DynamoDBLocker
+from s3pypi.locking import DummyLocker, DynamoDBLocker, Locker
 from s3pypi.storage import S3Config, S3Storage
 
 log = logging.getLogger(__prog__)
@@ -26,7 +26,6 @@
 class Config:
     s3: S3Config
     lock_indexes: bool = False
-    put_root_index: bool = False
     profile: Optional[str] = None
     region: Optional[str] = None
 
@@ -42,18 +41,27 @@ def normalize_package_name(name: str) -> str:
     return re.sub(r"[-_.]+", "-", name.lower())
 
 
-def upload_packages(
-    cfg: Config, dist: List[Path], strict: bool = False, force: bool = False
-) -> None:
+def build_storage_and_locker(cfg: Config) -> Tuple[S3Storage, Locker]:
     session = boto3.Session(profile_name=cfg.profile, region_name=cfg.region)
     storage = S3Storage(session, cfg.s3)
     lock = (
         DynamoDBLocker(session, table=f"{cfg.s3.bucket}-locks")
         if cfg.lock_indexes
         else DummyLocker()
     )
+    return storage, lock
 
+
+def upload_packages(
+    cfg: Config,
+    dist: List[Path],
+    put_root_index: bool = False,
+    strict: bool = False,
+    force: bool = False,
+) -> None:
+    storage, lock = build_storage_and_locker(cfg)
     distributions = parse_distributions(dist)
+
     get_name = attrgetter("name")
     existing_files = []
 
@@ -76,7 +84,7 @@ def upload_packages(
 
             storage.put_index(directory, index)
 
-    if cfg.put_root_index:
+    if put_root_index:
         with lock(storage.root):
             index = storage.build_root_index()
             storage.put_index(storage.root, index)
@@ -135,4 +143,29 @@ def extract_wheel_metadata(path: Path) -> PackageMetadata:
 
 
 def delete_package(cfg: Config, name: str, version: str) -> None:
-    raise NotImplementedError("Deleting packages is not implemented")
+    storage, lock = build_storage_and_locker(cfg)
+    directory = normalize_package_name(name)
+
+    with lock(directory):
+        index = storage.get_index(directory)
+
+        filenames = [f for f in index.filenames if f.split("-", 2)[1] == version]
+        if not filenames:
+            raise S3PyPiError(f"Package not found: {name} {version}")
+
+        for filename in filenames:
+            log.info("Deleting %s", filename)
+            storage.delete(directory, filename)
+            del index.filenames[filename]
+
+        if not index.filenames:
+            storage.delete(directory, storage.index_name)
+        else:
+            storage.put_index(directory, index)
+
+    if not index.filenames:
+        with lock(storage.root):
+            index = storage.get_index(storage.root)
+            if directory in index.filenames:
+                del index.filenames[directory]
+                storage.put_index(storage.root, index)

s3pypi/storage.py

@@ -76,3 +76,6 @@ def put_distribution(self, directory: str, local_path: Path) -> None:
                 ContentType="application/x-gzip",
                 **self.cfg.put_kwargs,  # type: ignore
             )
+
+    def delete(self, directory: str, filename: str) -> None:
+        self._object(directory, filename).delete()

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.0.0rc0
+current_version = 2.0.0rc1
 commit = True
 message = chore: bump version to {new_version}
 

tests/integration/test_main.py

@@ -106,3 +106,31 @@ def get_index():
             "sha256", "c5a2633aecf5adc5ae49b868e12faf01f2199b914d4296399b52dec62cb70fb3"
         ),
     }
+
+
+def test_main_delete_package(chdir, data_dir, s3_bucket):
+    with chdir(data_dir):
+        s3pypi("upload", "dists/*", "--bucket", s3_bucket.name, "--put-root-index")
+        s3pypi("delete", "hello-world", "0.1.0", "--bucket", s3_bucket.name)
+
+    def read(key: str) -> bytes:
+        return s3_bucket.Object(key).get()["Body"].read()
+
+    root_index = read("index.html").decode()
+
+    def assert_pkg_exists(pkg: str, filename: str):
+        path = f"{pkg}/"
+        assert read(path + filename)
+        assert f">{filename}</a>" in read(path).decode()
+        assert f">{pkg}</a>" in root_index
+
+    for deleted_key in [
+        "hello-world/",
+        "hello-world/hello_world-0.1.0-py3-none-any.whl",
+    ]:
+        with pytest.raises(s3_bucket.meta.client.exceptions.NoSuchKey):
+            s3_bucket.Object(deleted_key).get()
+
+    assert ">hello-world</a>" not in root_index
+    assert_pkg_exists("foo", "foo-0.1.0.tar.gz")
+    assert_pkg_exists("xyz", "xyz-0.1.0.zip")