diff --git a/README.md b/README.md index 325434b9..697285a9 100644 --- a/README.md +++ b/README.md @@ -46,8 +46,11 @@ This tap accepts the following configuration options: - `metrics_log_level` - `stream_maps` - `stream_maps_config` - - `rate_limit_buffer` - A buffer to avoid consuming all query points for the auth_token at hand. Defaults to 1000. - - `expiry_time_buffer` - A buffer used when determining when to refresh Github app tokens. Only relevant when authenticating as a GitHub app. Defaults to 10 minutes. Tokens generated by GitHub apps expire 1 hour after creation, and will be refreshed once fewer than `expiry_time_buffer` minutes remain until the anticipated expiry time. + - `stream_options`: Options which can change the behaviour of a specific stream are nested within. + - `milestones`: Valid options for the `milestones` stream are nested within. + - `state`: Determines which milestones will be extracted. One of `open` (default), `closed`, `all`. + - `rate_limit_buffer`: A buffer to avoid consuming all query points for the auth_token at hand. Defaults to 1000. + - `expiry_time_buffer`: A buffer used when determining when to refresh GitHub app tokens. Only relevant when authenticating as a GitHub app. Defaults to 10 minutes. Tokens generated by GitHub apps expire 1 hour after creation, and will be refreshed once fewer than `expiry_time_buffer` minutes remain until the anticipated expiry time. Note that modes 1-3 are `repository` modes and 4-5 are `user` modes and will not run the same set of streams. diff --git a/meltano.yml b/meltano.yml index 8742c8a1..d69e8821 100644 --- a/meltano.yml +++ b/meltano.yml @@ -35,6 +35,15 @@ plugins: kind: array - name: user_ids kind: array + - name: stream_options.milestones.state + kind: options + options: + - label: Open + value: open + - label: Closed + value: closed + - label: All + value: all - name: start_date kind: date_iso8601 value: '2010-01-01T00:00:00Z' diff --git a/tap_github/repository_streams.py b/tap_github/repository_streams.py index c1ba9bc2..dcb486ca 100644 --- a/tap_github/repository_streams.py +++ b/tap_github/repository_streams.py @@ -635,6 +635,19 @@ class MilestonesStream(GitHubRestStream): state_partitioning_keys: ClassVar[list[str]] = ["repo", "org"] ignore_parent_replication_key = True + def get_url_params( + self, context: dict | None, next_page_token: Any | None + ) -> dict[str, Any]: + """Return a dictionary of values to be used in URL parameterization.""" + assert context is not None, f"Context cannot be empty for '{self.name}' stream." + params = super().get_url_params(context, next_page_token) + params["state"] = "open" + + if "milestones" in self.config.get("stream_options", {}): + params.update(self.config["stream_options"]["milestones"]) + + return params + schema = th.PropertiesList( # Parent Keys th.Property("repo", th.StringType), diff --git a/tap_github/tap.py b/tap_github/tap.py index 3ab8e40e..311bc244 100644 --- a/tap_github/tap.py +++ b/tap_github/tap.py @@ -94,6 +94,30 @@ def logger(cls) -> logging.Logger: "streams (such as repositories) if it is not selected but children are" ), ), + th.Property( + "stream_options", + th.ObjectType( + th.Property( + "milestones", + th.ObjectType( + th.Property( + "state", + th.StringType, + description=( + "Configures which states are of interest. " + "Must be one of [open, closed, all], defaults to open." + ), + default="open", + allowed_values=["open", "closed", "all"], + ), + additional_properties=False, + ), + description="Options specific to the 'milestones' stream.", + ), + additional_properties=False, + ), + description="Options which change the behaviour of a specific stream.", + ), ).to_dict() def discover_streams(self) -> list[Stream]: