Skip to content

Commit

Permalink
feat: Allow IssuesStreams to be filtered with JQL [MFLP-7] (#100)
Browse files Browse the repository at this point in the history 
* fix: Change date range to properly use the end_date param

* Quote the start_date / end_date params for future handling of date parsing

* Implement new issues_jql param, allows arbitrary filtering

* Update README with new setting param

* Simplify jql expression creation

* Add available env vars to pytest report header

* Nest issues JQL setting

---------

Co-authored-by: Edgar Ramírez Mondragón <[email protected]>
  • Loading branch information
@jeffgeorge @edgarrmondragon
jeffgeorge and edgarrmondragon authored Dec 15, 2024
1 parent 1f83650 commit 7a46e3e
Show file tree
Hide file tree
Showing 5 changed files with 91 additions and 28 deletions.
23 changes: 13 additions & 10 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,19 @@ Built with the [Meltano Singer SDK](https://sdk.meltano.com).

## Settings

| Setting | Required | Default | Description |
| :-------------------------------- | :------- | :------ | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| start_date | False | None | Earliest record date to sync |
| end_date | False | None | Latest record date to sync |
| domain | True | None | The Domain for your Jira account, e.g. meltano.atlassian.net |
| api_token | True | None | Jira API Token. |
| email | True | None | The user email for your Jira account. |
| page_size | False | None | |
| page_size.issues | False | 100 | Page size for issues stream |
| include_audit_logs | False | False | Include the audit logs stream |
| Setting | Required | Default | Description |
| :------------------------ | :------- | :------ | :----------------------------------------------------------- |
| start_date | False | None | Earliest record date to sync |
| end_date | False | None | Latest record date to sync |
| domain | True | None | The Domain for your Jira account, e.g. meltano.atlassian.net |
| api_token | True | None | Jira API Token. |
| email | True | None | The user email for your Jira account. |
| page_size | False | None | |
| page_size.issues | False | 100 | Page size for issues stream |
| stream_options | False | None | Options for individual streams |
| stream_options.issues | False | None | Options specific to the issues stream |
| stream_options.issues.jql | False | None | A JQL query to filter issues |
| include_audit_logs | False | False | Include the audit logs stream |

### Built-in capabilities

Expand Down
37 changes: 27 additions & 10 deletions meltano.yml
View file
Original file line number Diff line number Diff line change
@@ -1,30 +1,47 @@
version: 1
send_anonymous_usage_stats: true
project_id: "tap-jira"
project_id: tap-jira
default_environment: dev
plugins:
extractors:
- name: "tap-jira"
namespace: "tap_jira"
- name: tap-jira
namespace: tap_jira
pip_url: -e .
capabilities:
- state
- catalog
- discover
- about
- stream-maps
settings_group_validation:
- [domain, api_token, email]
settings:
- name: start_date
kind: date_iso8601
description: Earliest record date to sync
- name: end_date
kind: date_iso8601
description: Latest record date to sync
- name: domain
- name: auth_type
- name: auth.flow
- name: auth.access_token
kind: password
- name: auth.username
- name: auth.password
kind: password
kind: string
description: The Domain for your Jira account, e.g. meltano.atlassian.net
- name: api_token
kind: string
description: Jira API Token
sensitive: true
- name: email
kind: string
description: The user email for your Jira account
- name: page_size.issues
kind: integer
value: 100
description: Page size for issues stream
- name: stream_options.issues.jql
kind: string
description: A JQL query to filter issues
environments:
- name: dev
- name: staging
- name: prod
venv:
backend: uv
19 changes: 11 additions & 8 deletions tap_jira/streams.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1661,7 +1661,7 @@ def get_url_params(

params["maxResults"] = self.config.get("page_size", {}).get("issues", 10)

params["jql"] = [] # init a query param
jql: list[str] = []

if next_page_token:
params["startAt"] = next_page_token
Expand All @@ -1672,18 +1672,21 @@ def get_url_params(

if "start_date" in self.config:
start_date = self.config["start_date"]
params["jql"].append(f"(created>={start_date} or updated>={start_date})")
jql.append(f"(created>='{start_date}' or updated>='{start_date}')")

if "end_date" in self.config:
end_date = self.config["end_date"]
params["jql"].append(f"(created<{end_date} or updated<{start_date})")
jql.append(f"(created<'{end_date}' or updated<'{end_date}')")

if params["jql"]:
jql = " and ".join(params["jql"])
params["jql"] = jql
if (
base_jql := self.config.get("stream_options", {})
.get("issues", {})
.get("jql")
):
jql.append(f"({base_jql})")

else:
params.pop("jql") # drop if there's no query
if jql:
params["jql"] = " and ".join(jql)

return params

Expand Down
21 changes: 21 additions & 0 deletions tap_jira/tap.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ class TapJira(Tap):
th.StringType,
description="Jira API Token.",
required=True,
secret=True,
title="API Token",
),
th.Property(
"email",
Expand All @@ -53,6 +55,25 @@ class TapJira(Tap):
),
),
),
th.Property(
"stream_options",
th.ObjectType(
th.Property(
"issues",
th.ObjectType(
th.Property(
"jql",
th.StringType,
description="A JQL query to filter issues",
title="JQL Query",
),
),
title="Issues Stream Options",
description="Options specific to the issues stream",
),
),
description="Options for individual streams",
),
th.Property(
"include_audit_logs",
th.BooleanType,
Expand Down
19 changes: 19 additions & 0 deletions tests/conftest.py
View file
Original file line number Diff line number Diff line change
@@ -1 +1,20 @@
"""Test Configuration."""

from __future__ import annotations

import os
import typing as t

if t.TYPE_CHECKING:
import pathlib

import pytest


def pytest_report_header(
config: pytest.Config, # noqa: ARG001
start_path: pathlib.Path, # noqa: ARG001
) -> list[str] | str:
"""Add a header to the test report."""
tap_jira_vars = [var for var in os.environ if var.startswith("TAP_JIRA")]
return f"tap-jira environment variables: {tap_jira_vars}"

0 comments on commit 7a46e3e

Please sign in to comment.