Skip to content

Commit

Permalink
merged develop
Browse files Browse the repository at this point in the history
  • Loading branch information
grallewellyn committed Oct 26, 2024
2 parents d294363 + c9eb86a commit f1ad56b
Show file tree
Hide file tree
Showing 9 changed files with 643 additions and 347 deletions.
19 changes: 11 additions & 8 deletions .github/workflows/build-pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ name: Build
on:
# Triggers the workflow on push events
push:
branches: [ develop, release/**, main, feature/**, issue/**, issues/**, dependabot/** ]
branches: [ develop, release/**, main, master, feature/**, issue/**, issues/**, dependabot/** ]
tags-ignore:
- '*'
# Do not trigger build if pyproject.toml was the only thing changed
Expand Down Expand Up @@ -75,6 +75,7 @@ jobs:
if: |
github.ref != 'refs/heads/develop' &&
github.ref != 'refs/heads/main' &&
github.ref != 'refs/heads/master' &&
!startsWith(github.ref, 'refs/heads/release/')
run: |
new_ver="${{ steps.get-version.outputs.current_version }}+$(git rev-parse --short ${GITHUB_SHA})"
Expand Down Expand Up @@ -105,7 +106,7 @@ jobs:
- name: Release version
# If triggered by push to the main branch
if: |
startsWith(github.ref, 'refs/heads/main') &&
(startsWith(github.ref, 'refs/heads/main') || startsWith(github.ref, 'refs/heads/master')) &&
github.event_name != 'workflow_dispatch'
id: release
env:
Expand Down Expand Up @@ -179,8 +180,8 @@ jobs:
run: |
git config user.name "${GITHUB_ACTOR}"
git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
git tag -a "${{ env.software_version }}" -m "Version ${{ env.software_version }}"
git push origin "${{ env.software_version }}"
git tag -a "v${{ env.software_version }}" -m "Version ${{ env.software_version }}"
git push origin "v${{ env.software_version }}"
- name: Create GH release
if: |
steps.alpha.conclusion == 'success' ||
Expand All @@ -189,9 +190,9 @@ jobs:
uses: ncipollo/release-action@v1
with:
generateReleaseNotes: true
name: ${{ env.software_version }}
name: v${{ env.software_version }}
prerelease: ${{ steps.alpha.conclusion == 'success' || steps.rc.conclusion == 'success'}}
tag: ${{ env.software_version }}
tag: v${{ env.software_version }}

publish-pypi:
needs: [ build ]
Expand All @@ -202,7 +203,8 @@ jobs:
if: |
github.ref == 'refs/heads/develop' ||
startsWith(github.ref, 'refs/heads/release') ||
github.ref == 'refs/heads/main'
github.ref == 'refs/heads/main' ||
github.ref == 'refs/heads/master'
steps:
- name: Download python dist
uses: actions/download-artifact@v4
Expand All @@ -219,6 +221,7 @@ jobs:
repository-url: https://test.pypi.org/legacy/
- name: Publish to pypi.org
if: |
github.ref == 'refs/heads/main'
github.ref == 'refs/heads/main' ||
github.ref == 'refs/heads/master'
id: pypi-publish
uses: pypa/gh-action-pypi-publish@release/v1
8 changes: 6 additions & 2 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]
### Added
- [issues/95](https://github.com/MAAP-Project/maap-py/issues/95): Added github action workflow for publishing
### Changed
### Deprecated
### Removed
### Fixed
### Security

## [4.0.1]
### Added
- [issues/95](https://github.com/MAAP-Project/maap-py/issues/95): Added github action workflow for publishing and switched to poetry for builds

## [4.0.0]
### Added
- Started tracking changelog

[Unreleased]: https://github.com/MAAP-Project/maap-py/compare/v4.0.0...HEAD
[Unreleased]: https://github.com/MAAP-Project/maap-py/compare/v4.0.1...develop
[4.0.1]: https://github.com/MAAP-Project/maap-py/compare/v4.0.0...v4.0.1
[4.0.0]: https://github.com/MAAP-Project/maap-py/compare/1cd11b6e05781d757b8bad7e6e899855ce3e3682...v4.0.0
131 changes: 131 additions & 0 deletions maap/Secrets.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
import requests
import logging
import json
from maap.utils import endpoints
from maap.utils import requests_utils
from maap.utils import endpoints

logger = logging.getLogger(__name__)

class Secrets:
"""
Functions used for member secrets API interfacing
"""
def __init__(self, member_endpoint, api_header):
self._api_header = api_header
self._members_endpoint = f"{member_endpoint}/{endpoints.MEMBERS_SECRETS}"


def get_secrets(self):
"""
Returns a list of secrets for a given user.
Returns:
list: Returns a list of dicts containing secret names e.g. [{'secret_name': 'secret1'}, {'secret_name': 'secret2'}].
"""
try:
response = requests.get(
url = self._members_endpoint,
headers=self._api_header
)
logger.debug(f"Response from get_secrets request: {response.text}")
return json.loads(response.text)
except Exception as e:
raise(f"Error retrieving secrets: {e}")


def get_secret(self, secret_name):
"""
Returns secret value for provided secret name.
Args:
secret_name (str, required): Secret name.
Returns:
string: Secret value.
Raises:
ValueError: If secret name is not provided.
"""
if secret_name is None:
raise ValueError("Secret name parameter cannot be None.")

try:
response = requests.get(
url = f"{self._members_endpoint}/{secret_name}",
headers=self._api_header
)

# Return secret value directly for user ease-of-use
if response.ok:
response = response.json()
return response["secret_value"]

logger.debug(f"Response from get_secret request: {response.text}")
return json.loads(response.text)
except Exception as e:
raise(f"Error retrieving secret: {e}")


def add_secret(self, secret_name=None, secret_value=None):
"""
Adds a secret. Secret name must be provided. Secret value may be null.
Args:
secret_name (str, required): Secret name.
secret_value (str, optional): Secret value.
Returns:
dict: Containing name and value of secret that was just added.
Raises:
ValueError: If secret name or secret value is not provided.
"""
if secret_name is None or secret_value is None:
raise ValueError("Failed to add secret. Secret name and secret value must not be 'None'.")

try:
response = requests.post(
url = self._members_endpoint,
headers=self._api_header,
data=json.dumps({"secret_name": secret_name, "secret_value": secret_value})
)

logger.debug(f"Response from add_secret: {response.text}")
return json.loads(response.text)
except Exception as e:
raise(f"Error adding secret: {e}")


def delete_secret(self, secret_name=None):
"""
Deletes a secret.
Args:
secret_name (str, required): Secret name.
Returns:
dict: Containing response code and message indicating whether or not deletion was successful.
Raises:
ValueError: If secret name is not provided.
"""
if secret_name is None:
raise ValueError("Failed to delete secret. Please provide secret name.")

try:
response = requests.delete(
url = f"{self._members_endpoint}/{secret_name}",
headers=self._api_header
)

logger.debug(f"Response from delete_secret: {response.text}")
return json.loads(response.text)
except Exception as e:
raise(f"Error deleting secret: {e}")






76 changes: 72 additions & 4 deletions maap/maap.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,10 @@
from maap.utils import algorithm_utils
from maap.Profile import Profile
from maap.AWS import AWS
from maap.Secrets import Secrets
from maap.dps.DpsHelper import DpsHelper
from maap.utils import endpoints
from maap.utils import job

logger = logging.getLogger(__name__)

Expand All @@ -40,10 +42,11 @@ def __init__(self, maap_host=os.getenv('MAAP_API_HOST', 'api.maap-project.org'))
self.config.workspace_bucket_credentials,
self._get_api_header()
)
self.secrets = Secrets(self.config.member, self._get_api_header(content_type="application/json"))

def _get_api_header(self, content_type=None):

api_header = {'Accept': content_type if content_type else self.config.content_type, 'token': self.config.maap_token}
api_header = {'Accept': content_type if content_type else self.config.content_type, 'token': self.config.maap_token, 'Content-Type': content_type if content_type else self.config.content_type}

if os.environ.get("MAAP_PGT"):
api_header['proxy-ticket'] = os.environ.get("MAAP_PGT")
Expand Down Expand Up @@ -256,10 +259,75 @@ def cancelJob(self, jobid):
job.id = jobid
return job.cancel_job()

def listJobs(self, page_size=None, offset=None):
url = os.path.join(self.config.dps_job, endpoints.DPS_JOB_LIST)
params = {k: v for k, v in (("page_size", page_size), ("offset", offset)) if v}
def listJobs(self, *,
algo_id=None,
end_time=None,
get_job_details=True,
offset=0,
page_size=10,
queue=None,
start_time=None,
status=None,
tag=None,
version=None):
"""
Returns a list of jobs for a given user that matches query params provided.
Args:
algo_id (str, optional): Algorithm type.
end_time (str, optional): Specifying this parameter will return all jobs that have completed from the provided end time to now. e.g. 2024-01-01 or 2024-01-01T00:00:00.000000Z.
get_job_details (bool, optional): Flag that determines whether to return a detailed job list or a compact list containing just the job ids and their associated job tags. Default is True.
offset (int, optional): Offset for pagination. Default is 0.
page_size (int, optional): Page size for pagination. Default is 10.
queue (str, optional): Job processing resource.
start_time (str, optional): Specifying this parameter will return all jobs that have started from the provided start time to now. e.g. 2024-01-01 or 2024-01-01T00:00:00.000000Z.
status (str, optional): Job status, e.g. job-completed, job-failed, job-started, job-queued.
tag (str, optional): User job tag/identifier.
version (str, optional): Algorithm version, e.g. GitHub branch or tag.
Returns:
list: List of jobs for a given user that matches query params provided.
Raises:
ValueError: If either algo_id or version is provided, but not both.
"""
url = "/".join(
segment.strip("/")
for segment in (self.config.dps_job, endpoints.DPS_JOB_LIST)
)

params = {
k: v
for k, v in (
("algo_id", algo_id),
("end_time", end_time),
("get_job_details", get_job_details),
("offset", offset),
("page_size", page_size),
("queue", queue),
("start_time", start_time),
("status", status),
("tag", tag),
("version", version),
)
if v is not None
}

if (not algo_id) != (not version):
# Either algo_id or version was supplied as a non-empty string, but not both.
# Either both must be non-empty strings or both must be None.
raise ValueError("Either supply non-empty strings for both algo_id and version, or supply neither.")

# DPS requests use 'job_type', which is a concatenation of 'algo_id' and 'version'
if algo_id and version:
params['job_type'] = f"{algo_id}:{version}"

algo_id = params.pop('algo_id', None)
version = params.pop('version', None)

if status is not None:
params['status'] = job.validate_job_status(status)

headers = self._get_api_header()
logger.debug('GET request sent to {}'.format(url))
logger.debug('headers:')
Expand Down
2 changes: 2 additions & 0 deletions maap/utils/endpoints.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,5 @@
DPS_JOB_DISMISS = "cancel"
DPS_JOB_LIST = "list"
CMR_ALGORITHM_DATA = "data"

MEMBERS_SECRETS = "secrets"
21 changes: 21 additions & 0 deletions maap/utils/job.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# Valid job statuses (loosely based on OGC job status types)
JOB_STATUSES = {'Accepted', 'Running', 'Succeeded', 'Failed', 'Dismissed', 'Deduped', 'Offline'}

def validate_job_status(status):
'''
Validates job status
Args:
status (str): Job status. Accepted values are: 'Accepted', 'Running', 'Succeeded', 'Failed, 'Dismissed', 'Deduped', and 'Offline'.
Returns:
status (str): Returns unmodified job status if job status is valid.
Raises:
ValueError: If invalid job status is provided.
'''
if status not in JOB_STATUSES:
valid_statuses = ", ".join(str(status) for status in JOB_STATUSES)
raise ValueError("Invalid job status: '{}'. Job status must be one of the following: {}".format(status, valid_statuses))

return status
696 changes: 364 additions & 332 deletions poetry.lock

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "maap-py"
version = "4.0.1a0"
version = "4.1.0a3"
description = "Python client API for interacting with the NASA MAAP API"
repository = "https://github.com/MAAP-Project/maap-py"
authors = ["Jet Propulsion Laboratory <[email protected]>"]
Expand Down
Loading

0 comments on commit f1ad56b

Please sign in to comment.