Definitions Update

Add gh token
Add defintions automatic release pipeline
Add documentation

.travis.yml

@@ -1,17 +1,55 @@
+env:
+  matrix:
+  - GITHUB_REPO=alertlogic/alertlogic-sdk-definitions
+  global:
+  - secure: BXki2OvWA47YSLeKnAyNENgMNz148IFl+8ohpV7liXHORW/rkbUE3MYCqNJgT+zNqOJDhGSO/fiAb8mOxjBNjeWgqhxu8H+KqOBKU9B0XpagqM1fqpAn2duKxb51ULNKQvjRcsn/tEXqyIe2ipO0G7P8VJQzuHvrVEqVWDXMbbJAj2s1ZAfgzzFRiI7dDweWrPwLNwK9kMxjauTEpaAVuhFCGl2ga6nG8a26ORxecxvELXwEcKBGW7gJWZhpk+Y1Fhd6YeJKNmvok0XthqLJNaRV2JRErzyYpaLnnw+FgTpfSF2taREge4wlDzYbKRP4yVlC3IFn2QLq77kff/NsPck5n6M7B+PIzKrsu5bwCU3Apx1y78pstLTVVNb0xCLh61JGn8Va/vfExX1lpDMd42+9a2RTALEXHcgvdF+gx/KL8Hwa4HsmOaAJqWmfivdQilLzvdbxmnu0f+lsDnwsJy5FciBLjdZ/VTmphWOTMzopMHJdl7dNzFafwDTTAVaS0FA7EoHdZAm1m/4C5iZ+B+85SFCQxt1/Ot3/8ntTub0OPep98S1RNEqGqjeIXosh3KKdT8gp27kIcuqj3iQvsqFGMVeDqQqij/RUkizX1z0CoX8M4Pbi9BQpM7g8aNGqTJtLPaYbccURy56UarkbK7mG63tWTW4pba0vyfSwTg0=
 language: python
-python:
-- 3.8
-- 3.7
-install: pip install -U tox-travis
-script: tox
-deploy:
-  provider: pypi
-  distributions: sdist bdist_wheel
-  skip_cleanup: true
-  user: __token__
-  password:
-    secure: eJMWcJY8inSzxPccLTlEN5nWzeDa2zzovgYX7HB+oeoNGEMSu+XS6HV1ILARHyDJovhPlkgZplFcJxjRZITBsjFAatpFeBSTqeV1/qkXs4rSY4wYF71x7dkBRQxEuAjsH9v1qzHvE9JXdzdvy/z/U36Dh1HXLhW9uYQ+Rfo0k/+VR6sAByMHAEFWCibt3UbrMm3l1j8X/95Z0h5+xJODnTc4QCiNK/HYhdzq5MU415CLHo1BLFlP6GyWK+Mi+bQu2cZnU+qnKlSYjmfKJYtL3tJ97TtXNRgvhiS2pgi+qldfNv8kaj9trkUJgsn6QaTGl93CEMthMCcVyYPd7qDjpN2If5NrWBqiTVid/g2HO33WleFRQjp9UF6eRsDozecSSVGefVZc6Aorl8usL9Kf0nKFHGm9P2lbjHofiJ23la8W43CT1gOSxUFs4u8v9AFh4aCI1v6TwdlVX7/1EbcsFsOFVXBEuE6RB0ZGbSQcafcObdxbg8wNJsDBpz/JVFicWdfTZ1C4LEZkAsuwwHvLlfwYX51984fr5493h7CNmWfHziMXTp68YvZsoQ8OCVDEAerYMf6eeWBXhZWDJLUUBPl89u7pVnQrKq9YyZCsUJSfrq6pQ4+3bsiLgwp4Zyho4XS8w8Y1bXbjX/6jFAK5tXMexKkg19jLd/V0ef6eEsY=
-  on:
-    tags: true
-    repo: alertlogic/alertlogic-sdk-definitions
+jobs:
+  include:
+  - stage: test
+    name: 'Run python tests '
     python: 3.8
+    install: pip install -U tox-travis
+    script: tox
+  - python: 3.7
+    install: pip install -U tox-travis
+    script: tox
+  - stage: merge-automated-prs
+    name: "(Pipeline Stage 1)Merge pull request produced by automation"
+    if: "(fork = false) AND (head_branch =~ /^auto-update-\\d*/) AND (type = pull_request)"
+    python: 3.8
+    script: BRANCHES_TO_MERGE_REGEX='^auto-update-' BRANCH_TO_MERGE_INTO=master scripts/travis-automerge.sh
+  - stage: auto_release
+    name: "(Pipeline Stage 2)Automatically release definitions update"
+    if: branch = master AND type = push
+    python: 3.8
+    install: pip install -U packaging requests
+    script: scripts/create_release.py --repo $GITHUB_REPO -re "^Definitions Update.*"  --create_release
+  - stage: github_release
+    if: type = push AND tag =~ /^v\d*\.\d*\.\d*$/
+    name: "(Pipeline Stage 3)Create release containing just definitions"
+    script: zip -rj AlertLogicOpenAPIDefinitions.zip alsdkdefs/apis
+    deploy:
+      provider: releases
+      api_key: "$GITHUB_SECRET_TOKEN"
+      skip_cleanup: true
+      file: AlertLogicOpenAPIDefinitions.zip
+      on:
+        tags: true
+  - stage: deploy
+    name: "(Pipeline Stage 4)Deploy to PyPI"
+    if: type = push AND tag =~ /^v\d*\.\d*\.\d*$/
+    python: 3.8
+    install: pip install -U tox-travis
+    script: tox
+    deploy:
+      provider: pypi
+      distributions: sdist bdist_wheel
+      skip_cleanup: true
+      user: __token__
+      password:
+        secure: eJMWcJY8inSzxPccLTlEN5nWzeDa2zzovgYX7HB+oeoNGEMSu+XS6HV1ILARHyDJovhPlkgZplFcJxjRZITBsjFAatpFeBSTqeV1/qkXs4rSY4wYF71x7dkBRQxEuAjsH9v1qzHvE9JXdzdvy/z/U36Dh1HXLhW9uYQ+Rfo0k/+VR6sAByMHAEFWCibt3UbrMm3l1j8X/95Z0h5+xJODnTc4QCiNK/HYhdzq5MU415CLHo1BLFlP6GyWK+Mi+bQu2cZnU+qnKlSYjmfKJYtL3tJ97TtXNRgvhiS2pgi+qldfNv8kaj9trkUJgsn6QaTGl93CEMthMCcVyYPd7qDjpN2If5NrWBqiTVid/g2HO33WleFRQjp9UF6eRsDozecSSVGefVZc6Aorl8usL9Kf0nKFHGm9P2lbjHofiJ23la8W43CT1gOSxUFs4u8v9AFh4aCI1v6TwdlVX7/1EbcsFsOFVXBEuE6RB0ZGbSQcafcObdxbg8wNJsDBpz/JVFicWdfTZ1C4LEZkAsuwwHvLlfwYX51984fr5493h7CNmWfHziMXTp68YvZsoQ8OCVDEAerYMf6eeWBXhZWDJLUUBPl89u7pVnQrKq9YyZCsUJSfrq6pQ4+3bsiLgwp4Zyho4XS8w8Y1bXbjX/6jFAK5tXMexKkg19jLd/V0ef6eEsY=
+      on:
+        tags: true
+        repo: "$GITHUB_REPO"
+        python: 3.8

MANIFEST.in

@@ -0,0 +1,3 @@
+include LICENSE
+include README.md
+exclude scripts/

README.md

@@ -1,2 +1,40 @@
 # alertlogic-sdk-definitions
 Alert Logic APIs definitions
+
+[![Build Status](https://travis-ci.com/alertlogic/alertlogic-sdk-definitions.svg?branch=master)](https://travis-ci.com/alertlogic/alertlogic-sdk-definitions)
+
+Repository contains static definitions of Alert Logic APIs, used for documentation generation, 
+[SDK](https://github.com/alertlogic/alertlogic-sdk-python) and [CLI](https://github.com/alertlogic/alcli).
+
+### Usage
+
+#### Install 
+`pip install alertlogic-sdk-definitions`
+
+For the one who doesn't require python code, GitHub releases are produced 
+containing an archive with OpenAPI definitions only, see
+[here](https://github.com/alertlogic/alertlogic-sdk-definitions/releases)
+
+#### Test
+`python -m unittest`
+
+#### Use
+
+List available service definitions:
+```
+>>> import alsdkdefs
+>>> alsdkdefs.list_services()
+['aefr', 'aerta', 'aetag', 'aetuner', 'aims', 'assets_query', 'credentials', 'deployments', 'ingest', 'iris', 'policies', 'search', 'themis']
+```
+
+Get path to a service definitions paths:
+```
+>>> import alsdkdefs
+>>> alsdkdefs.get_service_defs("aerta")
+['/usr/local/lib/python3.8/site-packages/alsdkdefs/apis/aerta/aerta.v1.yaml']
+```
+
+### Development
+
+Please submit a PR. Please note that API definitions are updated automatically and any changes to it will be overwritten, see:
+[automatic update process](doc/automatic_releases.md)

doc/CI.puml

@@ -0,0 +1,15 @@
+@startuml
+(*)  --> "service API defintion update detected"
+--> "Automated PR is created"
+If "PR change test passed" then
+--> [Yes] "PR automatically merged"
+--> "master branch is tagged with new version \nif commit is detected to be made by automation"
+--> "package is deployed to PyPi"
+--> "github release is created"
+-> (*)
+else
+--> "repo maintainer notified"
+--> "probem resolved"
+->(*)
+Endif
+@enduml

doc/automatic_releases.md

@@ -0,0 +1,16 @@
+## Automatic microservices API releases
+
+Alert Logic microservices code is maintained privately, OpenAPI definitions source code 
+is kept along with the code of the microservices. 
+Automated process is keeping current repository up-to-date:
+
+![CI process](images/CI.png)
+
+Definitions release pipeline is broken onto stages:
+- (Stage 0)Automation creates a pull request with definitions change
+- (Stage 1)Merge pull request produced by automation
+- (Stage 2)Automatically release definitions update(calculate new version and tag changes)
+- (Stage 3)Create release containing just definitions
+- (Stage 4)Deploy to PyPI
+
+Please note, that automation only changes X.Y.<micro> version, major and minor would be changed manually. 

scripts/create_release.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+import requests
+from packaging import version
+from argparse import ArgumentParser
+import re
+import os
+import datetime
+import json
+from functools import reduce
+
+
+def make_auth_header(token):
+    return {"Authorization": f"token {token}"}
+
+
+def list_github_tags(token, repo):
+    return requests.get(f"https://api.github.com/repos/{repo}/tags", headers=make_auth_header(token)).json()
+
+
+def list_version_tags(github_tags):
+    return list(filter(lambda v: isinstance(v, version.Version), map(lambda t: version.parse(t['name']), github_tags)))
+
+
+def reduce_tag(acc, tag):
+    acc[version.parse(tag['name'])] = tag
+    return acc
+
+
+def make_tags_search_hash(github_tags):
+    return reduce(reduce_tag, github_tags, {})
+
+
+def get_latest_version(parsed_tags):
+    sorted_tags = sorted(parsed_tags, reverse=True)
+    if sorted_tags:
+        return sorted_tags[0]
+    else:
+        return version.parse("0.0.0")
+
+
+def get_branch_commit_sha(token, repo, branch):
+    url = f"https://api.github.com/repos/{repo}/branches/{branch}"
+    return requests.get(url, headers=make_auth_header(token)).json()['commit']['sha']
+
+
+def get_branch_commit_message(token, repo, branch):
+    url = f"https://api.github.com/repos/{repo}/branches/{branch}"
+    return requests.get(url, headers=make_auth_header(token)).json()['commit']['commit']['message']
+
+
+def create_lightweight_tag(token, repo, tag_obj):
+    url = f"https://api.github.com/repos/{repo}/git/refs"
+    r = requests.post(url, headers=make_auth_header(token), json=tag_obj)
+    # meh, just roughly
+    if 201 <= r.status_code < 300:
+        return True
+    else:
+        print(
+            f"Failed to create tag {json.dumps(tag_obj, indent=4)}, because {r.status_code} {json.dumps(r.json(), indent=4)}")
+        return False
+
+
+def create_annotated_tag(token, repo, tag_obj):
+    url = f"https://api.github.com/repos/{repo}/git/tags"
+    r = requests.post(url, headers=make_auth_header(token), json=tag_obj)
+    # meh, just roughly
+    if 201 <= r.status_code < 300:
+        return True
+    else:
+        print(
+            f"Failed to create tag {json.dumps(tag_obj, indent=4)}, because {r.status_code} {json.dumps(r.json(), indent=4)}")
+        return False
+
+
+def make_lightweight_tag_object(version, sha):
+    return {
+        "ref": f"refs/tags/v{version}",
+        "sha": sha
+    }
+
+
+def make_annotated_tag_object(version, tag_message, sha, tagger='CI Bot', email='travis@travis'):
+    date = datetime.datetime.now().astimezone().replace(microsecond=0).isoformat()
+    return {
+        "tag": f"v{version}",
+        "message": tag_message,
+        "object": sha,
+        "type": "commit",
+        "tagger": {
+            "name": tagger,
+            "email": email,
+            "date": date
+        }
+    }
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser(description="Calculates and returns new version based on the version tags of the "
+                                        "specified repo, only PEP440 versions are taken into account. "
+                                        "If micro version is not passed, "
+                                        "version is incremented by --increment, default is 1")
+    parser.add_argument("-t", "--token", dest="token",
+                        help="github api token, if not set taken from GITHUB_SECRET_TOKEN",
+                        default=os.getenv('GITHUB_SECRET_TOKEN'))
+    parser.add_argument("-c", "--create_release", action="store_true", default=False,
+                        dest="do_release", help="create calculated new release")
+    parser.add_argument("-b", "--branch", dest="branch", help="branch to tag", default="master")
+    parser.add_argument("-r", "--repo", dest="repo", help="github repo", required=True)
+    parser.add_argument("-re", "--commit_message_regex", dest="regex", default=".*",
+                        help="commit regex to be tagged, if not set any commit on given branch will be tagged")
+    parser.add_argument("-m", "--micro", dest="micro", type=int, help="new micro version <major>.<minor>.<micro>")
+    parser.add_argument("-i", "--increment", dest="increment", help="increment minor version by this number",
+                        default=1, type=int)
+    options = parser.parse_args()
+    sec_token = options.token
+    if not sec_token:
+        print("Secret token is not set neither by parameter nor by environment variable")
+        exit(1)
+    repo = options.repo
+    do_release = options.do_release
+    newmicro = options.micro
+    increment = options.increment
+    branch = options.branch
+    regex = options.regex
+    tags = list_github_tags(sec_token, repo)
+    tag_search = make_tags_search_hash(tags)
+    parsed_tags = list_version_tags(tags)
+    latest = get_latest_version(parsed_tags)
+    latest_tag_sha = tag_search[latest]['commit']['sha']
+    ma = latest.major
+    mi = latest.minor
+    mic = latest.micro
+    if newmicro:
+        if mic > newmicro:
+            print(f"latest micro version {ma}.{mi}.{mic} is bigger than provided, provided {newmicro}")
+            exit(1)
+        else:
+            newrel_version = f"{ma}.{mi}.{newmicro}"
+    else:
+        newrel_version = f"{ma}.{mi}.{mic + 1}"
+    new_commit_sha = get_branch_commit_sha(sec_token, repo, branch)
+    commit_message = get_branch_commit_message(sec_token, repo, branch)
+    tag_anno_obj = make_annotated_tag_object(newrel_version, commit_message, new_commit_sha)
+    tag_ref_obj = make_lightweight_tag_object(newrel_version, new_commit_sha)
+    create_log = f"Will create tag \n{json.dumps(tag_anno_obj, indent=4)} \n" \
+                 f" new commit {new_commit_sha}, tag {newrel_version}\n " \
+                 f"old commit {latest_tag_sha}, tag {str(latest)}"
+    if re.match(regex, commit_message):
+        print(f"Commit message {commit_message} matched {regex}, proceeding to release {newrel_version}")
+        if latest_tag_sha == new_commit_sha:
+            print(f"Release aborted release {latest} already created for {new_commit_sha}")
+        else:
+            if do_release:
+                print(create_log)
+                create_annotated_tag(sec_token, repo, tag_anno_obj)
+                create_lightweight_tag(sec_token, repo, tag_ref_obj)
+            else:
+                print(create_log)
+                print("Release aborted, specify -c to actually do release")
+    else:
+        print(f"Commit message {commit_message} don't match {regex}, aborted release")

scripts/travis-automerge.sh

@@ -0,0 +1,48 @@
+#!/bin/bash -e
+
+# Inspired by https://github.com/cdown/travis-automerge
+# See https://github.com/cdown/travis-automerge/blob/90cc2a65a887d0a80f78c098892f29d458b29813/LICENSE
+
+: "${BRANCHES_TO_MERGE_REGEX?}" "${BRANCH_TO_MERGE_INTO?}"
+: "${GITHUB_SECRET_TOKEN?}" "${GITHUB_REPO?}"
+
+export GIT_COMMITTER_EMAIL='[email protected]'
+export GIT_COMMITTER_NAME='CI bot'
+
+if [ "$TRAVIS_REPO_SLUG" != "$GITHUB_REPO" ]; then
+    printf "PR repo %s is not current %s, exiting\\n" \
+        "$TRAVIS_REPO_SLUG" "$TRAVIS_REPO_SLUG" >&2
+    exit 0
+fi
+
+if ! grep -q "$BRANCHES_TO_MERGE_REGEX" <<< "$TRAVIS_PULL_REQUEST_BRANCH"; then
+    printf "Current branch %s doesn't match regex %s, exiting\\n" \
+        "$TRAVIS_PULL_REQUEST_BRANCH" "$BRANCHES_TO_MERGE_REGEX" >&2
+    exit 0
+fi
+
+# Since Travis does a partial checkout, we need to get the whole thing
+repo_temp=$(mktemp -d)
+git clone "https://github.com/$GITHUB_REPO" "$repo_temp"
+
+# shellcheck disable=SC2164
+cd "$repo_temp"
+
+printf 'Fetching branch %s to merge to %s\n' "$TRAVIS_PULL_REQUEST_BRANCH" "$BRANCH_TO_MERGE_INTO" >&2
+git fetch origin $TRAVIS_PULL_REQUEST_BRANCH:$TRAVIS_PULL_REQUEST_BRANCH
+
+printf 'Checking out %s\n' "$BRANCH_TO_MERGE_INTO" >&2
+git checkout "$BRANCH_TO_MERGE_INTO"
+
+MERGE_COMMIT_MESSAGE=`git show-branch --no-name $TRAVIS_PULL_REQUEST_BRANCH`
+
+printf 'Merging %s using "%s" message for if merge commit\n' "$TRAVIS_PULL_REQUEST_BRANCH" "MERGE_COMMIT_MESSAGE" >&2
+git merge "$TRAVIS_PULL_REQUEST_BRANCH" -m '$MERGE_COMMIT_MESSAGE'
+
+printf 'Pushing to %s\n' "$GITHUB_REPO" >&2
+
+push_uri="https://$GITHUB_SECRET_TOKEN@github.com/$GITHUB_REPO"
+
+# Redirect to /dev/null to avoid secret leakage
+git push "$push_uri" "$BRANCH_TO_MERGE_INTO" >/dev/null 2>&1
+git push "$push_uri" :"$TRAVIS_PULL_REQUEST_BRANCH" >/dev/null 2>&1