diff --git a/.env b/.env index 617b889b..4872a4a8 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -DOCKER_TAG=v1.14.0 +DOCKER_TAG=v2.1.0 REPOSITORY_DOCKER_URL=ghcr.io/nasa-ammos AERIE_USERNAME=aerie diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d9fed4ec..45f57d53 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,7 +47,7 @@ jobs: strategy: matrix: python-version: [ "3.6.15", "3.11" ] - aerie-version: ["1.13.0", "1.14.0"] + aerie-version: ["2.1.0"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} diff --git a/docker-compose-test.yml b/docker-compose-test.yml index ee2fd124..cf15135e 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -8,8 +8,7 @@ services: GQL_API_URL: http://localhost:8080/v1/graphql HASURA_GRAPHQL_JWT_SECRET: "${HASURA_GRAPHQL_JWT_SECRET}" LOG_FILE: console - LOG_LEVEL: debug - NODE_TLS_REJECT_UNAUTHORIZED: "0" + LOG_LEVEL: warn PORT: 9000 POSTGRES_AERIE_MERLIN_DB: aerie_merlin POSTGRES_HOST: postgres @@ -21,6 +20,8 @@ services: restart: always volumes: - aerie_file_store:/app/files + networks: + - aerie_net aerie_merlin: container_name: aerie_merlin depends_on: ["postgres"] @@ -43,8 +44,10 @@ services: restart: always volumes: - aerie_file_store:/usr/src/app/merlin_file_store + networks: + - aerie_net aerie_merlin_worker_1: - container_name: aerie_merlin_worker_1 + container_name: aerie_merlin_worker depends_on: ["postgres"] environment: MERLIN_WORKER_DB: "aerie_merlin" @@ -64,27 +67,8 @@ services: restart: always volumes: - aerie_file_store:/usr/src/app/merlin_file_store:ro - aerie_merlin_worker_2: - container_name: aerie_merlin_worker_2 - depends_on: ["postgres"] - environment: - MERLIN_WORKER_DB: "aerie_merlin" - MERLIN_WORKER_DB_PASSWORD: "${AERIE_PASSWORD}" - MERLIN_WORKER_DB_PORT: 5432 - MERLIN_WORKER_DB_SERVER: postgres - MERLIN_WORKER_DB_USER: "${AERIE_USERNAME}" - MERLIN_WORKER_LOCAL_STORE: /usr/src/app/merlin_file_store - SIMULATION_PROGRESS_POLL_PERIOD_MILLIS: 2000 - JAVA_OPTS: > - -Dorg.slf4j.simpleLogger.defaultLogLevel=INFO - -Dorg.slf4j.simpleLogger.log.com.zaxxer.hikari=WARN - -Dorg.slf4j.simpleLogger.logFile=System.err - UNTRUE_PLAN_START: "2000-01-01T11:58:55.816Z" - image: "${REPOSITORY_DOCKER_URL}/aerie-merlin-worker:${DOCKER_TAG}" - ports: ["27188:8080"] - restart: always - volumes: - - aerie_file_store:/usr/src/app/merlin_file_store:ro + networks: + - aerie_net aerie_scheduler: container_name: aerie_scheduler depends_on: ["aerie_merlin", "postgres"] @@ -105,6 +89,8 @@ services: restart: always volumes: - aerie_file_store:/usr/src/app/merlin_file_store + networks: + - aerie_net aerie_scheduler_worker_1: container_name: aerie_scheduler_worker_1 depends_on: ["postgres"] @@ -127,27 +113,8 @@ services: restart: always volumes: - aerie_file_store:/usr/src/app/merlin_file_store:ro - aerie_scheduler_worker_2: - depends_on: ["postgres"] - environment: - HASURA_GRAPHQL_ADMIN_SECRET: "${HASURA_GRAPHQL_ADMIN_SECRET}" - MERLIN_GRAPHQL_URL: http://hasura:8080/v1/graphql - SCHEDULER_WORKER_DB: "aerie_scheduler" - SCHEDULER_WORKER_DB_PASSWORD: "${AERIE_PASSWORD}" - SCHEDULER_WORKER_DB_PORT: 5432 - SCHEDULER_WORKER_DB_SERVER: postgres - SCHEDULER_WORKER_DB_USER: "${AERIE_USERNAME}" - SCHEDULER_OUTPUT_MODE: UpdateInputPlanWithNewActivities - MERLIN_LOCAL_STORE: /usr/src/app/merlin_file_store - SCHEDULER_RULES_JAR: /usr/src/app/merlin_file_store/scheduler_rules.jar - JAVA_OPTS: > - -Dorg.slf4j.simpleLogger.log.com.zaxxer.hikari=INFO - -Dorg.slf4j.simpleLogger.logFile=System.err - image: "${REPOSITORY_DOCKER_URL}/aerie-scheduler-worker:${DOCKER_TAG}" - ports: ["27190:8080"] - restart: always - volumes: - - aerie_file_store:/usr/src/app/merlin_file_store:ro + networks: + - aerie_net aerie_sequencing: container_name: aerie_sequencing depends_on: ["postgres"] @@ -168,12 +135,13 @@ services: restart: always volumes: - aerie_file_store:/usr/src/app/sequencing_file_store + networks: + - aerie_net aerie_ui: container_name: aerie_ui depends_on: ["postgres"] environment: ORIGIN: http://localhost - NODE_TLS_REJECT_UNAUTHORIZED: "0" PUBLIC_LOGIN_PAGE: "enabled" PUBLIC_GATEWAY_CLIENT_URL: http://localhost:9000 PUBLIC_GATEWAY_SERVER_URL: http://aerie_gateway:9000 @@ -183,6 +151,8 @@ services: image: "${REPOSITORY_DOCKER_URL}/aerie-ui:${DOCKER_TAG}" ports: ["80:80"] restart: always + networks: + - aerie_net hasura: container_name: aerie_hasura depends_on: ["postgres"] @@ -205,9 +175,12 @@ services: image: "${REPOSITORY_DOCKER_URL}/aerie-hasura:${DOCKER_TAG}" ports: ["8080:8080"] restart: always + networks: + - aerie_net postgres: container_name: aerie_postgres environment: + POSTGRES_DB: postgres POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}" POSTGRES_USER: "${POSTGRES_USER}" AERIE_USERNAME: "${AERIE_USERNAME}" @@ -217,8 +190,13 @@ services: restart: always volumes: - postgres_data:/var/lib/postgresql/data + networks: + - aerie_net volumes: aerie_file_store: mission_file_store: postgres_data: + +networks: + aerie_net: diff --git a/pyproject.toml b/pyproject.toml index f93ddce6..ad7e73ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "aerie-cli" -version = "2.1.1" +version = "2.2.0" description = "A CLI application and Python API for interacting with Aerie." authors = [] license = "MIT" diff --git a/src/aerie_cli/aerie_client.py b/src/aerie_cli/aerie_client.py index a0c35bb4..3c606482 100644 --- a/src/aerie_cli/aerie_client.py +++ b/src/aerie_cli/aerie_client.py @@ -823,7 +823,7 @@ def create_expansion_rule( return data["id"] def create_expansion_set( - self, command_dictionary_id: int, model_id: int, expansion_ids: List[int] + self, command_dictionary_id: int, model_id: int, expansion_ids: List[int], name: str ) -> int: """Create an Aerie expansion set given a list of activity IDs @@ -831,6 +831,7 @@ def create_expansion_set( command_dictionary_id (int): ID of Aerie command dictionary model_id (int): ID of Aerie mission model expansion_ids (List[int]): List of expansion IDs to include in the set + name (str): Name of the expansion set Returns: int: Expansion set ID @@ -841,11 +842,13 @@ def create_expansion_set( $command_dictionary_id: Int! $mission_model_id: Int! $expansion_ids: [Int!]! + $name: String! ) { createExpansionSet( commandDictionaryId: $command_dictionary_id missionModelId: $mission_model_id expansionIds: $expansion_ids + name: $name ) { id } @@ -856,6 +859,7 @@ def create_expansion_set( command_dictionary_id=command_dictionary_id, mission_model_id=model_id, expansion_ids=expansion_ids, + name=name ) return data["id"] @@ -864,11 +868,17 @@ def list_expansion_sets(self) -> List[ExpansionSet]: query ListExpansionSets { expansion_set { id - command_dict_id created_at + updated_at + owner + updated_by + command_dict_id expansion_rules { id } + description + name + mission_model_id } } """ @@ -1631,21 +1641,32 @@ def get_constraint_by_id(self, id): def get_constraint_violations(self, plan_id): get_violations_query = """ query ($plan_id: Int!) { - constraintResults: constraintViolations(planId: $plan_id) { - constraintId - constraintName - type - resourceIds - violations { - activityInstanceIds - windows { - start + constraintResponses: constraintViolations(planId: $plan_id) { + success + results { + constraintId + constraintName + resourceIds + type + gaps { end + start + } + violations { + activityInstanceIds + windows { + end + start + } } } - gaps { - start - end + errors { + message + stack + location { + column + line + } } } } diff --git a/src/aerie_cli/commands/expansion.py b/src/aerie_cli/commands/expansion.py index fff57453..998ee619 100644 --- a/src/aerie_cli/commands/expansion.py +++ b/src/aerie_cli/commands/expansion.py @@ -281,18 +281,20 @@ def list_expansion_sets(): table = Table( title="Expansion Sets" ) - table.add_column("Set ID", no_wrap=True) - table.add_column("Mission", no_wrap=True) - table.add_column("CMD Dict. Version", no_wrap=True) + table.add_column("ID", no_wrap=True) + table.add_column("Name", no_wrap=True) + table.add_column("Model ID", no_wrap=True) table.add_column("CMD Dict. ID", no_wrap=True) - table.add_column("Created At", no_wrap=True) + table.add_column("CMD Dict. Version", no_wrap=True) + table.add_column("Owner", no_wrap=True) for e_set in sets: table.add_row( str(e_set.id), - cmd_dicts[e_set.command_dictionary_id].mission, - cmd_dicts[e_set.command_dictionary_id].version, + str(e_set.name), + str(e_set.mission_model_id), str(e_set.command_dictionary_id), - e_set.created_at.ctime() + cmd_dicts[e_set.command_dictionary_id].version, + e_set.owner ) Console().print(table) @@ -341,41 +343,65 @@ def create_expansion_set( ..., '--command-dict-id', '-d', prompt='Command Dictionary ID', help='Command Dictionary ID' ), + name: str = typer.Option( + ..., "--name", "-n", prompt="Expansion set name", + help="Expansion set name" + ), activity_types: List[str] = typer.Option( None, '--activity-types', '-a', help='Activity types to be included in the set' + ), + rule_ids: List[int] = typer.Option( + None, '--rule-ids', '-r', + help='Expansion rules to be included in the set' ) ): """ Create an expansion set Uses the newest expansion rules for each given activity type. - Filters to only use rules designated for the given mission model and - command dictionary. + Specify either a list of activity type names or rule IDs. If activity type + names are given, rules are filtered to only those designated for the given + mission model and command dictionary, then the highest rule ID is used for + each activity type named. """ client = CommandContext.get_client() expansion_rules = client.get_rules_by_type() - if not activity_types: - types_str = typer.prompt('Activity Types (separate with spaces)') - activity_types = types_str.split(' ') - - rule_ids = [] - for at in activity_types: - - try: - at_rules = expansion_rules[at] - for r in at_rules: - if not r.authoring_command_dict_id == command_dictionary_id: - at_rules.remove(r) - if not r.authoring_mission_model_id == model_id: - at_rules.remove(r) - rule_ids.append(max([r.id for r in at_rules])) - except (KeyError, ValueError): - Console().print( - f"No expansion rules for activity type: {at}", style='red') - return + if activity_types and rule_ids: + Console().print( + "Only specify either activity type name(s) or rule ID(s).", style='red') + return + + if not activity_types and not rule_ids: + choice = select_from_list( + ["By name", "By expansion rule ID"], "Choose how to specify rules") + if choice == "By name": + types_str = typer.prompt('Activity Types (separate with spaces)') + activity_types = types_str.split(' ') + else: + rule_ids_str = typer.prompt( + 'Expansion rule IDs (separate with spaces)') + rule_ids = [int(r) for r in rule_ids_str.split(' ')] + + if activity_types: + + rule_ids = [] + for at in activity_types: + + try: + at_rules = expansion_rules[at] + for r in at_rules: + if not r.authoring_command_dict_id == command_dictionary_id: + at_rules.remove(r) + if not r.authoring_mission_model_id == model_id: + at_rules.remove(r) + rule_ids.append(max([r.id for r in at_rules])) + except (KeyError, ValueError): + Console().print( + f"No expansion rules for activity type: {at}", style='red') + return set_id = client.create_expansion_set( - command_dictionary_id, model_id, rule_ids) + command_dictionary_id, model_id, rule_ids, name) Console().print(f"Created expansion set: {set_id}") diff --git a/src/aerie_cli/commands/plans.py b/src/aerie_cli/commands/plans.py index 91557576..7b616b84 100644 --- a/src/aerie_cli/commands/plans.py +++ b/src/aerie_cli/commands/plans.py @@ -66,7 +66,7 @@ def download_resources( False, "--absolute-time", help="Change relative timestamps to absolute" ), specific_states: str = typer.Option( - None, help="The file with the specific states [defaults to all]" + None, help="The file with the specific states, one state per line [defaults to all]" ) ): """ diff --git a/src/aerie_cli/schemas/client.py b/src/aerie_cli/schemas/client.py index a5b8a1a5..c8ecdc00 100644 --- a/src/aerie_cli/schemas/client.py +++ b/src/aerie_cli/schemas/client.py @@ -340,11 +340,19 @@ class ExpansionSet(ClientSerialize): created_at: Arrow = field( converter = arrow.get ) + updated_at: Arrow = field( + converter = arrow.get + ) + updated_by: str command_dictionary_id: int = field( alias="command_dict_id" ) expansion_rules: List[int] = field( converter = lambda x: [i['id'] for i in x]) + description: str + name: str + mission_model_id: int + owner: str @define diff --git a/tests/integration_tests/README.md b/tests/integration_tests/README.md index a9790914..cfabf9a1 100644 --- a/tests/integration_tests/README.md +++ b/tests/integration_tests/README.md @@ -1,11 +1,14 @@ -# Test against an active instance of Aerie +# Integration Tests + +Aerie-CLI integration tests exercise commands against an actual instance of Aerie. ## WARNING + These tests will delete and modify data permanently. Expect the localhost instance of Aerie to be modified heavily. These test will delete all models. See: [localhost configuration](files/configuration/localhost_config.json) -## Running locally +## Running Locally To set up a local test environment, use the test environment and docker-compose files in the root of the repo: @@ -15,7 +18,19 @@ docker compose -f docker-compose-test.yml up Invoke the tests using `pytest` from the `tests/integration_tests` directory. -## Tests +## Updating Tests for New Aerie Versions + +Integration tests are automatically run by CI against all supported Aerie versions. To add and test support for a new Aerie version: + +1. Download the appropriate version release JAR for the [Banananation model](https://github.com/NASA-AMMOS/aerie/packages/1171106/versions) and add it to `tests/integration_tests/models`, named as `banananation-X.X.X.jar` (substituting the correct version number). +2. Update the [`.env`](../../.env) file `DOCKER_TAG` value to the new version string. This defaults the local deployment to the latest Aerie version. +3. Update [`docker-compose-test.yml`](../../docker-compose-test.yml) as necessary to match the new Aerie version. The [aerie-ui compose file](https://github.com/NASA-AMMOS/aerie-ui/blob/develop/docker-compose-test.yml) can be a helpful reference to identify changes. +4. Manually run the integration tests and update the code and tests as necessary for any Aerie changes. +5. Update the `aerie-version` list in the [CI configuration](../../.github/workflows/test.yml) to include the new version. +6. If breaking changes are necessary to support the new Aerie version, remove any Aerie versions which are no longer supported from the CI configuration and remove the corresponding banananation JAR file. +7. Open a PR and verify all tests still pass. + +## Summary of Integration Tests ### [Configurations test](test_configurations.py) - Configuration initialization is in conftest.py to ensure all tests use localhost @@ -39,4 +54,4 @@ Invoke the tests using `pytest` from the `tests/integration_tests` directory. - Test all `metadata` commands ### [Constraints test](test_constraints.py) -- Test all `constraints` commands \ No newline at end of file +- Test all `constraints` commands diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py index e0f9953a..a5e20dfe 100644 --- a/tests/integration_tests/conftest.py +++ b/tests/integration_tests/conftest.py @@ -41,7 +41,7 @@ CONFIGURATIONS_PATH = os.path.join(FILES_PATH, "configuration") CONFIGURATION_PATH = os.path.join(CONFIGURATIONS_PATH, "localhost_config.json") MODELS_PATH = os.path.join(FILES_PATH, "models") -MODEL_VERSION = os.environ.get("AERIE_VERSION", "1.14.0") +MODEL_VERSION = os.environ.get("AERIE_VERSION", "2.1.0") MODEL_JAR = os.path.join(MODELS_PATH, f"banananation-{MODEL_VERSION}.jar") MODEL_NAME = "banananation" MODEL_VERSION = "0.0.1" diff --git a/tests/integration_tests/files/models/banananation-1.14.0.jar b/tests/integration_tests/files/models/banananation-1.14.0.jar deleted file mode 100644 index 91e6352d..00000000 Binary files a/tests/integration_tests/files/models/banananation-1.14.0.jar and /dev/null differ diff --git a/tests/integration_tests/files/models/banananation-1.13.0.jar b/tests/integration_tests/files/models/banananation-2.1.0.jar similarity index 97% rename from tests/integration_tests/files/models/banananation-1.13.0.jar rename to tests/integration_tests/files/models/banananation-2.1.0.jar index 26d343c4..6c7618d0 100644 Binary files a/tests/integration_tests/files/models/banananation-1.13.0.jar and b/tests/integration_tests/files/models/banananation-2.1.0.jar differ diff --git a/tests/integration_tests/test_configurations.py b/tests/integration_tests/test_configurations.py index e4e2430c..01959958 100644 --- a/tests/integration_tests/test_configurations.py +++ b/tests/integration_tests/test_configurations.py @@ -243,7 +243,7 @@ def test_last_activate(): result = runner.invoke( app, - ["activate"], + ["activate", "-r", "aerie_admin"], input=str(configuration_id) + "\n", catch_exceptions=False, ) diff --git a/tests/integration_tests/test_expansion.py b/tests/integration_tests/test_expansion.py index c8d68b27..c004fdb8 100644 --- a/tests/integration_tests/test_expansion.py +++ b/tests/integration_tests/test_expansion.py @@ -143,8 +143,19 @@ def test_expansion_set_create(): ) result = runner.invoke( app, - ["expansion", "sets", "create"], - input=str(model_id) + "\n" + str(command_dictionary_id) + "\n" + "BakeBananaBread" + "\n", + [ + "expansion", + "sets", + "create", + "-m", + str(model_id), + "-d", + str(command_dictionary_id), + "-n", + "integration_test-" + arrow.utcnow().format("YYYY-MM-DDTHH-mm-ss"), + "-a", + "BakeBananaBread" + ], catch_exceptions=False,) assert result.exit_code == 0,\ f"{result.stdout}"\ @@ -179,6 +190,7 @@ def test_expansion_set_list(): f"{result.stdout}"\ f"{result.stderr}" assert "Expansion Sets" in result.stdout + assert "integration_test" in result.stdout ####################### # TEST EXPANSION RUNS diff --git a/tests/integration_tests/test_scheduling.py b/tests/integration_tests/test_scheduling.py index bd237ee0..887c039f 100644 --- a/tests/integration_tests/test_scheduling.py +++ b/tests/integration_tests/test_scheduling.py @@ -66,7 +66,7 @@ def set_up_environment(request): def cli_schedule_upload(): schedule_file_path = os.path.join(GOALS_PATH, "schedule1.txt") - with open(schedule_file_path, "x") as fid: + with open(schedule_file_path, "w") as fid: fid.write(GOAL_PATH) result = runner.invoke( app,