Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Hasura Events Views #1418

Merged
merged 4 commits into from
Apr 26, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions .github/workflows/pgcmp.yml
Original file line number Diff line number Diff line change
@@ -120,7 +120,12 @@ jobs:
env:
AERIE_USERNAME: "${{secrets.AERIE_USERNAME}}"
AERIE_PASSWORD: "${{secrets.AERIE_PASSWORD}}"
- name: Dump Databases
- name: Clone PGCMP
uses: actions/checkout@v4
with:
repository: cbbrowne/pgcmp
path: pgcmp
- name: Dump Migrated Database
env:
AERIE_USERNAME: "${{secrets.AERIE_USERNAME}}"
AERIE_PASSWORD: "${{secrets.AERIE_PASSWORD}}"
@@ -193,7 +198,7 @@ jobs:
with:
repository: cbbrowne/pgcmp
path: pgcmp
- name: Dump Databases
- name: Dump Current Database
env:
AERIE_USERNAME: "${{secrets.AERIE_USERNAME}}"
AERIE_PASSWORD: "${{secrets.AERIE_PASSWORD}}"
@@ -224,7 +229,7 @@ jobs:
env:
AERIE_USERNAME: "${{secrets.AERIE_USERNAME}}"
AERIE_PASSWORD: "${{secrets.AERIE_PASSWORD}}"
- name: Dump Databases
- name: Dump Migrated Database
env:
AERIE_USERNAME: "${{secrets.AERIE_USERNAME}}"
AERIE_PASSWORD: "${{secrets.AERIE_PASSWORD}}"
23 changes: 15 additions & 8 deletions deployment/aerie_db_migration.py
Original file line number Diff line number Diff line change
@@ -89,7 +89,7 @@ def step_by_step_migration(db_migration, apply):
return
input("Press Enter to continue...")

def bulk_migration(db_migration, apply):
def bulk_migration(db_migration, apply, current_version):
# Migrate the database
exit_with = 0
if apply:
@@ -98,10 +98,15 @@ def bulk_migration(db_migration, apply):
if exit_code != 0:
exit_with = 1
else:
os.system(f'hasura migrate apply --goto 1 --database-name {db_migration.db_name} --dry-run --log-level WARN &&'
f'hasura migrate apply --down 1 --database-name {db_migration.db_name} --dry-run --log-level WARN')
exit_code = os.system(f'hasura migrate apply --goto 1 --database-name {db_migration.db_name} &&'
f'hasura migrate apply --down 1 --database-name {db_migration.db_name}')
# Performing GOTO 1 when the database is at migration 1 will cause Hasura to attempt to reapply migration 1
if current_version == 1:
os.system(f'hasura migrate apply --down 1 --database-name {db_migration.db_name} --dry-run --log-level WARN')
exit_code = os.system(f'hasura migrate apply --down 1 --database-name {db_migration.db_name}')
else:
os.system(f'hasura migrate apply --goto 1 --database-name {db_migration.db_name} --dry-run --log-level WARN &&'
f'hasura migrate apply --down 1 --database-name {db_migration.db_name} --dry-run --log-level WARN')
exit_code = os.system(f'hasura migrate apply --goto 1 --database-name {db_migration.db_name} &&'
f'hasura migrate apply --down 1 --database-name {db_migration.db_name}')
if exit_code != 0:
exit_with = 1

@@ -128,7 +133,9 @@ def mark_current_version(username, password, netloc):

# Mark everything up to that as applied
for i in range(0, current_schema+1):
os.system('hasura migrate apply --skip-execution --version '+str(i)+' --database-name aerie >/dev/null 2>&1')
os.system('hasura migrate apply --skip-execution --version '+str(i)+' --database-name Aerie >/dev/null 2>&1')

return current_schema

def main():
# Create a cli parser
@@ -227,7 +234,7 @@ def main():
os.chdir(HASURA_PATH)

# Mark all migrations previously applied to the databases to be updated as such
mark_current_version(username, password, args.network_location)
current_version = mark_current_version(username, password, args.network_location)

clear_screen()
print(f'\n###############################'
@@ -238,7 +245,7 @@ def main():
# Go step-by-step through the migrations available for the selected database
step_by_step_migration(migration, args.apply)
else:
bulk_migration(migration, args.apply)
bulk_migration(migration, args.apply, current_version)

if __name__ == "__main__":
main()
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
table:
name: refresh_activity_type_logs
schema: hasura
configuration:
custom_name: "refresh_activity_type_logs"
object_relationships:
- name: model
using:
manual_configuration:
remote_table:
name: mission_model
schema: merlin
column_mapping:
model_id: id
select_permissions:
- role: aerie_admin
permission:
columns: '*'
filter: {}
allow_aggregations: true
- role: user
permission:
columns: '*'
filter: {}
allow_aggregations: true
- role: viewer
permission:
columns: '*'
filter: {}
allow_aggregations: true
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
table:
name: refresh_model_parameter_logs
schema: hasura
configuration:
custom_name: "refresh_model_parameter_logs"
object_relationships:
- name: model
using:
manual_configuration:
remote_table:
name: mission_model
schema: merlin
column_mapping:
model_id: id
select_permissions:
- role: aerie_admin
permission:
columns: '*'
filter: {}
allow_aggregations: true
- role: user
permission:
columns: '*'
filter: {}
allow_aggregations: true
- role: viewer
permission:
columns: '*'
filter: {}
allow_aggregations: true
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
table:
name: refresh_resource_type_logs
schema: hasura
configuration:
custom_name: "refresh_resource_type_logs"
object_relationships:
- name: model
using:
manual_configuration:
remote_table:
name: mission_model
schema: merlin
column_mapping:
model_id: id
select_permissions:
- role: aerie_admin
permission:
columns: '*'
filter: {}
allow_aggregations: true
- role: user
permission:
columns: '*'
filter: {}
allow_aggregations: true
- role: viewer
permission:
columns: '*'
filter: {}
allow_aggregations: true
Original file line number Diff line number Diff line change
@@ -57,6 +57,30 @@ array_relationships:
table:
name: scheduling_model_specification_goals
schema: scheduler
- name: refresh_activity_type_logs
using:
manual_configuration:
remote_table:
name: refresh_activity_type_logs
schema: hasura
column_mapping:
id: model_id
- name: refresh_model_parameter_logs
using:
manual_configuration:
remote_table:
name: refresh_model_parameter_logs
schema: hasura
column_mapping:
id: model_id
- name: refresh_resource_type_logs
using:
manual_configuration:
remote_table:
name: refresh_resource_type_logs
schema: hasura
column_mapping:
id: model_id
select_permissions:
- role: aerie_admin
permission:
3 changes: 3 additions & 0 deletions deployment/hasura/metadata/databases/tables/tables.yaml
Original file line number Diff line number Diff line change
@@ -46,6 +46,9 @@
- "!include hasura/get_conflicting_activities_return_value.yaml"
- "!include hasura/get_non_conflicting_activities_return_value.yaml"
- "!include hasura/get_plan_history_return_value.yaml"
- "!include hasura/refresh_activity_type_logs.yaml"
- "!include hasura/refresh_model_parameter_logs.yaml"
- "!include hasura/refresh_resource_types_logs.yaml"
- "!include hasura/resource_at_start_offset_return_value.yaml"
- "!include hasura/withdraw_merge_request_return_value.yaml"

Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
drop view hasura.refresh_resource_type_logs;
drop view hasura.refresh_model_parameter_logs;
drop view hasura.refresh_activity_type_logs;

drop function hasura.get_event_logs(_trigger_name text);

call migrations.mark_migration_rolled_back('1');
59 changes: 59 additions & 0 deletions deployment/hasura/migrations/Aerie/1_hasura_events_views/up.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
create function hasura.get_event_logs(_trigger_name text)
returns table (
model_id int,
model_name text,
model_version text,
triggering_user text,
delivered boolean,
success boolean,
tries int,
created_at timestamp,
next_retry_at timestamp,
status int,
error jsonb,
error_message text,
error_type text
)
stable
security invoker
language plpgsql as $$
begin
return query (
select
(el.payload->'data'->'new'->>'id')::int as model_id,
el.payload->'data'->'new'->>'name' as model_name,
el.payload->'data'->'new'->>'version' as model_version,
el.payload->'session_variables'->>'x-hasura-user-id' as triggering_user,
el.delivered,
eil.status is not distinct from 200 as success, -- is not distinct from to catch `null`
el.tries,
el.created_at,
el.next_retry_at,
eil.status,
(eil.response -> 'data'->> 'message')::jsonb as error,
(eil.response -> 'data'->> 'message')::jsonb->>'message' as error_message,
(eil.response -> 'data'->> 'message')::jsonb->>'type' as error_type
from hdb_catalog.event_log el
join hdb_catalog.event_invocation_logs eil on el.id = eil.event_id
where trigger_name = _trigger_name);
end;
$$;
comment on function hasura.get_event_logs(_trigger_name text) is e''
'Get the logs for every run of a Hasura event with the specified trigger name.';

create view hasura.refresh_activity_type_logs as
select * from hasura.get_event_logs('refreshActivityTypes');
comment on view hasura.refresh_activity_type_logs is e''
'View containing logs for every run of the Hasura event `refreshActivityTypes`.';

create view hasura.refresh_model_parameter_logs as
select * from hasura.get_event_logs('refreshModelParameters');
comment on view hasura.refresh_model_parameter_logs is e''
'View containing logs for every run of the Hasura event `refreshModelParameters`.';

create view hasura.refresh_resource_type_logs as
select * from hasura.get_event_logs('refreshResourceTypes');
comment on view hasura.refresh_resource_type_logs is e''
'View containing logs for every run of the Hasura event `refreshResourceTypes`.';

call migrations.mark_migration_applied('1');
1 change: 1 addition & 0 deletions deployment/postgres-init-db/sql/applied_migrations.sql
Original file line number Diff line number Diff line change
@@ -3,3 +3,4 @@ This file denotes which migrations occur "before" this version of the schema.
*/

call migrations.mark_migration_applied('0');
call migrations.mark_migration_applied('1');
3 changes: 3 additions & 0 deletions deployment/postgres-init-db/sql/init_hasura.sql
Original file line number Diff line number Diff line change
@@ -13,4 +13,7 @@ begin;
\ir functions/hasura/plan_branching_functions.sql
\ir functions/hasura/plan_merge_functions.sql
\ir functions/hasura/snapshot_functions.sql

-- Event Views
\ir views/hasura/hasura_event_logs.sql
end;
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
create function hasura.get_event_logs(_trigger_name text)
returns table (
model_id int,
model_name text,
model_version text,
triggering_user text,
delivered boolean,
success boolean,
tries int,
created_at timestamp,
next_retry_at timestamp,
status int,
error jsonb,
error_message text,
error_type text
)
stable
security invoker
language plpgsql as $$
begin
return query (
select
(el.payload->'data'->'new'->>'id')::int as model_id,
el.payload->'data'->'new'->>'name' as model_name,
el.payload->'data'->'new'->>'version' as model_version,
el.payload->'session_variables'->>'x-hasura-user-id' as triggering_user,
el.delivered,
eil.status is not distinct from 200 as success, -- is not distinct from to catch `null`
el.tries,
el.created_at,
el.next_retry_at,
eil.status,
(eil.response -> 'data'->> 'message')::jsonb as error,
skovati marked this conversation as resolved.
Show resolved Hide resolved
(eil.response -> 'data'->> 'message')::jsonb->>'message' as error_message,
(eil.response -> 'data'->> 'message')::jsonb->>'type' as error_type
from hdb_catalog.event_log el
join hdb_catalog.event_invocation_logs eil on el.id = eil.event_id
where trigger_name = _trigger_name);
end;
$$;
comment on function hasura.get_event_logs(_trigger_name text) is e''
'Get the logs for every run of a Hasura event with the specified trigger name.';

create view hasura.refresh_activity_type_logs as
select * from hasura.get_event_logs('refreshActivityTypes');
comment on view hasura.refresh_activity_type_logs is e''
'View containing logs for every run of the Hasura event `refreshActivityTypes`.';

create view hasura.refresh_model_parameter_logs as
select * from hasura.get_event_logs('refreshModelParameters');
comment on view hasura.refresh_model_parameter_logs is e''
'View containing logs for every run of the Hasura event `refreshModelParameters`.';

create view hasura.refresh_resource_type_logs as
select * from hasura.get_event_logs('refreshResourceTypes');
comment on view hasura.refresh_resource_type_logs is e''
'View containing logs for every run of the Hasura event `refreshResourceTypes`.';



Loading