From f1e4a70f71d843a8af2128d1256bf9949c397a21 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Fri, 27 Sep 2024 09:45:48 -0400 Subject: [PATCH 01/89] Updating with new layout --- .../sql/api/admin_api_v1_1_0/base.sql | 29 + .../admin_api_v1_1_0/create_access_tables.sql | 43 + .../api/admin_api_v1_1_0/create_functions.sql | 438 +++ .../api/admin_api_v1_1_0/create_schema.sql | 56 + .../sql/api/admin_api_v1_1_0/create_views.sql | 92 + .../sql/api/admin_api_v1_1_0/drop_schema.sql | 9 + .../sql/api/admin_api_v1_1_0/drop_views.sql | 8 + .../sql/api/admin_api_v1_1_0/test.rest | 114 + .../sql/api/admin_api_v1_1_1/base.sql | 29 + .../admin_api_v1_1_1/create_access_tables.sql | 43 + .../api/admin_api_v1_1_1/create_functions.sql | 389 ++ .../api/admin_api_v1_1_1/create_schema.sql | 56 + .../sql/api/admin_api_v1_1_1/create_views.sql | 92 + .../sql/api/admin_api_v1_1_1/drop_schema.sql | 9 + .../sql/api/admin_api_v1_1_1/drop_views.sql | 8 + .../dissemination/sql/api/api_v1_0_3/base.sql | 29 + .../sql/api/api_v1_0_3/create_functions.sql | 24 + .../sql/api/api_v1_0_3/create_schema.sql | 52 + .../sql/api/api_v1_0_3/create_views.sql | 335 ++ .../sql/api/api_v1_0_3/drop_schema.sql | 11 + .../sql/api/api_v1_0_3/drop_views.sql | 17 + .../sql/api/api_v1_0_3/standup.sql | 440 +++ .../sql/api/api_v1_0_3/teardown.sql | 27 + .../dissemination/sql/api/api_v1_1_0/base.sql | 29 + .../sql/api/api_v1_1_0/config1.yaml | 43 + .../sql/api/api_v1_1_0/create_functions.sql | 126 + .../sql/api/api_v1_1_0/create_schema.sql | 60 + .../sql/api/api_v1_1_0/create_views.sql | 308 ++ .../sql/api/api_v1_1_0/db_to_db.yaml | 316 ++ .../sql/api/api_v1_1_0/drop_schema.sql | 11 + .../sql/api/api_v1_1_0/drop_views.sql | 17 + .../sql/api/api_v1_1_0/performance.sql | 36 + .../sql/api/api_v1_1_0/public_data.general | 314 ++ .../dissemination/sql/api/api_v1_1_0/sling.md | 41 + .../sql/api/api_v1_1_0/standup.sql | 523 +++ .../sql/api/api_v1_1_0/teardown.sql | 27 + .../sql/api/public_api_v2_0_0_alpha/base.sql | 36 + .../create_functions.sql | 4 + .../public_api_v2_0_0_alpha/create_schema.sql | 42 + .../public_api_v2_0_0_alpha/create_views.sql | 182 + .../public_api_v2_0_0_alpha/drop_schema.sql | 12 + .../public_api_v2_0_0_alpha/drop_views.sql | 18 + .../sql/api/public_api_v2_0_0_alpha/setup.sql | 0 .../api/public_api_v2_0_0_alpha/standup.sql | 269 ++ .../api/public_api_v2_0_0_alpha/teardown.sql | 9 + .../public_data_v1_0_0.yaml | 469 +++ .../sql/sling/public_data_v1_0_0/standup.sql | 0 .../sql/sling/public_data_v1_0_0/teardown.sql | 78 + backend/docker-compose.yml | 3 +- backend/run.sh | 19 + backend/tools/api_standup.sh | 34 +- backend/tools/api_teardown.sh | 26 +- .../all_audits_that_failed_migrations.csv | 3329 +++++++++++++++++ backend/tools/setup_cgov_env.sh | 9 + backend/tools/setup_local_env.sh | 22 + backend/tools/sling_public_tables.sh | 9 + backend/tools/util_startup.sh | 57 + 57 files changed, 8801 insertions(+), 27 deletions(-) create mode 100644 backend/dissemination/sql/api/admin_api_v1_1_0/base.sql create mode 100644 backend/dissemination/sql/api/admin_api_v1_1_0/create_access_tables.sql create mode 100644 backend/dissemination/sql/api/admin_api_v1_1_0/create_functions.sql create mode 100644 backend/dissemination/sql/api/admin_api_v1_1_0/create_schema.sql create mode 100644 backend/dissemination/sql/api/admin_api_v1_1_0/create_views.sql create mode 100644 backend/dissemination/sql/api/admin_api_v1_1_0/drop_schema.sql create mode 100644 backend/dissemination/sql/api/admin_api_v1_1_0/drop_views.sql create mode 100644 backend/dissemination/sql/api/admin_api_v1_1_0/test.rest create mode 100644 backend/dissemination/sql/api/admin_api_v1_1_1/base.sql create mode 100644 backend/dissemination/sql/api/admin_api_v1_1_1/create_access_tables.sql create mode 100644 backend/dissemination/sql/api/admin_api_v1_1_1/create_functions.sql create mode 100644 backend/dissemination/sql/api/admin_api_v1_1_1/create_schema.sql create mode 100644 backend/dissemination/sql/api/admin_api_v1_1_1/create_views.sql create mode 100644 backend/dissemination/sql/api/admin_api_v1_1_1/drop_schema.sql create mode 100644 backend/dissemination/sql/api/admin_api_v1_1_1/drop_views.sql create mode 100644 backend/dissemination/sql/api/api_v1_0_3/base.sql create mode 100644 backend/dissemination/sql/api/api_v1_0_3/create_functions.sql create mode 100644 backend/dissemination/sql/api/api_v1_0_3/create_schema.sql create mode 100644 backend/dissemination/sql/api/api_v1_0_3/create_views.sql create mode 100644 backend/dissemination/sql/api/api_v1_0_3/drop_schema.sql create mode 100644 backend/dissemination/sql/api/api_v1_0_3/drop_views.sql create mode 100644 backend/dissemination/sql/api/api_v1_0_3/standup.sql create mode 100644 backend/dissemination/sql/api/api_v1_0_3/teardown.sql create mode 100644 backend/dissemination/sql/api/api_v1_1_0/base.sql create mode 100644 backend/dissemination/sql/api/api_v1_1_0/config1.yaml create mode 100644 backend/dissemination/sql/api/api_v1_1_0/create_functions.sql create mode 100644 backend/dissemination/sql/api/api_v1_1_0/create_schema.sql create mode 100644 backend/dissemination/sql/api/api_v1_1_0/create_views.sql create mode 100644 backend/dissemination/sql/api/api_v1_1_0/db_to_db.yaml create mode 100644 backend/dissemination/sql/api/api_v1_1_0/drop_schema.sql create mode 100644 backend/dissemination/sql/api/api_v1_1_0/drop_views.sql create mode 100644 backend/dissemination/sql/api/api_v1_1_0/performance.sql create mode 100644 backend/dissemination/sql/api/api_v1_1_0/public_data.general create mode 100644 backend/dissemination/sql/api/api_v1_1_0/sling.md create mode 100644 backend/dissemination/sql/api/api_v1_1_0/standup.sql create mode 100644 backend/dissemination/sql/api/api_v1_1_0/teardown.sql create mode 100644 backend/dissemination/sql/api/public_api_v2_0_0_alpha/base.sql create mode 100644 backend/dissemination/sql/api/public_api_v2_0_0_alpha/create_functions.sql create mode 100644 backend/dissemination/sql/api/public_api_v2_0_0_alpha/create_schema.sql create mode 100644 backend/dissemination/sql/api/public_api_v2_0_0_alpha/create_views.sql create mode 100644 backend/dissemination/sql/api/public_api_v2_0_0_alpha/drop_schema.sql create mode 100644 backend/dissemination/sql/api/public_api_v2_0_0_alpha/drop_views.sql create mode 100644 backend/dissemination/sql/api/public_api_v2_0_0_alpha/setup.sql create mode 100644 backend/dissemination/sql/api/public_api_v2_0_0_alpha/standup.sql create mode 100644 backend/dissemination/sql/api/public_api_v2_0_0_alpha/teardown.sql create mode 100644 backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml create mode 100644 backend/dissemination/sql/sling/public_data_v1_0_0/standup.sql create mode 100644 backend/dissemination/sql/sling/public_data_v1_0_0/teardown.sql create mode 100644 backend/tools/historic_migrator/all_audits_that_failed_migrations.csv create mode 100644 backend/tools/sling_public_tables.sh diff --git a/backend/dissemination/sql/api/admin_api_v1_1_0/base.sql b/backend/dissemination/sql/api/admin_api_v1_1_0/base.sql new file mode 100644 index 0000000000..dedabe0cb7 --- /dev/null +++ b/backend/dissemination/sql/api/admin_api_v1_1_0/base.sql @@ -0,0 +1,29 @@ +DO +$do$ +BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = 'authenticator') THEN + RAISE NOTICE 'Role "authenticator" already exists. Skipping.'; + ELSE + CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; + END IF; +END +$do$; + +DO +$do$ +BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = 'api_fac_gov') THEN + RAISE NOTICE 'Role "api_fac_gov" already exists. Skipping.'; + ELSE + CREATE ROLE api_fac_gov NOLOGIN; + END IF; +END +$do$; + +GRANT api_fac_gov TO authenticator; + +NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/api/admin_api_v1_1_0/create_access_tables.sql b/backend/dissemination/sql/api/admin_api_v1_1_0/create_access_tables.sql new file mode 100644 index 0000000000..18db91956d --- /dev/null +++ b/backend/dissemination/sql/api/admin_api_v1_1_0/create_access_tables.sql @@ -0,0 +1,43 @@ +-- This is explicitly not a Django managed table. +-- In order to have an administrative key added, +-- it must be added via a Github commit, and a PR +-- must be performed to merge the key into the tree. + +-- This is because administrative keys can read/write +-- to some tables in the database. They can read internal and +-- in-flight data. + +DROP TABLE IF EXISTS support_administrative_key_uuids; + +CREATE TABLE support_administrative_key_uuids + ( + id BIGSERIAL PRIMARY KEY, + email TEXT, + uuid TEXT, + permissions TEXT, + added DATE + ); + +INSERT INTO support_administrative_key_uuids + (email, uuid, permissions, added) + VALUES + ( + 'matthew.jadud@gsa.gov', + '61ba59b2-f545-4c2f-9b24-9655c706a06c', + 'CREATE,READ,DELETE', + '2023-12-04' + ), + ( + 'daniel.swick@gsa.gov', + 'b6e08808-ecb2-4b6a-b928-46d4205497ff', + 'CREATE,READ,DELETE', + '2023-12-08' + ), + ( + 'fac-gov-test-users+api-tester-admin@gsa.gov', + 'dd60c3f9-053d-4d82-a309-c89da53559f4', + 'CREATE,READ,DELETE', + '2024-07-10' + ) + ; + diff --git a/backend/dissemination/sql/api/admin_api_v1_1_0/create_functions.sql b/backend/dissemination/sql/api/admin_api_v1_1_0/create_functions.sql new file mode 100644 index 0000000000..b89b001186 --- /dev/null +++ b/backend/dissemination/sql/api/admin_api_v1_1_0/create_functions.sql @@ -0,0 +1,438 @@ +-- WARNING +-- Under PostgreSQL 12, the functions below work. +-- Under PostgreSQL 14, these will break. +-- +-- Note the differences: +-- +-- raise info 'Works under PostgreSQL 12'; +-- raise info 'request.header.x-magic %', (SELECT current_setting('request.header.x-magic', true)); +-- raise info 'request.jwt.claim.expires %', (SELECT current_setting('request.jwt.claim.expires', true)); +-- raise info 'Works under PostgreSQL 14'; +-- raise info 'request.headers::json->>x-magic %', (SELECT current_setting('request.headers', true)::json->>'x-magic'); +-- raise info 'request.jwt.claims::json->expires %', (SELECT current_setting('request.jwt.claims', true)::json->>'expires'); +-- +-- To quote the work of Dav Pilkey, "remember this now." + +begin; + + +CREATE OR REPLACE FUNCTION admin_api_v1_1_0_functions.get_header(item text) RETURNS text + AS $get_header$ + declare res text; + begin + SELECT (current_setting('request.headers', true)::json)->>item into res; + return res; + end; +$get_header$ LANGUAGE plpgsql; + +create or replace function admin_api_v1_1_0_functions.get_api_key_uuid() returns TEXT +as $gaku$ +declare uuid text; +begin + select admin_api_v1_1_0_functions.get_header('x-api-user-id') into uuid; + return uuid; +end; +$gaku$ LANGUAGE plpgsql; + +-- log_api_event +-- Maintain an internal table of administrative API events. +-- Also RAISE INFO so that NR gets a copy. +create or replace function admin_api_v1_1_0_functions.log_admin_api_event(event TEXT, meta JSON) +returns boolean +as $log_admin_api_event$ +DECLARE + uuid_header text; +BEGIN + SELECT admin_api_v1_1_0_functions.get_api_key_uuid() INTO uuid_header; + + INSERT INTO public.support_adminapievent + (api_key_uuid, event, event_data, "timestamp") + VALUES (uuid_header, event, meta, NOW()); + + RAISE INFO 'ADMIN_API % % %', uuid_header, event, meta; + RETURN 1; +END; +$log_admin_api_event$ LANGUAGE plpgsql; + + +-- has_admin_data_access :: permission -> bool +-- The permissions (insert, select, delete) allow us to have users who can +-- read administrative data in addition to users who can (say) update +-- select tables like the tribal access lists. +create or replace function admin_api_v1_1_0_functions.has_admin_data_access(perm TEXT) returns boolean +as $has_admin_data_access$ +DECLARE + uuid_header text; + key_exists boolean; + has_permission boolean; +BEGIN + SELECT admin_api_v1_1_0_functions.get_api_key_uuid() INTO uuid_header; + + SELECT + CASE WHEN EXISTS ( + SELECT uuid + FROM public.support_administrative_key_uuids aku + WHERE aku.uuid = uuid_header) + THEN 1::BOOLEAN + ELSE 0::BOOLEAN + END + INTO key_exists; + + SELECT + CASE WHEN EXISTS ( + SELECT permissions + FROM public.support_administrative_key_uuids aku + WHERE aku.uuid = uuid_header + AND aku.permissions like '%' || perm || '%') + THEN 1::BOOLEAN + ELSE 0::BOOLEAN + END + INTO has_permission; + + -- This log event is an INSERT. When called from a VIEW (a SELECT-only context), + -- a call to log_admin_api_event() fails. So, we'll RAISE INFO right here, so we can + -- see the resultse of access checks in the log. We might later comment this out if + -- it becomes too noisy. + RAISE INFO 'ADMIN_API has_access_check % % %', uuid_header, key_exists, has_permission; + + RETURN key_exists AND has_permission; +END; +$has_admin_data_access$ LANGUAGE plpgsql; + +-- Takes an email address and, if that address is not in the access table, +-- inserts it. If the address already exists, the insert is skipped. +-- +-- ### Example from REST client +-- POST http://localhost:3000/rpc/add_tribal_access_email +-- authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} +-- content-profile: admin_api_v1_1_0 +-- content-type: application/json +-- Prefer: params=single-object +-- // Not actually a key UUID. +-- X-Api-User-Id: 18ef0e72-8976-11ee-ad35-3f80b454d3cc +-- { +-- "email": "darth.vader@deathstar.org" +-- } +create or replace function admin_api_v1_1_0.add_tribal_access_email(params JSON) +returns BOOLEAN +as $add_tribal_access_email$ +DECLARE + already_exists INTEGER; + read_tribal_id INTEGER; +BEGIN + -- If the API user has insert permissions, give it a go + IF admin_api_v1_1_0_functions.has_admin_data_access('CREATE') + THEN + -- Are they already in the table? + SELECT count(up.email) + FROM public.users_userpermission as up + WHERE LOWER(email) = LOWER(params->>'email') INTO already_exists; + + -- If they are, we're going to exit. + IF already_exists <> 0 + THEN + RETURN 0; + END IF; + + -- Grab the permission ID that we need for the insert below. + -- We want the 'read-tribal' permission, which has a human-readable + -- slug. But, we need it's ID, because that is the PK. + SELECT up.id INTO read_tribal_id + FROM public.users_permission AS up + WHERE up.slug = 'read-tribal'; + + IF already_exists = 0 + THEN + -- Can we make the 1 not magic... do a select into. + INSERT INTO public.users_userpermission + (email, permission_id, user_id) + VALUES (LOWER(params->>'email'), read_tribal_id, null); + + RAISE INFO 'ADMIN_API add_tribal_access_email OK %', LOWER(params->>'email'); + RETURN admin_api_v1_1_0_functions.log_admin_api_event('tribal-access-email-added', + json_build_object('email', LOWER(params->>'email'))); + END IF; + ELSE + RETURN 0; + END IF; +end; +$add_tribal_access_email$ LANGUAGE plpgsql; + +-- Adds many email addresses. Calls `add_tribal_access_email` for each address. +-- +-- ### Example from REST client +-- POST http://localhost:3000/rpc/add_tribal_access_emails +-- authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} +-- content-profile: admin_api_v1_1_0 +-- content-type: application/json +-- Prefer: params=single-object +-- // Not actually a key UUID. +-- X-Api-User-Id: 18ef0e72-8976-11ee-ad35-3f80b454d3cc +-- { +-- "emails": [ +-- "darth.vader@deathstar.org", +-- "bob.darth.vader@deathstar.org", +-- "darthy.vader@deathstar.org", +-- "bob@deathstar.org" +-- ] +-- } +CREATE OR REPLACE FUNCTION admin_api_v1_1_0.add_tribal_access_emails(params JSON) +returns BOOLEAN +as $add_tribal_access_emails$ +DECLARE + ele TEXT; + em record; +BEGIN + IF admin_api_v1_1_0_functions.has_admin_data_access('CREATE') + THEN + -- This is a FOR loop over a JSON array in plPgSQL + FOR em IN (SELECT json_array_elements_text((params->>'emails')::JSON) ele) + LOOP + -- PERFORM is how to execute code that does not return anything. + -- If a SELECT was used here, the SQL compiler would complain. + PERFORM admin_api_v1_1_0.add_tribal_access_email(json_build_object('email', em.ele)::JSON); + END LOOP; + RETURN 1; + END IF; + RETURN 0; +END; +$add_tribal_access_emails$ LANGUAGE plpgsql; + +-- Removes the email. Will remove multiple rows. That shouldn't happen, but still. +-- +-- ### Example from REST client +-- POST http://localhost:3000/rpc/remove_tribal_access_email +-- authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} +-- content-profile: admin_api_v1_1_0 +-- content-type: application/json +-- Prefer: params=single-object +-- // Not actually a key UUID. +-- X-Api-User-Id: 18ef0e72-8976-11ee-ad35-3f80b454d3cc +-- { +-- "email": "darth.vader@deathstar.org" +-- } +CREATE OR REPLACE FUNCTION admin_api_v1_1_0.remove_tribal_access_email(params JSON) +returns BOOLEAN +as $remove_tribal_access_email$ +DECLARE + affected_rows INTEGER; +BEGIN + + IF admin_api_v1_1_0_functions.has_admin_data_access('DELETE') + THEN + -- Delete rows where the email address matches + DELETE FROM public.users_userpermission as up + WHERE LOWER(up.email) = LOWER(params->>'email'); + -- This is the Postgres way to find out how many rows + -- were affected by a DELETE. + GET DIAGNOSTICS affected_rows = ROW_COUNT; + -- If that is greater than zero, we were successful. + IF affected_rows > 0 + THEN + RETURN admin_api_v1_1_0_functions.log_admin_api_event('tribal-access-email-removed', + json_build_object('email', LOWER(params->>'email'))); + ELSE + RETURN 0; + END IF; + ELSE + -- If we did not have permission, consider it a failure. + RETURN 0; + END IF; +end; +$remove_tribal_access_email$ LANGUAGE plpgsql; + +-- Removes many email addresses. Calls `remove_tribal_access_email` for each address. +-- +-- ### Example from REST client +-- POST http://localhost:3000/rpc/remove_tribal_access_emails +-- authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} +-- content-profile: admin_api_v1_1_0 +-- content-type: application/json +-- Prefer: params=single-object +-- // Not actually a key UUID. +-- X-Api-User-Id: 18ef0e72-8976-11ee-ad35-3f80b454d3cc +-- { +-- "emails": [ +-- "darth.vader@deathstar.org", +-- "bob.darth.vader@deathstar.org", +-- "darthy.vader@deathstar.org", +-- "bob@deathstar.org" +-- ] +-- } +CREATE OR REPLACE FUNCTION admin_api_v1_1_0.remove_tribal_access_emails(params JSON) +returns BOOLEAN +as $remove_tribal_access_emails$ +DECLARE + ele TEXT; + em record; +BEGIN + IF admin_api_v1_1_0_functions.has_admin_data_access('DELETE') + THEN + FOR em IN (SELECT json_array_elements_text((params->>'emails')::JSON) ele) + LOOP + PERFORM admin_api_v1_1_0.remove_tribal_access_email(json_build_object('email', em.ele)::JSON); + END LOOP; + RETURN 1; + END IF; + RETURN 0; +END; +$remove_tribal_access_emails$ LANGUAGE plpgsql; + + + + +--The function below add_tribal_api_key_access adds read access to a tribal API for a specified email. +--It checks if the API user has read permissions. +--Otherwise, it adds the email with 'read-tribal' permission, logs the event, and returns true. + +CREATE OR REPLACE FUNCTION admin_api_v1_1_0.add_tribal_api_key_access(params JSON) +RETURNS JSON +AS $add_tribal_api_key_access$ +DECLARE + user_exists BOOLEAN; +BEGIN + -- If the API user has create permissions, we can proceed + IF admin_api_v1_1_0_functions.has_admin_data_access('CREATE') THEN + -- Check if the user with the given email + SELECT EXISTS ( + SELECT 1 + FROM public.dissemination_TribalApiAccessKeyIds + WHERE LOWER(email) = LOWER(params->>'email') + ) + INTO user_exists; + + -- If the user already exists, it means they have access. + -- For purposes of this function, lets call that "succses", and return true. + IF user_exists THEN + RAISE INFO 'ADMIN_API add_tribal_api_key_access ALREADY_EXISTS %', LOWER(params->>'email'); + RETURN json_build_object( + 'result', 'success', + 'message', 'User with this key already exists')::JSON; + + END IF; + + -- If the user does not exist, add a new record + INSERT INTO public.dissemination_TribalApiAccessKeyIds (email, key_id, date_added) + VALUES (LOWER(params->>'email'), params->>'key_id', CURRENT_TIMESTAMP); + RAISE INFO 'ADMIN_API add_tribal_api_key_access ACCESS_GRANTED % %', LOWER(params->>'email'), params->>'key_id'; + RETURN json_build_object( + 'result', 'success', + 'message', 'User access granted')::JSON; + ELSE + -- If the user does not have CREATE permissions, then we should return a message to that effect. + -- It is a permissions error, but still, we need to know this failed. + RAISE INFO 'ADMIN_API add_tribal_api_key_access ADMIN_LACKS_CREATE'; + RETURN json_build_object( + 'result', 'failure', + 'message', 'Admin user lacks CREATE permissions')::JSON; + END IF; + + -- Return false by default. + RAISE INFO 'ADMIN_API add_tribal_api_key_access WAT %', LOWER(params->>'email'); + RETURN json_build_object( + 'result', 'failure', + 'message', 'Unknown error in access addition')::JSON; +END; +$add_tribal_api_key_access$ LANGUAGE plpgsql; + +-- The function below removes tribal API key access for a specified email. +-- It checks if the API user has read permissions. +-- If the email exists in the database with 'read-tribal' permission, it removes the entry, logs the removal event, and returns true. +-- If the email doesn't exist or the user lacks proper permissions, the function returns false. + +CREATE OR REPLACE FUNCTION admin_api_v1_1_0.remove_tribal_api_key_access(params JSON) +RETURNS JSON +AS $remove_tribal_api_key_access$ +DECLARE + user_exists BOOLEAN; +BEGIN + -- If the API user has read permissions, give it a go + IF admin_api_v1_1_0_functions.has_admin_data_access('DELETE') THEN + -- Check if the user with the given email exists + SELECT EXISTS ( + SELECT 1 + FROM public.dissemination_TribalApiAccessKeyIds + WHERE LOWER(email) = LOWER(params->>'email') + ) + INTO user_exists; + + -- If the user exists, remove the record + IF user_exists THEN + DELETE FROM public.dissemination_TribalApiAccessKeyIds + WHERE LOWER(email) = LOWER(params->>'email'); + RAISE INFO 'ADMIN_API remove_tribal_api_key_access ACCESS_REMOVED %', LOWER(params->>'email'); + RETURN json_build_object( + 'result', 'success', + 'message', 'Removed record')::JSON; + ELSE + RAISE INFO 'ADMIN_API remove_tribal_api_key_access DID_NOT_EXIST %', LOWER(params->>'email'); + RETURN json_build_object( + 'result', 'failure', + 'message', 'User did not exist in table')::JSON; + END IF; + ELSE + RAISE INFO 'ADMIN_API remove_tribal_api_key_access ADMIN_LACKS_DELETE'; + RETURN json_build_object( + 'result', 'failure', + 'message', 'Admin user lacks DELETE permissions')::JSON; -- Return false if the API user doesn't have read permissions + END IF; + RAISE INFO 'ADMIN_API add_tribal_api_key_access WAT %', LOWER(params->>'email'); + RETURN json_build_object( + 'result', 'failure', + 'message', 'Uknown error in access removal')::JSON; +END; +$remove_tribal_api_key_access$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION admin_api_v1_1_0.request_file_access( + report_id TEXT +) RETURNS JSON LANGUAGE plpgsql AS +$$ +DECLARE + v_uuid_header TEXT; + v_access_uuid VARCHAR(200); + v_key_exists BOOLEAN; + v_key_added_date DATE; +BEGIN + + SELECT admin_api_v1_1_0_functions.get_api_key_uuid() INTO v_uuid_header; + + -- Check if the provided API key exists in public.dissemination_TribalApiAccessKeyIds + SELECT + EXISTS( + SELECT 1 + FROM public.dissemination_TribalApiAccessKeyIds + WHERE key_id = v_uuid_header + ) INTO v_key_exists; + + + -- Get the added date of the key from public.dissemination_TribalApiAccessKeyIds + SELECT date_added + INTO v_key_added_date + FROM public.dissemination_TribalApiAccessKeyIds + WHERE key_id = v_uuid_header; + + + -- Check if the key is less than 6 months old + IF v_uuid_header IS NOT NULL AND v_key_exists AND v_key_added_date >= CURRENT_DATE - INTERVAL '6 months' THEN + -- Generate UUID (using PostgreSQL's gen_random_uuid function) + SELECT gen_random_uuid() INTO v_access_uuid; + + -- Inserting data into the one_time_access table + INSERT INTO public.dissemination_onetimeaccess (uuid, api_key_id, timestamp, report_id) + VALUES (v_access_uuid::UUID, v_uuid_header, CURRENT_TIMESTAMP, report_id); + + -- Return the UUID to the user + RETURN json_build_object('access_uuid', v_access_uuid); + ELSE + -- Return an error for unauthorized access + RETURN json_build_object('error', 'Unauthorized access or key older than 6 months')::JSON; + END IF; +END; +$$; + + + +commit; + +NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/api/admin_api_v1_1_0/create_schema.sql b/backend/dissemination/sql/api/admin_api_v1_1_0/create_schema.sql new file mode 100644 index 0000000000..25f913689d --- /dev/null +++ b/backend/dissemination/sql/api/admin_api_v1_1_0/create_schema.sql @@ -0,0 +1,56 @@ +begin; + +do +$$ +begin + DROP SCHEMA IF EXISTS admin_api_v1_1_0 CASCADE; + DROP SCHEMA IF EXISTS admin_api_v1_1_0_functions CASCADE; + + if not exists (select schema_name from information_schema.schemata where schema_name = 'admin_api_v1_1_0') then + create schema admin_api_v1_1_0; + create schema admin_api_v1_1_0_functions; + + grant usage on schema admin_api_v1_1_0_functions to api_fac_gov; + + -- Grant access to tables and views + alter default privileges + in schema admin_api_v1_1_0 + grant select + -- this includes views + on tables + to api_fac_gov; + + -- Grant access to sequences, if we have them + grant usage on schema admin_api_v1_1_0 to api_fac_gov; + grant select, usage on all sequences in schema admin_api_v1_1_0 to api_fac_gov; + alter default privileges + in schema admin_api_v1_1_0 + grant select, usage + on sequences + to api_fac_gov; + + -- The admin API needs to be able to write user permissions. + -- This is so we can add and remove people who will have tribal data access + -- via the administrative API. + GRANT INSERT, SELECT, DELETE on public.users_userpermission to api_fac_gov; + -- We need to be able to look up slugs and turn them into permission IDs. + GRANT SELECT on public.users_permission to api_fac_gov; + -- It also needs to be able to log events. + GRANT INSERT on public.support_adminapievent to api_fac_gov; + -- And, it wants to read the UUIDs of administrative keys + GRANT SELECT ON public.support_administrative_key_uuids TO api_fac_gov; + -- We want to see data in flight as admins. + GRANT SELECT ON public.audit_singleauditchecklist TO api_fac_gov; + + GRANT INSERT, SELECT, DELETE on public.dissemination_tribalapiaccesskeyids to api_fac_gov; + GRANT INSERT on public.dissemination_onetimeaccess to api_fac_gov; + end if; +end +$$ +; + +commit; + +notify pgrst, + 'reload schema'; + diff --git a/backend/dissemination/sql/api/admin_api_v1_1_0/create_views.sql b/backend/dissemination/sql/api/admin_api_v1_1_0/create_views.sql new file mode 100644 index 0000000000..04b747662f --- /dev/null +++ b/backend/dissemination/sql/api/admin_api_v1_1_0/create_views.sql @@ -0,0 +1,92 @@ + +begin; + + +--------------------------------------- +-- accesses +--------------------------------------- +-- public.audit_access definition + +-- Drop table + +-- DROP TABLE public.audit_access; + +CREATE OR REPLACE VIEW admin_api_v1_1_0.audit_access AS + SELECT + aa.role, + aa.fullname, + aa.email, + aa.sac_id, + aa.user_id + FROM + public.audit_access aa + WHERE + admin_api_v1_1_0_functions.has_admin_data_access('READ') + ORDER BY aa.id +; + +CREATE OR REPLACE VIEW admin_api_v1_1_0.singleauditchecklist AS + SELECT + sac.id, + sac.date_created, + sac.submission_status, + sac.data_source, + sac.transition_name, + sac.transition_date, + sac.report_id, + sac.audit_type, + sac.general_information, + sac.audit_information, + sac.federal_awards, + sac.corrective_action_plan, + sac.findings_text, + sac.findings_uniform_guidance, + sac.additional_ueis, + sac.additional_eins, + sac.secondary_auditors, + sac.notes_to_sefa, + sac.auditor_certification, + sac.auditee_certification, + sac.tribal_data_consent, + sac.cognizant_agency, + sac.oversight_agency, + sac.submitted_by_id + from + public.audit_singleauditchecklist sac + where + admin_api_v1_1_0_functions.has_admin_data_access('READ') + order by sac.id +; + +CREATE OR REPLACE VIEW admin_api_v1_1_0.tribal_access AS + SELECT + uup.email, + up.slug as permission + FROM + users_userpermission uup, + users_permission up + WHERE + (uup.permission_id = up.id) + AND (up.slug = 'read-tribal') + AND admin_api_v1_1_0_functions.has_admin_data_access('READ') + ORDER BY uup.id +; + +CREATE OR REPLACE VIEW admin_api_v1_1_0.admin_api_events AS + SELECT + ae.timestamp, + ae.api_key_uuid, + ae.event, + ae.event_data + FROM + public.support_adminapievent ae + WHERE + admin_api_v1_1_0_functions.has_admin_data_access('READ') + ORDER BY ae.id +; + + +commit; + +notify pgrst, + 'reload schema'; diff --git a/backend/dissemination/sql/api/admin_api_v1_1_0/drop_schema.sql b/backend/dissemination/sql/api/admin_api_v1_1_0/drop_schema.sql new file mode 100644 index 0000000000..b0d4cc1109 --- /dev/null +++ b/backend/dissemination/sql/api/admin_api_v1_1_0/drop_schema.sql @@ -0,0 +1,9 @@ + +begin; + +DROP SCHEMA IF EXISTS admin_api_v1_1_0 CASCADE; + +commit; + +notify pgrst, + 'reload schema'; diff --git a/backend/dissemination/sql/api/admin_api_v1_1_0/drop_views.sql b/backend/dissemination/sql/api/admin_api_v1_1_0/drop_views.sql new file mode 100644 index 0000000000..41236e55d7 --- /dev/null +++ b/backend/dissemination/sql/api/admin_api_v1_1_0/drop_views.sql @@ -0,0 +1,8 @@ +begin; + + drop view if exists admin_api_v1_1_0.audit_access; + +commit; + +notify pgrst, + 'reload schema'; diff --git a/backend/dissemination/sql/api/admin_api_v1_1_0/test.rest b/backend/dissemination/sql/api/admin_api_v1_1_0/test.rest new file mode 100644 index 0000000000..16e6ab054c --- /dev/null +++ b/backend/dissemination/sql/api/admin_api_v1_1_0/test.rest @@ -0,0 +1,114 @@ +@api_user_id = 61ba59b2-f545-4c2f-9b24-9655c706a06c +# @api_user_id = 1e2845a0-c844-4a6f-84ac-f398b58ce7c9 +@scheme = http +@api_url = localhost:3000 +# @scheme = https +# @api_url = api-preview.fac.gov + +### +GET {{scheme}}://{{api_url}}/general?limit=1 +// Have to use the JWT when testing locally. +authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} +x-api-key: {{$processEnv API_GOV_KEY}} + + +### +GET {{scheme}}://{{api_url}}/audit_access +// Have to use the JWT and API UUID when testing locally. +authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} +x-api-user-id: {{api_user_id}} +x-api-key: {{$processEnv API_GOV_KEY}} +accept-profile: admin_api_v1_1_0 + +### +GET {{scheme}}://{{api_url}}/singleauditchecklist +authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} +x-api-user-id: {{api_user_id}} +accept-profile: admin_api_v1_1_0 +x-api-key: {{$processEnv API_GOV_KEY}} + +### +// This should list everything using a view. +GET {{scheme}}://{{api_url}}/tribal_access +authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} +x-api-user-id: {{api_user_id}} +accept-profile: admin_api_v1_1_0 +x-api-key: {{$processEnv API_GOV_KEY}} + + +### +// Call the RPC to add access +POST {{scheme}}://{{api_url}}/rpc/add_tribal_access_email +authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} +x-api-user-id: {{api_user_id}} +content-profile: admin_api_v1_1_0 +content-type: application/json +Prefer: params=single-object +x-api-key: {{$processEnv API_GOV_KEY}} + +{ + "email": "darth.vader@deathstar.gsa.gov" +} + +### +// Call the RPC to remove access +POST {{scheme}}://{{api_url}}/rpc/add_tribal_access_emails +authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} +x-api-user-id: {{api_user_id}} +content-profile: admin_api_v1_1_0 +content-type: application/json +Prefer: params=single-object +x-api-key: {{$processEnv API_GOV_KEY}} + +{ + "emails": [ + "darth.vader@deathstar.gsa.gov", + "bob.darth.vader@deathstar.gsa.gov", + "darthy.vader@deathstar.gsa.gov", + "bob@deathstar.gsa.gov" + ] +} + +### +// Call the RPC to remove access +POST {{scheme}}://{{api_url}}/rpc/remove_tribal_access_email +authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} +x-api-user-id: {{api_user_id}} +content-profile: admin_api_v1_1_0 +content-type: application/json +Prefer: params=single-object +x-api-key: {{$processEnv API_GOV_KEY}} + +{ + "email": "darth.vader@deathstar.gsa.gov" +} + + +### +// Call the RPC to remove access +POST {{scheme}}://{{api_url}}/rpc/remove_tribal_access_emails +authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} +x-api-user-id: {{api_user_id}} +content-profile: admin_api_v1_1_0 +content-type: application/json +Prefer: params=single-object +x-api-key: {{$processEnv API_GOV_KEY}} + +{ + "emails": [ + "darth.vader@deathstar.gsa.gov", + "bob.darth.vader@deathstar.gsa.gov", + "darthy.vader@deathstar.gsa.gov", + "bob@deathstar.gsa.gov" + ] +} + + +### +// This should list everything using a view. +GET {{scheme}}://{{api_url}}/admin_api_events +authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} +x-api-user-id: {{api_user_id}} +accept-profile: admin_api_v1_1_0 +x-api-key: {{$processEnv API_GOV_KEY}} + diff --git a/backend/dissemination/sql/api/admin_api_v1_1_1/base.sql b/backend/dissemination/sql/api/admin_api_v1_1_1/base.sql new file mode 100644 index 0000000000..dedabe0cb7 --- /dev/null +++ b/backend/dissemination/sql/api/admin_api_v1_1_1/base.sql @@ -0,0 +1,29 @@ +DO +$do$ +BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = 'authenticator') THEN + RAISE NOTICE 'Role "authenticator" already exists. Skipping.'; + ELSE + CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; + END IF; +END +$do$; + +DO +$do$ +BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = 'api_fac_gov') THEN + RAISE NOTICE 'Role "api_fac_gov" already exists. Skipping.'; + ELSE + CREATE ROLE api_fac_gov NOLOGIN; + END IF; +END +$do$; + +GRANT api_fac_gov TO authenticator; + +NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/api/admin_api_v1_1_1/create_access_tables.sql b/backend/dissemination/sql/api/admin_api_v1_1_1/create_access_tables.sql new file mode 100644 index 0000000000..18db91956d --- /dev/null +++ b/backend/dissemination/sql/api/admin_api_v1_1_1/create_access_tables.sql @@ -0,0 +1,43 @@ +-- This is explicitly not a Django managed table. +-- In order to have an administrative key added, +-- it must be added via a Github commit, and a PR +-- must be performed to merge the key into the tree. + +-- This is because administrative keys can read/write +-- to some tables in the database. They can read internal and +-- in-flight data. + +DROP TABLE IF EXISTS support_administrative_key_uuids; + +CREATE TABLE support_administrative_key_uuids + ( + id BIGSERIAL PRIMARY KEY, + email TEXT, + uuid TEXT, + permissions TEXT, + added DATE + ); + +INSERT INTO support_administrative_key_uuids + (email, uuid, permissions, added) + VALUES + ( + 'matthew.jadud@gsa.gov', + '61ba59b2-f545-4c2f-9b24-9655c706a06c', + 'CREATE,READ,DELETE', + '2023-12-04' + ), + ( + 'daniel.swick@gsa.gov', + 'b6e08808-ecb2-4b6a-b928-46d4205497ff', + 'CREATE,READ,DELETE', + '2023-12-08' + ), + ( + 'fac-gov-test-users+api-tester-admin@gsa.gov', + 'dd60c3f9-053d-4d82-a309-c89da53559f4', + 'CREATE,READ,DELETE', + '2024-07-10' + ) + ; + diff --git a/backend/dissemination/sql/api/admin_api_v1_1_1/create_functions.sql b/backend/dissemination/sql/api/admin_api_v1_1_1/create_functions.sql new file mode 100644 index 0000000000..77b2b2a303 --- /dev/null +++ b/backend/dissemination/sql/api/admin_api_v1_1_1/create_functions.sql @@ -0,0 +1,389 @@ +-- WARNING +-- Under PostgreSQL 12, the functions below work. +-- Under PostgreSQL 14, these will break. +-- +-- Note the differences: +-- +-- raise info 'Works under PostgreSQL 12'; +-- raise info 'request.header.x-magic %', (SELECT current_setting('request.header.x-magic', true)); +-- raise info 'request.jwt.claim.expires %', (SELECT current_setting('request.jwt.claim.expires', true)); +-- raise info 'Works under PostgreSQL 14'; +-- raise info 'request.headers::json->>x-magic %', (SELECT current_setting('request.headers', true)::json->>'x-magic'); +-- raise info 'request.jwt.claims::json->expires %', (SELECT current_setting('request.jwt.claims', true)::json->>'expires'); +-- +-- To quote the work of Dav Pilkey, "remember this now." + +begin; + + +CREATE OR REPLACE FUNCTION admin_api_v1_1_1_functions.get_header(item text) RETURNS text + AS $get_header$ + declare res text; + begin + SELECT (current_setting('request.headers', true)::json)->>item into res; + return res; + end; +$get_header$ LANGUAGE plpgsql; + +create or replace function admin_api_v1_1_1_functions.get_api_key_uuid() returns TEXT +as $gaku$ +declare uuid text; +begin + select admin_api_v1_1_1_functions.get_header('x-api-user-id') into uuid; + return uuid; +end; +$gaku$ LANGUAGE plpgsql; + +-- log_api_event +-- Maintain an internal table of administrative API events. +-- Also RAISE INFO so that NR gets a copy. +create or replace function admin_api_v1_1_1_functions.log_admin_api_event(event TEXT, meta JSON) +returns boolean +as $log_admin_api_event$ +DECLARE + uuid_header text; +BEGIN + SELECT admin_api_v1_1_1_functions.get_api_key_uuid() INTO uuid_header; + + INSERT INTO public.support_adminapievent + (api_key_uuid, event, event_data, "timestamp") + VALUES (uuid_header, event, meta, NOW()); + + RAISE INFO 'ADMIN_API % % %', uuid_header, event, meta; + RETURN 1; +END; +$log_admin_api_event$ LANGUAGE plpgsql; + + +-- has_admin_data_access :: permission -> bool +-- The permissions (insert, select, delete) allow us to have users who can +-- read administrative data in addition to users who can (say) update +-- select tables like the tribal access lists. +create or replace function admin_api_v1_1_1_functions.has_admin_data_access(perm TEXT) returns boolean +as $has_admin_data_access$ +DECLARE + uuid_header text; + key_exists boolean; + has_permission boolean; +BEGIN + SELECT admin_api_v1_1_1_functions.get_api_key_uuid() INTO uuid_header; + + SELECT + CASE WHEN EXISTS ( + SELECT uuid + FROM public.support_administrative_key_uuids aku + WHERE aku.uuid = uuid_header) + THEN 1::BOOLEAN + ELSE 0::BOOLEAN + END + INTO key_exists; + + SELECT + CASE WHEN EXISTS ( + SELECT permissions + FROM public.support_administrative_key_uuids aku + WHERE aku.uuid = uuid_header + AND aku.permissions like '%' || perm || '%') + THEN 1::BOOLEAN + ELSE 0::BOOLEAN + END + INTO has_permission; + + -- This log event is an INSERT. When called from a VIEW (a SELECT-only context), + -- a call to log_admin_api_event() fails. So, we'll RAISE INFO right here, so we can + -- see the resultse of access checks in the log. We might later comment this out if + -- it becomes too noisy. + RAISE INFO 'ADMIN_API has_access_check % % %', uuid_header, key_exists, has_permission; + + RETURN key_exists AND has_permission; +END; +$has_admin_data_access$ LANGUAGE plpgsql; + +-- Takes an email address and, if that address is not in the access table, +-- inserts it. If the address already exists, the insert is skipped. +-- +-- ### Example from REST client +-- POST http://localhost:3000/rpc/add_tribal_access_email +-- authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} +-- content-profile: admin_api_v1_1_1 +-- content-type: application/json +-- Prefer: params=single-object +-- // Not actually a key UUID. +-- X-Api-User-Id: 18ef0e72-8976-11ee-ad35-3f80b454d3cc +-- { +-- "email": "darth.vader@deathstar.org" +-- } +create or replace function admin_api_v1_1_1.add_tribal_access_email(params JSON) +returns BOOLEAN +as $add_tribal_access_email$ +DECLARE + already_exists INTEGER; + read_tribal_id INTEGER; +BEGIN + -- If the API user has insert permissions, give it a go + IF admin_api_v1_1_1_functions.has_admin_data_access('CREATE') + THEN + -- Are they already in the table? + SELECT count(up.email) + FROM public.users_userpermission as up + WHERE LOWER(email) = LOWER(params->>'email') INTO already_exists; + + -- If they are, we're going to exit. + IF already_exists <> 0 + THEN + RETURN 0; + END IF; + + -- Grab the permission ID that we need for the insert below. + -- We want the 'read-tribal' permission, which has a human-readable + -- slug. But, we need it's ID, because that is the PK. + SELECT up.id INTO read_tribal_id + FROM public.users_permission AS up + WHERE up.slug = 'read-tribal'; + + IF already_exists = 0 + THEN + -- Can we make the 1 not magic... do a select into. + INSERT INTO public.users_userpermission + (email, permission_id, user_id) + VALUES (LOWER(params->>'email'), read_tribal_id, null); + + RAISE INFO 'ADMIN_API add_tribal_access_email OK %', LOWER(params->>'email'); + RETURN admin_api_v1_1_1_functions.log_admin_api_event('tribal-access-email-added', + json_build_object('email', LOWER(params->>'email'))); + END IF; + ELSE + RETURN 0; + END IF; +end; +$add_tribal_access_email$ LANGUAGE plpgsql; + +-- Adds many email addresses. Calls `add_tribal_access_email` for each address. +-- +-- ### Example from REST client +-- POST http://localhost:3000/rpc/add_tribal_access_emails +-- authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} +-- content-profile: admin_api_v1_1_1 +-- content-type: application/json +-- Prefer: params=single-object +-- // Not actually a key UUID. +-- X-Api-User-Id: 18ef0e72-8976-11ee-ad35-3f80b454d3cc +-- { +-- "emails": [ +-- "darth.vader@deathstar.org", +-- "bob.darth.vader@deathstar.org", +-- "darthy.vader@deathstar.org", +-- "bob@deathstar.org" +-- ] +-- } +CREATE OR REPLACE FUNCTION admin_api_v1_1_1.add_tribal_access_emails(params JSON) +returns BOOLEAN +as $add_tribal_access_emails$ +DECLARE + ele TEXT; + em record; +BEGIN + IF admin_api_v1_1_1_functions.has_admin_data_access('CREATE') + THEN + -- This is a FOR loop over a JSON array in plPgSQL + FOR em IN (SELECT json_array_elements_text((params->>'emails')::JSON) ele) + LOOP + -- PERFORM is how to execute code that does not return anything. + -- If a SELECT was used here, the SQL compiler would complain. + PERFORM admin_api_v1_1_1.add_tribal_access_email(json_build_object('email', em.ele)::JSON); + END LOOP; + RETURN 1; + END IF; + RETURN 0; +END; +$add_tribal_access_emails$ LANGUAGE plpgsql; + +-- Removes the email. Will remove multiple rows. That shouldn't happen, but still. +-- +-- ### Example from REST client +-- POST http://localhost:3000/rpc/remove_tribal_access_email +-- authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} +-- content-profile: admin_api_v1_1_1 +-- content-type: application/json +-- Prefer: params=single-object +-- // Not actually a key UUID. +-- X-Api-User-Id: 18ef0e72-8976-11ee-ad35-3f80b454d3cc +-- { +-- "email": "darth.vader@deathstar.org" +-- } +CREATE OR REPLACE FUNCTION admin_api_v1_1_1.remove_tribal_access_email(params JSON) +returns BOOLEAN +as $remove_tribal_access_email$ +DECLARE + affected_rows INTEGER; +BEGIN + + IF admin_api_v1_1_1_functions.has_admin_data_access('DELETE') + THEN + -- Delete rows where the email address matches + DELETE FROM public.users_userpermission as up + WHERE LOWER(up.email) = LOWER(params->>'email'); + -- This is the Postgres way to find out how many rows + -- were affected by a DELETE. + GET DIAGNOSTICS affected_rows = ROW_COUNT; + -- If that is greater than zero, we were successful. + IF affected_rows > 0 + THEN + RETURN admin_api_v1_1_1_functions.log_admin_api_event('tribal-access-email-removed', + json_build_object('email', LOWER(params->>'email'))); + ELSE + RETURN 0; + END IF; + ELSE + -- If we did not have permission, consider it a failure. + RETURN 0; + END IF; +end; +$remove_tribal_access_email$ LANGUAGE plpgsql; + +-- Removes many email addresses. Calls `remove_tribal_access_email` for each address. +-- +-- ### Example from REST client +-- POST http://localhost:3000/rpc/remove_tribal_access_emails +-- authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} +-- content-profile: admin_api_v1_1_1 +-- content-type: application/json +-- Prefer: params=single-object +-- // Not actually a key UUID. +-- X-Api-User-Id: 18ef0e72-8976-11ee-ad35-3f80b454d3cc +-- { +-- "emails": [ +-- "darth.vader@deathstar.org", +-- "bob.darth.vader@deathstar.org", +-- "darthy.vader@deathstar.org", +-- "bob@deathstar.org" +-- ] +-- } +CREATE OR REPLACE FUNCTION admin_api_v1_1_1.remove_tribal_access_emails(params JSON) +returns BOOLEAN +as $remove_tribal_access_emails$ +DECLARE + ele TEXT; + em record; +BEGIN + IF admin_api_v1_1_1_functions.has_admin_data_access('DELETE') + THEN + FOR em IN (SELECT json_array_elements_text((params->>'emails')::JSON) ele) + LOOP + PERFORM admin_api_v1_1_1.remove_tribal_access_email(json_build_object('email', em.ele)::JSON); + END LOOP; + RETURN 1; + END IF; + RETURN 0; +END; +$remove_tribal_access_emails$ LANGUAGE plpgsql; + + + + +--The function below add_tribal_api_key_access adds read access to a tribal API for a specified email. +--It checks if the API user has read permissions. +--Otherwise, it adds the email with 'read-tribal' permission, logs the event, and returns true. + +CREATE OR REPLACE FUNCTION admin_api_v1_1_1.add_tribal_api_key_access(params JSON) +RETURNS JSON +AS $add_tribal_api_key_access$ +DECLARE + user_exists BOOLEAN; +BEGIN + -- If the API user has create permissions, we can proceed + IF admin_api_v1_1_1_functions.has_admin_data_access('CREATE') THEN + -- Check if the user with the given email + SELECT EXISTS ( + SELECT 1 + FROM public.dissemination_TribalApiAccessKeyIds + WHERE LOWER(email) = LOWER(params->>'email') + ) + INTO user_exists; + + -- If the user already exists, it means they have access. + -- For purposes of this function, lets call that "succses", and return true. + IF user_exists THEN + RAISE INFO 'ADMIN_API add_tribal_api_key_access ALREADY_EXISTS %', LOWER(params->>'email'); + RETURN json_build_object( + 'result', 'success', + 'message', 'User with this key already exists')::JSON; + + END IF; + + -- If the user does not exist, add a new record + INSERT INTO public.dissemination_TribalApiAccessKeyIds (email, key_id, date_added) + VALUES (LOWER(params->>'email'), params->>'key_id', CURRENT_TIMESTAMP); + RAISE INFO 'ADMIN_API add_tribal_api_key_access ACCESS_GRANTED % %', LOWER(params->>'email'), params->>'key_id'; + RETURN json_build_object( + 'result', 'success', + 'message', 'User access granted')::JSON; + ELSE + -- If the user does not have CREATE permissions, then we should return a message to that effect. + -- It is a permissions error, but still, we need to know this failed. + RAISE INFO 'ADMIN_API add_tribal_api_key_access ADMIN_LACKS_CREATE'; + RETURN json_build_object( + 'result', 'failure', + 'message', 'Admin user lacks CREATE permissions')::JSON; + END IF; + + -- Return false by default. + RAISE INFO 'ADMIN_API add_tribal_api_key_access WAT %', LOWER(params->>'email'); + RETURN json_build_object( + 'result', 'failure', + 'message', 'Unknown error in access addition')::JSON; +END; +$add_tribal_api_key_access$ LANGUAGE plpgsql; + +-- The function below removes tribal API key access for a specified email. +-- It checks if the API user has read permissions. +-- If the email exists in the database with 'read-tribal' permission, it removes the entry, logs the removal event, and returns true. +-- If the email doesn't exist or the user lacks proper permissions, the function returns false. + +CREATE OR REPLACE FUNCTION admin_api_v1_1_1.remove_tribal_api_key_access(params JSON) +RETURNS JSON +AS $remove_tribal_api_key_access$ +DECLARE + user_exists BOOLEAN; +BEGIN + -- If the API user has read permissions, give it a go + IF admin_api_v1_1_1_functions.has_admin_data_access('DELETE') THEN + -- Check if the user with the given email exists + SELECT EXISTS ( + SELECT 1 + FROM public.dissemination_TribalApiAccessKeyIds + WHERE LOWER(email) = LOWER(params->>'email') + ) + INTO user_exists; + + -- If the user exists, remove the record + IF user_exists THEN + DELETE FROM public.dissemination_TribalApiAccessKeyIds + WHERE LOWER(email) = LOWER(params->>'email'); + RAISE INFO 'ADMIN_API remove_tribal_api_key_access ACCESS_REMOVED %', LOWER(params->>'email'); + RETURN json_build_object( + 'result', 'success', + 'message', 'Removed record')::JSON; + ELSE + RAISE INFO 'ADMIN_API remove_tribal_api_key_access DID_NOT_EXIST %', LOWER(params->>'email'); + RETURN json_build_object( + 'result', 'failure', + 'message', 'User did not exist in table')::JSON; + END IF; + ELSE + RAISE INFO 'ADMIN_API remove_tribal_api_key_access ADMIN_LACKS_DELETE'; + RETURN json_build_object( + 'result', 'failure', + 'message', 'Admin user lacks DELETE permissions')::JSON; -- Return false if the API user doesn't have read permissions + END IF; + RAISE INFO 'ADMIN_API add_tribal_api_key_access WAT %', LOWER(params->>'email'); + RETURN json_build_object( + 'result', 'failure', + 'message', 'Uknown error in access removal')::JSON; +END; +$remove_tribal_api_key_access$ LANGUAGE plpgsql; + + +commit; + +NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/api/admin_api_v1_1_1/create_schema.sql b/backend/dissemination/sql/api/admin_api_v1_1_1/create_schema.sql new file mode 100644 index 0000000000..9eff082bfe --- /dev/null +++ b/backend/dissemination/sql/api/admin_api_v1_1_1/create_schema.sql @@ -0,0 +1,56 @@ +begin; + +do +$$ +begin + DROP SCHEMA IF EXISTS admin_api_v1_1_1 CASCADE; + DROP SCHEMA IF EXISTS admin_api_v1_1_1_functions CASCADE; + + if not exists (select schema_name from information_schema.schemata where schema_name = 'admin_api_v1_1_1') then + create schema admin_api_v1_1_1; + create schema admin_api_v1_1_1_functions; + + grant usage on schema admin_api_v1_1_1_functions to api_fac_gov; + + -- Grant access to tables and views + alter default privileges + in schema admin_api_v1_1_1 + grant select + -- this includes views + on tables + to api_fac_gov; + + -- Grant access to sequences, if we have them + grant usage on schema admin_api_v1_1_1 to api_fac_gov; + grant select, usage on all sequences in schema admin_api_v1_1_1 to api_fac_gov; + alter default privileges + in schema admin_api_v1_1_1 + grant select, usage + on sequences + to api_fac_gov; + + -- The admin API needs to be able to write user permissions. + -- This is so we can add and remove people who will have tribal data access + -- via the administrative API. + GRANT INSERT, SELECT, DELETE on public.users_userpermission to api_fac_gov; + -- We need to be able to look up slugs and turn them into permission IDs. + GRANT SELECT on public.users_permission to api_fac_gov; + -- It also needs to be able to log events. + GRANT INSERT on public.support_adminapievent to api_fac_gov; + -- And, it wants to read the UUIDs of administrative keys + GRANT SELECT ON public.support_administrative_key_uuids TO api_fac_gov; + -- We want to see data in flight as admins. + GRANT SELECT ON public.audit_singleauditchecklist TO api_fac_gov; + + GRANT INSERT, SELECT, DELETE on public.dissemination_tribalapiaccesskeyids to api_fac_gov; + GRANT INSERT on public.dissemination_onetimeaccess to api_fac_gov; + end if; +end +$$ +; + +commit; + +notify pgrst, + 'reload schema'; + diff --git a/backend/dissemination/sql/api/admin_api_v1_1_1/create_views.sql b/backend/dissemination/sql/api/admin_api_v1_1_1/create_views.sql new file mode 100644 index 0000000000..6c84abf4d8 --- /dev/null +++ b/backend/dissemination/sql/api/admin_api_v1_1_1/create_views.sql @@ -0,0 +1,92 @@ + +begin; + + +--------------------------------------- +-- accesses +--------------------------------------- +-- public.audit_access definition + +-- Drop table + +-- DROP TABLE public.audit_access; + +CREATE OR REPLACE VIEW admin_api_v1_1_1.audit_access AS + SELECT + aa.role, + aa.fullname, + aa.email, + aa.sac_id, + aa.user_id + FROM + public.audit_access aa + WHERE + admin_api_v1_1_1_functions.has_admin_data_access('READ') + ORDER BY aa.id +; + +CREATE OR REPLACE VIEW admin_api_v1_1_1.singleauditchecklist AS + SELECT + sac.id, + sac.date_created, + sac.submission_status, + sac.data_source, + sac.transition_name, + sac.transition_date, + sac.report_id, + sac.audit_type, + sac.general_information, + sac.audit_information, + sac.federal_awards, + sac.corrective_action_plan, + sac.findings_text, + sac.findings_uniform_guidance, + sac.additional_ueis, + sac.additional_eins, + sac.secondary_auditors, + sac.notes_to_sefa, + sac.auditor_certification, + sac.auditee_certification, + sac.tribal_data_consent, + sac.cognizant_agency, + sac.oversight_agency, + sac.submitted_by_id + from + public.audit_singleauditchecklist sac + where + admin_api_v1_1_1_functions.has_admin_data_access('READ') + order by sac.id +; + +CREATE OR REPLACE VIEW admin_api_v1_1_1.tribal_access AS + SELECT + uup.email, + up.slug as permission + FROM + users_userpermission uup, + users_permission up + WHERE + (uup.permission_id = up.id) + AND (up.slug = 'read-tribal') + AND admin_api_v1_1_1_functions.has_admin_data_access('READ') + ORDER BY uup.id +; + +CREATE OR REPLACE VIEW admin_api_v1_1_1.admin_api_events AS + SELECT + ae.timestamp, + ae.api_key_uuid, + ae.event, + ae.event_data + FROM + public.support_adminapievent ae + WHERE + admin_api_v1_1_1_functions.has_admin_data_access('READ') + ORDER BY ae.id +; + + +commit; + +notify pgrst, + 'reload schema'; diff --git a/backend/dissemination/sql/api/admin_api_v1_1_1/drop_schema.sql b/backend/dissemination/sql/api/admin_api_v1_1_1/drop_schema.sql new file mode 100644 index 0000000000..0f53c84a81 --- /dev/null +++ b/backend/dissemination/sql/api/admin_api_v1_1_1/drop_schema.sql @@ -0,0 +1,9 @@ + +begin; + +DROP SCHEMA IF EXISTS admin_api_v1_1_1 CASCADE; + +commit; + +notify pgrst, + 'reload schema'; diff --git a/backend/dissemination/sql/api/admin_api_v1_1_1/drop_views.sql b/backend/dissemination/sql/api/admin_api_v1_1_1/drop_views.sql new file mode 100644 index 0000000000..6185d9b83d --- /dev/null +++ b/backend/dissemination/sql/api/admin_api_v1_1_1/drop_views.sql @@ -0,0 +1,8 @@ +begin; + + drop view if exists admin_api_v1_1_1.audit_access; + +commit; + +notify pgrst, + 'reload schema'; diff --git a/backend/dissemination/sql/api/api_v1_0_3/base.sql b/backend/dissemination/sql/api/api_v1_0_3/base.sql new file mode 100644 index 0000000000..dedabe0cb7 --- /dev/null +++ b/backend/dissemination/sql/api/api_v1_0_3/base.sql @@ -0,0 +1,29 @@ +DO +$do$ +BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = 'authenticator') THEN + RAISE NOTICE 'Role "authenticator" already exists. Skipping.'; + ELSE + CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; + END IF; +END +$do$; + +DO +$do$ +BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = 'api_fac_gov') THEN + RAISE NOTICE 'Role "api_fac_gov" already exists. Skipping.'; + ELSE + CREATE ROLE api_fac_gov NOLOGIN; + END IF; +END +$do$; + +GRANT api_fac_gov TO authenticator; + +NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/api/api_v1_0_3/create_functions.sql b/backend/dissemination/sql/api/api_v1_0_3/create_functions.sql new file mode 100644 index 0000000000..62d2b400e7 --- /dev/null +++ b/backend/dissemination/sql/api/api_v1_0_3/create_functions.sql @@ -0,0 +1,24 @@ +-- WARNING +-- Under PostgreSQL 12, the functions below work. +-- Under PostgreSQL 14, these will break. +-- +-- Note the differences: +-- +-- raise info 'Works under PostgreSQL 12'; +-- raise info 'request.header.x-magic %', (SELECT current_setting('request.header.x-magic', true)); +-- raise info 'request.jwt.claim.expires %', (SELECT current_setting('request.jwt.claim.expires', true)); +-- raise info 'Works under PostgreSQL 14'; +-- raise info 'request.headers::json->>x-magic %', (SELECT current_setting('request.headers', true)::json->>'x-magic'); +-- raise info 'request.jwt.claims::json->expires %', (SELECT current_setting('request.jwt.claims', true)::json->>'expires'); +-- +-- To quote the work of Dav Pilkey, "remember this now." + +-- We don't grant tribal access (yet) +create or replace function api_v1_0_3_functions.has_tribal_data_access() returns boolean +as $has_tribal_data_access$ +BEGIN + RETURN 0::BOOLEAN; +END; +$has_tribal_data_access$ LANGUAGE plpgsql; + +NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/api/api_v1_0_3/create_schema.sql b/backend/dissemination/sql/api/api_v1_0_3/create_schema.sql new file mode 100644 index 0000000000..089e746f2f --- /dev/null +++ b/backend/dissemination/sql/api/api_v1_0_3/create_schema.sql @@ -0,0 +1,52 @@ +begin; + +do +$$ +begin + DROP SCHEMA IF EXISTS api_v1_0_3 CASCADE; + DROP SCHEMA IF EXISTS api_v1_0_3_functions CASCADE; + + if not exists (select schema_name from information_schema.schemata where schema_name = 'api_v1_0_3') then + create schema api_v1_0_3; + create schema api_v1_0_3_functions; + + grant usage on schema api_v1_0_3_functions to api_fac_gov; + + -- Grant access to tables and views + alter default privileges + in schema api_v1_0_3 + grant select + -- this includes views + on tables + to api_fac_gov; + + -- Grant access to sequences, if we have them + grant usage on schema api_v1_0_3 to api_fac_gov; + grant select, usage on all sequences in schema api_v1_0_3 to api_fac_gov; + alter default privileges + in schema api_v1_0_3 + grant select, usage + on sequences + to api_fac_gov; + end if; +end +$$ +; + +-- This is the description +COMMENT ON SCHEMA api_v1_0_3 IS + 'The FAC dissemation API version 1.0.3.' +; + +-- https://postgrest.org/en/stable/references/api/openapi.html +-- This is the title +COMMENT ON SCHEMA api_v1_0_3 IS +$$v1.0.3 + +A RESTful API that serves data from the SF-SAC.$$; + +commit; + +notify pgrst, + 'reload schema'; + diff --git a/backend/dissemination/sql/api/api_v1_0_3/create_views.sql b/backend/dissemination/sql/api/api_v1_0_3/create_views.sql new file mode 100644 index 0000000000..598701bede --- /dev/null +++ b/backend/dissemination/sql/api/api_v1_0_3/create_views.sql @@ -0,0 +1,335 @@ +begin; + +--------------------------------------- +-- finding_text +--------------------------------------- +create view api_v1_0_3.findings_text as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + ft.finding_ref_number, + ft.contains_chart_or_table, + ft.finding_text + from + public.dissemination_findingtext ft, + public.dissemination_general gen + where + ft.report_id = gen.report_id + and + (gen.is_public = true + or + (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) + order by ft.id +; + +--------------------------------------- +-- additional_ueis +--------------------------------------- +create view api_v1_0_3.additional_ueis as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + uei.additional_uei + from + public.dissemination_general gen, + public.dissemination_additionaluei uei + where + gen.report_id = uei.report_id + and + (gen.is_public = true + or + (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) + order by uei.id +; + +--------------------------------------- +-- finding +--------------------------------------- +create view api_v1_0_3.findings as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + finding.award_reference, + finding.reference_number, + finding.is_material_weakness, + finding.is_modified_opinion, + finding.is_other_findings, + finding.is_other_matters, + finding.prior_finding_ref_numbers, + finding.is_questioned_costs, + finding.is_repeat_finding, + finding.is_significant_deficiency, + finding.type_requirement + from + public.dissemination_finding finding, + public.dissemination_general gen + where + finding.report_id = gen.report_id + and + (gen.is_public = true + or + (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) + order by finding.id +; + +--------------------------------------- +-- federal award +--------------------------------------- +create view api_v1_0_3.federal_awards as + select + award.report_id, + gen.auditee_uei, + gen.audit_year, + --- + award.award_reference, + award.federal_agency_prefix, + award.federal_award_extension, + award.additional_award_identification, + award.federal_program_name, + award.amount_expended, + award.cluster_name, + award.other_cluster_name, + award.state_cluster_name, + award.cluster_total, + award.federal_program_total, + award.is_major, + award.is_loan, + award.loan_balance, + award.is_direct, + award.audit_report_type, + award.findings_count, + award.is_passthrough_award, + award.passthrough_amount + from + public.dissemination_federalaward award, + public.dissemination_general gen + where + award.report_id = gen.report_id + and + (gen.is_public = true + or + (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) + order by award.id +; + + +--------------------------------------- +-- corrective_action_plan +--------------------------------------- +create view api_v1_0_3.corrective_action_plans as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + ct.finding_ref_number, + ct.contains_chart_or_table, + ct.planned_action + from + public.dissemination_CAPText ct, + public.dissemination_General gen + where + ct.report_id = gen.report_id + and + (gen.is_public = true + or + (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) + order by ct.id +; + +--------------------------------------- +-- notes_to_sefa +--------------------------------------- +create view api_v1_0_3.notes_to_sefa as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + note.note_title as title, + note.accounting_policies, + note.is_minimis_rate_used, + note.rate_explained, + note.content, + note.contains_chart_or_table + from + public.dissemination_general gen, + public.dissemination_note note + where + note.report_id = gen.report_id + and + (gen.is_public = true + or + (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) + order by note.id +; + +--------------------------------------- +-- passthrough +--------------------------------------- +create view api_v1_0_3.passthrough as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + pass.award_reference, + pass.passthrough_id, + pass.passthrough_name + from + public.dissemination_general as gen, + public.dissemination_passthrough as pass + where + gen.report_id = pass.report_id + and + (gen.is_public = true + or + (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) + order by pass.id +; + + +--------------------------------------- +-- general +--------------------------------------- +create view api_v1_0_3.general as + select + -- every table starts with report_id, UEI, and year + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + gen.auditee_certify_name, + gen.auditee_certify_title, + gen.auditee_contact_name, + gen.auditee_email, + gen.auditee_name, + gen.auditee_phone, + gen.auditee_contact_title, + gen.auditee_address_line_1, + gen.auditee_city, + gen.auditee_state, + gen.auditee_ein, + gen.auditee_zip, + -- auditor + gen.auditor_phone, + gen.auditor_state, + gen.auditor_city, + gen.auditor_contact_title, + gen.auditor_address_line_1, + gen.auditor_zip, + gen.auditor_country, + gen.auditor_contact_name, + gen.auditor_email, + gen.auditor_firm_name, + gen.auditor_foreign_address, + gen.auditor_ein, + -- agency + gen.cognizant_agency, + gen.oversight_agency, + -- dates + gen.date_created, + gen.ready_for_certification_date, + gen.auditor_certified_date, + gen.auditee_certified_date, + gen.submitted_date, + gen.fac_accepted_date, + gen.fy_end_date, + gen.fy_start_date, + gen.audit_type, + gen.gaap_results, + gen.sp_framework_basis, + gen.is_sp_framework_required, + gen.sp_framework_opinions, + gen.is_going_concern_included, + gen.is_internal_control_deficiency_disclosed, + gen.is_internal_control_material_weakness_disclosed, + gen.is_material_noncompliance_disclosed, + gen.dollar_threshold, + gen.is_low_risk_auditee, + gen.agencies_with_prior_findings, + gen.entity_type, + gen.number_months, + gen.audit_period_covered, + gen.total_amount_expended, + gen.type_audit_code, + gen.is_public, + gen.data_source, + gen.is_aicpa_audit_guide_included, + gen.is_additional_ueis, + CASE EXISTS(SELECT ein.report_id FROM dissemination_additionalein ein WHERE ein.report_id = gen.report_id) + WHEN FALSE THEN 'No' + ELSE 'Yes' + END AS is_multiple_eins, + CASE EXISTS(SELECT aud.report_id FROM dissemination_secondaryauditor aud WHERE aud.report_id = gen.report_id) + WHEN FALSE THEN 'No' + ELSE 'Yes' + END AS is_secondary_auditors + from + public.dissemination_general gen + where + gen.is_public = true + or + (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access()) + order by gen.id +; + +--------------------------------------- +-- auditor (secondary auditor) +--------------------------------------- +create view api_v1_0_3.secondary_auditors as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + sa.auditor_ein, + sa.auditor_name, + sa.contact_name, + sa.contact_title, + sa.contact_email, + sa.contact_phone, + sa.address_street, + sa.address_city, + sa.address_state, + sa.address_zipcode + from + public.dissemination_General gen, + public.dissemination_SecondaryAuditor sa + where + sa.report_id = gen.report_id + and + (gen.is_public=True + or + (gen.is_public=false and api_v1_0_3_functions.has_tribal_data_access())) + order by sa.id +; + +create view api_v1_0_3.additional_eins as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + ein.additional_ein + from + public.dissemination_general gen, + public.dissemination_additionalein ein + where + gen.report_id = ein.report_id + and + (gen.is_public = true + or + (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) + order by ein.id +; + +commit; + +notify pgrst, + 'reload schema'; + diff --git a/backend/dissemination/sql/api/api_v1_0_3/drop_schema.sql b/backend/dissemination/sql/api/api_v1_0_3/drop_schema.sql new file mode 100644 index 0000000000..cf1aca6d91 --- /dev/null +++ b/backend/dissemination/sql/api/api_v1_0_3/drop_schema.sql @@ -0,0 +1,11 @@ + +begin; + +DROP SCHEMA IF EXISTS api_v1_0_3 CASCADE; +-- DROP ROLE IF EXISTS authenticator; +-- DROP ROLE IF EXISTS api_fac_gov; + +commit; + +notify pgrst, + 'reload schema'; diff --git a/backend/dissemination/sql/api/api_v1_0_3/drop_views.sql b/backend/dissemination/sql/api/api_v1_0_3/drop_views.sql new file mode 100644 index 0000000000..2775087977 --- /dev/null +++ b/backend/dissemination/sql/api/api_v1_0_3/drop_views.sql @@ -0,0 +1,17 @@ +begin; + drop table if exists api_v1_0_3.metadata; + drop view if exists api_v1_0_3.general; + drop view if exists api_v1_0_3.auditor; + drop view if exists api_v1_0_3.federal_awards; + drop view if exists api_v1_0_3.findings; + drop view if exists api_v1_0_3.findings_text; + drop view if exists api_v1_0_3.corrective_action_plans; + drop view if exists api_v1_0_3.additional_ueis; + drop view if exists api_v1_0_3.notes_to_sefa; + drop view if exists api_v1_0_3.passthrough; + drop view if exists api_v1_0_3.secondary_auditors; + drop view if exists api_v1_0_3.additional_eins; +commit; + +notify pgrst, + 'reload schema'; diff --git a/backend/dissemination/sql/api/api_v1_0_3/standup.sql b/backend/dissemination/sql/api/api_v1_0_3/standup.sql new file mode 100644 index 0000000000..18e99e91d1 --- /dev/null +++ b/backend/dissemination/sql/api/api_v1_0_3/standup.sql @@ -0,0 +1,440 @@ +DO +$do$ +BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = 'authenticator') THEN + RAISE NOTICE 'Role "authenticator" already exists. Skipping.'; + ELSE + CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; + END IF; +END +$do$; + +DO +$do$ +BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = 'api_fac_gov') THEN + RAISE NOTICE 'Role "api_fac_gov" already exists. Skipping.'; + ELSE + CREATE ROLE api_fac_gov NOLOGIN; + END IF; +END +$do$; + +GRANT api_fac_gov TO authenticator; + +NOTIFY pgrst, 'reload schema'; +begin; + +do +$$ +begin + DROP SCHEMA IF EXISTS api_v1_0_3 CASCADE; + DROP SCHEMA IF EXISTS api_v1_0_3_functions CASCADE; + + if not exists (select schema_name from information_schema.schemata where schema_name = 'api_v1_0_3') then + create schema api_v1_0_3; + create schema api_v1_0_3_functions; + + grant usage on schema api_v1_0_3_functions to api_fac_gov; + + -- Grant access to tables and views + alter default privileges + in schema api_v1_0_3 + grant select + -- this includes views + on tables + to api_fac_gov; + + -- Grant access to sequences, if we have them + grant usage on schema api_v1_0_3 to api_fac_gov; + grant select, usage on all sequences in schema api_v1_0_3 to api_fac_gov; + alter default privileges + in schema api_v1_0_3 + grant select, usage + on sequences + to api_fac_gov; + end if; +end +$$ +; + +-- This is the description +COMMENT ON SCHEMA api_v1_0_3 IS + 'The FAC dissemation API version 1.0.3.' +; + +-- https://postgrest.org/en/stable/references/api/openapi.html +-- This is the title +COMMENT ON SCHEMA api_v1_0_3 IS +$$v1.0.3 + +A RESTful API that serves data from the SF-SAC.$$; + +commit; + +notify pgrst, + 'reload schema'; + +-- WARNING +-- Under PostgreSQL 12, the functions below work. +-- Under PostgreSQL 14, these will break. +-- +-- Note the differences: +-- +-- raise info 'Works under PostgreSQL 12'; +-- raise info 'request.header.x-magic %', (SELECT current_setting('request.header.x-magic', true)); +-- raise info 'request.jwt.claim.expires %', (SELECT current_setting('request.jwt.claim.expires', true)); +-- raise info 'Works under PostgreSQL 14'; +-- raise info 'request.headers::json->>x-magic %', (SELECT current_setting('request.headers', true)::json->>'x-magic'); +-- raise info 'request.jwt.claims::json->expires %', (SELECT current_setting('request.jwt.claims', true)::json->>'expires'); +-- +-- To quote the work of Dav Pilkey, "remember this now." + +-- We don't grant tribal access (yet) +create or replace function api_v1_0_3_functions.has_tribal_data_access() returns boolean +as $has_tribal_data_access$ +BEGIN + RETURN 0::BOOLEAN; +END; +$has_tribal_data_access$ LANGUAGE plpgsql; + +NOTIFY pgrst, 'reload schema'; +begin; + +--------------------------------------- +-- finding_text +--------------------------------------- +create view api_v1_0_3.findings_text as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + ft.finding_ref_number, + ft.contains_chart_or_table, + ft.finding_text + from + public.dissemination_findingtext ft, + public.dissemination_general gen + where + ft.report_id = gen.report_id + and + (gen.is_public = true + or + (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) + order by ft.id +; + +--------------------------------------- +-- additional_ueis +--------------------------------------- +create view api_v1_0_3.additional_ueis as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + uei.additional_uei + from + public.dissemination_general gen, + public.dissemination_additionaluei uei + where + gen.report_id = uei.report_id + and + (gen.is_public = true + or + (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) + order by uei.id +; + +--------------------------------------- +-- finding +--------------------------------------- +create view api_v1_0_3.findings as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + finding.award_reference, + finding.reference_number, + finding.is_material_weakness, + finding.is_modified_opinion, + finding.is_other_findings, + finding.is_other_matters, + finding.prior_finding_ref_numbers, + finding.is_questioned_costs, + finding.is_repeat_finding, + finding.is_significant_deficiency, + finding.type_requirement + from + public.dissemination_finding finding, + public.dissemination_general gen + where + finding.report_id = gen.report_id + and + (gen.is_public = true + or + (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) + order by finding.id +; + +--------------------------------------- +-- federal award +--------------------------------------- +create view api_v1_0_3.federal_awards as + select + award.report_id, + gen.auditee_uei, + gen.audit_year, + --- + award.award_reference, + award.federal_agency_prefix, + award.federal_award_extension, + award.additional_award_identification, + award.federal_program_name, + award.amount_expended, + award.cluster_name, + award.other_cluster_name, + award.state_cluster_name, + award.cluster_total, + award.federal_program_total, + award.is_major, + award.is_loan, + award.loan_balance, + award.is_direct, + award.audit_report_type, + award.findings_count, + award.is_passthrough_award, + award.passthrough_amount + from + public.dissemination_federalaward award, + public.dissemination_general gen + where + award.report_id = gen.report_id + and + (gen.is_public = true + or + (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) + order by award.id +; + + +--------------------------------------- +-- corrective_action_plan +--------------------------------------- +create view api_v1_0_3.corrective_action_plans as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + ct.finding_ref_number, + ct.contains_chart_or_table, + ct.planned_action + from + public.dissemination_CAPText ct, + public.dissemination_General gen + where + ct.report_id = gen.report_id + and + (gen.is_public = true + or + (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) + order by ct.id +; + +--------------------------------------- +-- notes_to_sefa +--------------------------------------- +create view api_v1_0_3.notes_to_sefa as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + note.note_title as title, + note.accounting_policies, + note.is_minimis_rate_used, + note.rate_explained, + note.content, + note.contains_chart_or_table + from + public.dissemination_general gen, + public.dissemination_note note + where + note.report_id = gen.report_id + and + (gen.is_public = true + or + (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) + order by note.id +; + +--------------------------------------- +-- passthrough +--------------------------------------- +create view api_v1_0_3.passthrough as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + pass.award_reference, + pass.passthrough_id, + pass.passthrough_name + from + public.dissemination_general as gen, + public.dissemination_passthrough as pass + where + gen.report_id = pass.report_id + and + (gen.is_public = true + or + (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) + order by pass.id +; + + +--------------------------------------- +-- general +--------------------------------------- +create view api_v1_0_3.general as + select + -- every table starts with report_id, UEI, and year + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + gen.auditee_certify_name, + gen.auditee_certify_title, + gen.auditee_contact_name, + gen.auditee_email, + gen.auditee_name, + gen.auditee_phone, + gen.auditee_contact_title, + gen.auditee_address_line_1, + gen.auditee_city, + gen.auditee_state, + gen.auditee_ein, + gen.auditee_zip, + -- auditor + gen.auditor_phone, + gen.auditor_state, + gen.auditor_city, + gen.auditor_contact_title, + gen.auditor_address_line_1, + gen.auditor_zip, + gen.auditor_country, + gen.auditor_contact_name, + gen.auditor_email, + gen.auditor_firm_name, + gen.auditor_foreign_address, + gen.auditor_ein, + -- agency + gen.cognizant_agency, + gen.oversight_agency, + -- dates + gen.date_created, + gen.ready_for_certification_date, + gen.auditor_certified_date, + gen.auditee_certified_date, + gen.submitted_date, + gen.fac_accepted_date, + gen.fy_end_date, + gen.fy_start_date, + gen.audit_type, + gen.gaap_results, + gen.sp_framework_basis, + gen.is_sp_framework_required, + gen.sp_framework_opinions, + gen.is_going_concern_included, + gen.is_internal_control_deficiency_disclosed, + gen.is_internal_control_material_weakness_disclosed, + gen.is_material_noncompliance_disclosed, + gen.dollar_threshold, + gen.is_low_risk_auditee, + gen.agencies_with_prior_findings, + gen.entity_type, + gen.number_months, + gen.audit_period_covered, + gen.total_amount_expended, + gen.type_audit_code, + gen.is_public, + gen.data_source, + gen.is_aicpa_audit_guide_included, + gen.is_additional_ueis, + CASE EXISTS(SELECT ein.report_id FROM dissemination_additionalein ein WHERE ein.report_id = gen.report_id) + WHEN FALSE THEN 'No' + ELSE 'Yes' + END AS is_multiple_eins, + CASE EXISTS(SELECT aud.report_id FROM dissemination_secondaryauditor aud WHERE aud.report_id = gen.report_id) + WHEN FALSE THEN 'No' + ELSE 'Yes' + END AS is_secondary_auditors + from + public.dissemination_general gen + where + gen.is_public = true + or + (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access()) + order by gen.id +; + +--------------------------------------- +-- auditor (secondary auditor) +--------------------------------------- +create view api_v1_0_3.secondary_auditors as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + sa.auditor_ein, + sa.auditor_name, + sa.contact_name, + sa.contact_title, + sa.contact_email, + sa.contact_phone, + sa.address_street, + sa.address_city, + sa.address_state, + sa.address_zipcode + from + public.dissemination_General gen, + public.dissemination_SecondaryAuditor sa + where + sa.report_id = gen.report_id + and + (gen.is_public=True + or + (gen.is_public=false and api_v1_0_3_functions.has_tribal_data_access())) + order by sa.id +; + +create view api_v1_0_3.additional_eins as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + ein.additional_ein + from + public.dissemination_general gen, + public.dissemination_additionalein ein + where + gen.report_id = ein.report_id + and + (gen.is_public = true + or + (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) + order by ein.id +; + +commit; + +notify pgrst, + 'reload schema'; + diff --git a/backend/dissemination/sql/api/api_v1_0_3/teardown.sql b/backend/dissemination/sql/api/api_v1_0_3/teardown.sql new file mode 100644 index 0000000000..d29d530cbc --- /dev/null +++ b/backend/dissemination/sql/api/api_v1_0_3/teardown.sql @@ -0,0 +1,27 @@ +begin; + +DROP SCHEMA IF EXISTS api_v1_0_3 CASCADE; +-- DROP ROLE IF EXISTS authenticator; +-- DROP ROLE IF EXISTS api_fac_gov; + +commit; + +notify pgrst, + 'reload schema'; +begin; + drop table if exists api_v1_0_3.metadata; + drop view if exists api_v1_0_3.general; + drop view if exists api_v1_0_3.auditor; + drop view if exists api_v1_0_3.federal_awards; + drop view if exists api_v1_0_3.findings; + drop view if exists api_v1_0_3.findings_text; + drop view if exists api_v1_0_3.corrective_action_plans; + drop view if exists api_v1_0_3.additional_ueis; + drop view if exists api_v1_0_3.notes_to_sefa; + drop view if exists api_v1_0_3.passthrough; + drop view if exists api_v1_0_3.secondary_auditors; + drop view if exists api_v1_0_3.additional_eins; +commit; + +notify pgrst, + 'reload schema'; diff --git a/backend/dissemination/sql/api/api_v1_1_0/base.sql b/backend/dissemination/sql/api/api_v1_1_0/base.sql new file mode 100644 index 0000000000..dedabe0cb7 --- /dev/null +++ b/backend/dissemination/sql/api/api_v1_1_0/base.sql @@ -0,0 +1,29 @@ +DO +$do$ +BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = 'authenticator') THEN + RAISE NOTICE 'Role "authenticator" already exists. Skipping.'; + ELSE + CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; + END IF; +END +$do$; + +DO +$do$ +BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = 'api_fac_gov') THEN + RAISE NOTICE 'Role "api_fac_gov" already exists. Skipping.'; + ELSE + CREATE ROLE api_fac_gov NOLOGIN; + END IF; +END +$do$; + +GRANT api_fac_gov TO authenticator; + +NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/api/api_v1_1_0/config1.yaml b/backend/dissemination/sql/api/api_v1_1_0/config1.yaml new file mode 100644 index 0000000000..09d55d7a06 --- /dev/null +++ b/backend/dissemination/sql/api/api_v1_1_0/config1.yaml @@ -0,0 +1,43 @@ +# This uses an env var called PG to +# set the connection string. +source: PG +target: LOCAL + +streams: + + public.dissemination_general.xlsx: + mode: full-refresh + target_options: + format: xlsx + object: file:///tmp/sling/dg.xlsx + sql: | + SELECT * from public.dissemination_general + WHERE is_public = true + + public.dissemination_general.csv: + mode: full-refresh + object: file:///tmp/sling/dg.csv + sql: | + SELECT * from public.dissemination_general + WHERE is_public = true + target_options: + format: csv + + + public.dissemination_federalaward: + mode: full-refresh + object: file:///tmp/sling/dfa/*.csv + target_options: + file_max_rows: 1000 + format: csv + sql: | + SELECT * FROM + public.dissemination_general dg, + public.dissemination_federalaward dfa + WHERE + dfa.report_id = dg.report_id + AND + dg.is_public = true + + + diff --git a/backend/dissemination/sql/api/api_v1_1_0/create_functions.sql b/backend/dissemination/sql/api/api_v1_1_0/create_functions.sql new file mode 100644 index 0000000000..b39ac337fb --- /dev/null +++ b/backend/dissemination/sql/api/api_v1_1_0/create_functions.sql @@ -0,0 +1,126 @@ +-- WARNING +-- Under PostgreSQL 12, the functions below work. +-- Under PostgreSQL 14, these will break. +-- +-- Note the differences: +-- +-- raise info 'Works under PostgreSQL 12'; +-- raise info 'request.header.x-magic %', (SELECT current_setting('request.header.x-magic', true)); +-- raise info 'request.jwt.claim.expires %', (SELECT current_setting('request.jwt.claim.expires', true)); +-- raise info 'Works under PostgreSQL 14'; +-- raise info 'request.headers::json->>x-magic %', (SELECT current_setting('request.headers', true)::json->>'x-magic'); +-- raise info 'request.jwt.claims::json->expires %', (SELECT current_setting('request.jwt.claims', true)::json->>'expires'); +-- +-- To quote the work of Dav Pilkey, "remember this now." + + +CREATE OR REPLACE FUNCTION api_v1_1_0_functions.get_header(item text) RETURNS text + AS $get_header$ + declare res text; + begin + SELECT (current_setting('request.headers', true)::json)->>item into res; + return res; + end; +$get_header$ LANGUAGE plpgsql; + +create or replace function api_v1_1_0_functions.get_api_key_uuid() returns TEXT +as $gaku$ +declare uuid text; +begin + select api_v1_1_0_functions.get_header('x-api-user-id') into uuid; + return uuid; +end; +$gaku$ LANGUAGE plpgsql; + +create or replace function api_v1_1_0_functions.has_tribal_data_access() +returns boolean +as $has_tribal_data_access$ +DECLARE + uuid_header UUID; + key_exists boolean; +BEGIN + + SELECT api_v1_1_0_functions.get_api_key_uuid() INTO uuid_header; + SELECT + CASE WHEN EXISTS ( + SELECT key_id + FROM public.dissemination_TribalApiAccessKeyIds taaki + WHERE taaki.key_id = uuid_header::TEXT) + THEN 1::BOOLEAN + ELSE 0::BOOLEAN + END + INTO key_exists; + RAISE INFO 'api_v1_1_0 has_tribal % %', uuid_header, key_exists; + RETURN key_exists; +END; +$has_tribal_data_access$ LANGUAGE plpgsql; + +-- If you change the constant defined by this function, +-- you must regenerate the index. +CREATE OR REPLACE FUNCTION api_v1_1_0_functions.batch_size() + RETURNS int + LANGUAGE sql IMMUTABLE PARALLEL SAFE AS +'SELECT 20000'; +GRANT EXECUTE ON FUNCTION api_v1_1_0_functions.batch_size() TO api_fac_gov; + +CREATE OR REPLACE FUNCTION api_v1_1_0_functions.batch (id bigint) +returns bigint +as $batch$ +declare num bigint; +begin + select div(id, api_v1_1_0_functions.batch_size()) into num; + return num; +end; +$batch$ +language plpgsql immutable; +GRANT EXECUTE ON FUNCTION api_v1_1_0_functions.batch(bigint) TO api_fac_gov; + +CREATE OR REPLACE FUNCTION api_v1_1_0.batches (in_table text) +returns integer +as $batches$ +declare count integer; +declare batch_size bigint; +begin + select api_v1_1_0_functions.batch_size() into batch_size; + case + when in_table = 'general' then + select div(count(*), batch_size) into count + from public.dissemination_general; + when in_table = 'federal_awards' then + select div(count(*), batch_size) into count + from public.dissemination_federalaward; + else + count := 0; + end case; + RETURN count; +end; +$batches$ +language plpgsql; +GRANT EXECUTE ON FUNCTION api_v1_1_0.batches(text) TO api_fac_gov; + +CREATE OR REPLACE FUNCTION api_v1_1_0.get_general_batch (batch_no bigint) +returns setof dissemination_general +as $batches$ + select * from public.dissemination_general where api_v1_1_0_functions.batch(id) = batch_no; +$batches$ +language sql; +GRANT EXECUTE ON FUNCTION api_v1_1_0.get_general_batch(bigint) TO api_fac_gov; + +CREATE OR REPLACE FUNCTION api_v1_1_0.get_federal_award_batch (batch_no bigint) +returns setof dissemination_federalaward +as $batches$ + select * from public.dissemination_federalaward where api_v1_1_0_functions.batch(public.dissemination_federalaward.id) = batch_no; +$batches$ +language sql; +GRANT EXECUTE ON FUNCTION api_v1_1_0.get_federal_award_batch(bigint) TO api_fac_gov; + + +-- We should consider dropping and regenerating this index +-- every night after MV generation. +-- drop index batch_by_id_dfa; +create index IF NOT EXISTS batch_by_id_dfa + on public.dissemination_federalaward + using btree(api_v1_1_0_functions.batch(public.dissemination_federalaward.id)); + + +NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/api/api_v1_1_0/create_schema.sql b/backend/dissemination/sql/api/api_v1_1_0/create_schema.sql new file mode 100644 index 0000000000..2d8811dd3a --- /dev/null +++ b/backend/dissemination/sql/api/api_v1_1_0/create_schema.sql @@ -0,0 +1,60 @@ +begin; + +do +$$ +begin + DROP SCHEMA IF EXISTS api_v1_1_0 CASCADE; + DROP SCHEMA IF EXISTS api_v1_1_0_functions CASCADE; + + if not exists (select schema_name from information_schema.schemata where schema_name = 'api_v1_1_0') then + create schema api_v1_1_0; + create schema api_v1_1_0_functions; + + grant usage on schema api_v1_1_0_functions to api_fac_gov; + grant usage on schema api_v1_1_0 to api_fac_gov; + grant select, usage on all sequences in schema api_v1_1_0 to api_fac_gov; + + -- Grant access to tables and views + -- NOTE 20240920: This grants SELECT to all tables + -- created in this schema *in the future*. Nothing + -- that already exists is affected. + -- https://www.postgresql.org/docs/current/sql-alterdefaultprivileges.html + alter default privileges + in schema api_v1_1_0 + grant select + -- this includes views + on tables + to api_fac_gov; + + alter default privileges + in schema api_v1_1_0_functions + grant select + -- this includes views + on tables + to api_fac_gov; + + -- Grant access to future sequences, if we have them + alter default privileges + in schema api_v1_1_0 + grant select, usage + on sequences + to api_fac_gov; + + end if; +end +$$ +; + +-- https://postgrest.org/en/stable/references/api/openapi.html +-- This is the title (version number) and description (text). +COMMENT ON SCHEMA api_v1_1_0 IS +$$v1.1.0 + +A RESTful API that serves data from the SF-SAC.$$; + + +commit; + +notify pgrst, + 'reload schema'; + diff --git a/backend/dissemination/sql/api/api_v1_1_0/create_views.sql b/backend/dissemination/sql/api/api_v1_1_0/create_views.sql new file mode 100644 index 0000000000..ec3fda2336 --- /dev/null +++ b/backend/dissemination/sql/api/api_v1_1_0/create_views.sql @@ -0,0 +1,308 @@ +begin; + +--------------------------------------- +-- finding_text +--------------------------------------- +create view api_v1_1_0.findings_text as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + ft.finding_ref_number, + ft.contains_chart_or_table, + ft.finding_text + from + dissemination_findingtext ft, + dissemination_general gen + where + ft.report_id = gen.report_id + and + (gen.is_public = true + or (gen.is_public = false and api_v1_1_0_functions.has_tribal_data_access())) + order by ft.id +; + +--------------------------------------- +-- additional_ueis +--------------------------------------- +create view api_v1_1_0.additional_ueis as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + uei.additional_uei + from + dissemination_general gen, + dissemination_additionaluei uei + where + gen.report_id = uei.report_id + order by uei.id +; + +--------------------------------------- +-- finding +--------------------------------------- +create view api_v1_1_0.findings as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + finding.award_reference, + finding.reference_number, + finding.is_material_weakness, + finding.is_modified_opinion, + finding.is_other_findings, + finding.is_other_matters, + finding.prior_finding_ref_numbers, + finding.is_questioned_costs, + finding.is_repeat_finding, + finding.is_significant_deficiency, + finding.type_requirement + from + dissemination_finding finding, + dissemination_general gen + where + finding.report_id = gen.report_id + order by finding.id +; + +--------------------------------------- +-- federal award +--------------------------------------- +create view api_v1_1_0.federal_awards as + select + award.report_id, + gen.auditee_uei, + gen.audit_year, + --- + award.award_reference, + award.federal_agency_prefix, + award.federal_award_extension, + award.additional_award_identification, + award.federal_program_name, + award.amount_expended, + award.cluster_name, + award.other_cluster_name, + award.state_cluster_name, + award.cluster_total, + award.federal_program_total, + award.is_major, + award.is_loan, + award.loan_balance, + award.is_direct, + award.audit_report_type, + award.findings_count, + award.is_passthrough_award, + award.passthrough_amount + from + dissemination_federalaward award, + dissemination_general gen + where + award.report_id = gen.report_id + order by award.id +; + + +--------------------------------------- +-- corrective_action_plan +--------------------------------------- +create view api_v1_1_0.corrective_action_plans as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + ct.finding_ref_number, + ct.contains_chart_or_table, + ct.planned_action + from + dissemination_CAPText ct, + dissemination_General gen + where + ct.report_id = gen.report_id + and + (gen.is_public = true + or (gen.is_public = false and api_v1_1_0_functions.has_tribal_data_access())) + order by ct.id +; + +--------------------------------------- +-- notes_to_sefa +--------------------------------------- +create view api_v1_1_0.notes_to_sefa as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + note.note_title as title, + note.accounting_policies, + note.is_minimis_rate_used, + note.rate_explained, + note.content, + note.contains_chart_or_table + from + dissemination_general gen, + dissemination_note note + where + note.report_id = gen.report_id + and + (gen.is_public = true + or (gen.is_public = false and api_v1_1_0_functions.has_tribal_data_access())) + order by note.id +; + +--------------------------------------- +-- passthrough +--------------------------------------- +create view api_v1_1_0.passthrough as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + pass.award_reference, + pass.passthrough_id, + pass.passthrough_name + from + dissemination_general as gen, + dissemination_passthrough as pass + where + gen.report_id = pass.report_id + order by pass.id +; + + +--------------------------------------- +-- general +--------------------------------------- +create view api_v1_1_0.general as + select + -- every table starts with report_id, UEI, and year + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + gen.auditee_certify_name, + gen.auditee_certify_title, + gen.auditee_contact_name, + gen.auditee_email, + gen.auditee_name, + gen.auditee_phone, + gen.auditee_contact_title, + gen.auditee_address_line_1, + gen.auditee_city, + gen.auditee_state, + gen.auditee_ein, + gen.auditee_zip, + -- auditor + gen.auditor_certify_name, + gen.auditor_certify_title, + gen.auditor_phone, + gen.auditor_state, + gen.auditor_city, + gen.auditor_contact_title, + gen.auditor_address_line_1, + gen.auditor_zip, + gen.auditor_country, + gen.auditor_contact_name, + gen.auditor_email, + gen.auditor_firm_name, + gen.auditor_foreign_address, + gen.auditor_ein, + -- agency + gen.cognizant_agency, + gen.oversight_agency, + -- dates + gen.date_created, + gen.ready_for_certification_date, + gen.auditor_certified_date, + gen.auditee_certified_date, + gen.submitted_date, + gen.fac_accepted_date, + gen.fy_end_date, + gen.fy_start_date, + gen.audit_type, + gen.gaap_results, + gen.sp_framework_basis, + gen.is_sp_framework_required, + gen.sp_framework_opinions, + gen.is_going_concern_included, + gen.is_internal_control_deficiency_disclosed, + gen.is_internal_control_material_weakness_disclosed, + gen.is_material_noncompliance_disclosed, + gen.dollar_threshold, + gen.is_low_risk_auditee, + gen.agencies_with_prior_findings, + gen.entity_type, + gen.number_months, + gen.audit_period_covered, + gen.total_amount_expended, + gen.type_audit_code, + gen.is_public, + gen.data_source, + gen.is_aicpa_audit_guide_included, + gen.is_additional_ueis, + CASE EXISTS(SELECT ein.report_id FROM dissemination_additionalein ein WHERE ein.report_id = gen.report_id) + WHEN FALSE THEN 'No' + ELSE 'Yes' + END AS is_multiple_eins, + CASE EXISTS(SELECT aud.report_id FROM dissemination_secondaryauditor aud WHERE aud.report_id = gen.report_id) + WHEN FALSE THEN 'No' + ELSE 'Yes' + END AS is_secondary_auditors + from + dissemination_general gen + order by gen.id +; + +--------------------------------------- +-- auditor (secondary auditor) +--------------------------------------- +create view api_v1_1_0.secondary_auditors as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + sa.auditor_ein, + sa.auditor_name, + sa.contact_name, + sa.contact_title, + sa.contact_email, + sa.contact_phone, + sa.address_street, + sa.address_city, + sa.address_state, + sa.address_zipcode + from + dissemination_General gen, + dissemination_SecondaryAuditor sa + where + sa.report_id = gen.report_id + order by sa.id +; + +create view api_v1_1_0.additional_eins as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + ein.additional_ein + from + dissemination_general gen, + dissemination_additionalein ein + where + gen.report_id = ein.report_id + order by ein.id +; + +commit; + + + +notify pgrst, + 'reload schema'; + diff --git a/backend/dissemination/sql/api/api_v1_1_0/db_to_db.yaml b/backend/dissemination/sql/api/api_v1_1_0/db_to_db.yaml new file mode 100644 index 0000000000..71a7cd5157 --- /dev/null +++ b/backend/dissemination/sql/api/api_v1_1_0/db_to_db.yaml @@ -0,0 +1,316 @@ +# This uses an env var called PG to +# set the connection string. +source: PG +target: PG +env: + SLING_ALLOW_EMPTY: true + +streams: + # The first table we create is the general table. All of the + # general table is public data, so we copy it wholesale. + public.dissemination_general: + mode: full-refresh + object: public_data.general + sql: | + -- All of the general table is public data. + SELECT + -- every table starts with report_id, UEI, and year + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.agencies_with_prior_findings, + gen.audit_period_covered, + gen.audit_type, + gen.auditee_address_line_1, + gen.auditee_certified_date, + gen.auditee_certify_name, + gen.auditee_certify_title, + gen.auditee_city, + gen.auditee_contact_name, + gen.auditee_contact_title, + gen.auditee_ein, + gen.auditee_email, + gen.auditee_name, + gen.auditee_phone, + gen.auditee_state, + gen.auditee_zip, + gen.auditor_address_line_1, + gen.auditor_certified_date, + gen.auditor_certify_name, + gen.auditor_certify_title, + gen.auditor_city, + gen.auditor_contact_name, + gen.auditor_contact_title, + gen.auditor_country, + gen.auditor_ein, + gen.auditor_email, + gen.auditor_firm_name, + gen.auditor_foreign_address, + gen.auditor_phone, + gen.auditor_state, + gen.auditor_zip, + gen.cognizant_agency, + gen.data_source, + gen.date_created, + gen.dollar_threshold, + gen.entity_type, + gen.fac_accepted_date, + gen.fy_end_date, + gen.fy_start_date, + gen.gaap_results, + gen.is_additional_ueis, + gen.is_aicpa_audit_guide_included, + gen.is_going_concern_included, + gen.is_internal_control_deficiency_disclosed, + gen.is_internal_control_material_weakness_disclosed, + gen.is_low_risk_auditee, + gen.is_material_noncompliance_disclosed, + CASE EXISTS + ( + SELECT + ein.report_id + FROM + dissemination_additionalein ein + WHERE + ein.report_id = gen.report_id + ) + WHEN FALSE THEN 'No' + ELSE 'Yes' + END AS is_multiple_eins, + gen.is_public, + CASE EXISTS + ( + SELECT + aud.report_id + FROM + dissemination_secondaryauditor aud + WHERE + aud.report_id = gen.report_id + ) + WHEN FALSE THEN 'No' + ELSE 'Yes' + END AS is_secondary_auditors, + gen.is_sp_framework_required, + gen.number_months, + gen.oversight_agency, + gen.ready_for_certification_date, + gen.sp_framework_basis, + gen.sp_framework_opinions, + gen.submitted_date, + gen.total_amount_expended, + gen.type_audit_code + FROM + public.dissemination_general gen + ORDER BY gen.id + + # All additional EINs are public. + public.additional_eins: + mode: full-refresh + object: public_data.additional_eins + sql: | + SELECT + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + ein.additional_ein + FROM + public_data.general gen, + public.dissemination_additionalein ein + WHERE + gen.report_id = ein.report_id + ORDER BY ein.id + + # All of the additional UEI info is public info. + public.dissemination_additional_ueis: + mode: full-refresh + object: public_data.additional_ueis + sql: | + SELECT + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + uei.additional_uei + FROM + public_data.general gen, + public.dissemination_additionaluei uei + WHERE + gen.report_id = uei.report_id + ORDER BY uei.id + + # Corrective action plans are NOT always public. + public.dissemination_corrective_action_plan: + mode: full-refresh + object: public_data.corrective_action_plans + sql: | + SELECT + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + ct.contains_chart_or_table, + ct.finding_ref_number, + ct.planned_action + FROM + public.dissemination_captext ct, + public_data.general gen + WHERE + ct.report_id = gen.report_id + AND + -- Only include the public corrective action plans. + gen.is_public = true + ORDER BY ct.id + + # All Federal award data is public. + public.dissemination_federalaward: + mode: full-refresh + object: public_data.federal_awards + sql: | + SELECT + dg.report_id, + dg.auditee_uei, + dg.audit_year, + dg.fac_accepted_date, + --- + award.additional_award_identification, + award.amount_expended, + award.audit_report_type, + award.award_reference, + award.cluster_name, + award.cluster_total, + award.federal_agency_prefix, + award.federal_award_extension, + award.federal_program_name, + award.federal_program_total, + award.findings_count, + award.is_direct, + award.is_loan, + award.is_major, + award.is_passthrough_award, + award.loan_balance, + award.other_cluster_name, + award.passthrough_amount, + award.state_cluster_name + FROM + public.dissemination_federalaward award, + public_data.general dg + WHERE + award.report_id = dg.report_id + ORDER BY award.id + + # The findings table is public. + public.dissemination_findings: + mode: full-refresh + object: public_data.findings + sql: | + SELECT + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + finding.award_reference, + finding.is_material_weakness, + finding.is_modified_opinion, + finding.is_other_findings, + finding.is_other_matters, + finding.is_questioned_costs, + finding.is_repeat_finding, + finding.is_significant_deficiency, + finding.prior_finding_ref_numbers, + finding.reference_number, + finding.type_requirement + FROM + public.dissemination_finding finding, + public_data.general gen + WHERE + finding.report_id = gen.report_id + ORDER BY finding.id + + # The notes to SEFA are NOT all public. + public.dissemination_notes: + mode: full-refresh + object: public_data.notes_to_sefa + sql: | + SELECT + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + note.accounting_policies, + note.contains_chart_or_table, + note.content, + note.is_minimis_rate_used, + note.note_title as title, + note.rate_explained + FROM + public_data.general gen, + public.dissemination_note note + WHERE + note.report_id = gen.report_id + AND + -- Some notes are not public. + gen.is_public = true + ORDER BY note.id + + # All passthrough information is public. + public.dissemination_passthrough: + mode: full-refresh + object: public_data.passthrough + sql: | + SELECT + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + pass.award_reference, + pass.passthrough_id, + pass.passthrough_name + FROM + public_data.general gen, + public.dissemination_passthrough pass + WHERE + gen.report_id = pass.report_id + ORDER BY pass.id + + # All secondary auditor info is public. + public.secondary_auditors: + mode: full-refresh + object: public_data.secondary_auditors + sql: | + SELECT + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + sa.address_city, + sa.address_state, + sa.address_street, + sa.address_zipcode, + sa.auditor_ein, + sa.auditor_name, + sa.contact_email, + sa.contact_name, + sa.contact_phone, + sa.contact_title + FROM + public_data.general gen, + public.dissemination_secondaryauditor sa + WHERE + sa.report_id = gen.report_id + ORDER BY sa.id + + + +# SELECT column_name +# FROM information_schema.columns +# WHERE table_schema = 'public' +# AND table_name = 'dissemination_general' +# AND column_name NOT IN ('id'); diff --git a/backend/dissemination/sql/api/api_v1_1_0/drop_schema.sql b/backend/dissemination/sql/api/api_v1_1_0/drop_schema.sql new file mode 100644 index 0000000000..e32038ee46 --- /dev/null +++ b/backend/dissemination/sql/api/api_v1_1_0/drop_schema.sql @@ -0,0 +1,11 @@ + +begin; + +DROP SCHEMA IF EXISTS api_v1_1_0 CASCADE; +-- DROP ROLE IF EXISTS authenticator; +-- DROP ROLE IF EXISTS api_fac_gov; + +commit; + +notify pgrst, + 'reload schema'; diff --git a/backend/dissemination/sql/api/api_v1_1_0/drop_views.sql b/backend/dissemination/sql/api/api_v1_1_0/drop_views.sql new file mode 100644 index 0000000000..877d5920c6 --- /dev/null +++ b/backend/dissemination/sql/api/api_v1_1_0/drop_views.sql @@ -0,0 +1,17 @@ +begin; + drop table if exists api_v1_1_0.metadata; + drop view if exists api_v1_1_0.general; + drop view if exists api_v1_1_0.auditor; + drop view if exists api_v1_1_0.federal_awards; + drop view if exists api_v1_1_0.findings; + drop view if exists api_v1_1_0.findings_text; + drop view if exists api_v1_1_0.corrective_action_plans; + drop view if exists api_v1_1_0.additional_ueis; + drop view if exists api_v1_1_0.notes_to_sefa; + drop view if exists api_v1_1_0.passthrough; + drop view if exists api_v1_1_0.secondary_auditors; + drop view if exists api_v1_1_0.additional_eins; +commit; + +notify pgrst, + 'reload schema'; diff --git a/backend/dissemination/sql/api/api_v1_1_0/performance.sql b/backend/dissemination/sql/api/api_v1_1_0/performance.sql new file mode 100644 index 0000000000..da21b5ba15 --- /dev/null +++ b/backend/dissemination/sql/api/api_v1_1_0/performance.sql @@ -0,0 +1,36 @@ +select count(report_id) from dissemination_general where api_v1_1_0.batch(id) = 2; + +grant select on dissemination_general to api_fac_gov; + + +alter default privileges + in schema api_v1_1_0_functions + grant select +-- this includes views +on tables +to api_fac_gov; + +SET SESSION SESSION AUTHORIZATION api_fac_gov; + +SELECT SESSION_USER, CURRENT_USER; +SET SESSION SESSION AUTHORIZATION 'api_fac_gov'; +SELECT SESSION_USER, CURRENT_USER; +RESET SESSION AUTHORIZATION; +SELECT SESSION_USER, CURRENT_USER; + +SET SESSION SESSION AUTHORIZATION 'api_fac_gov'; +select count(report_id) from dissemination_general where api_v1_1_0.batch(id) = 2; + +GRANT SELECT ON public.dissemination_general to api_fac_gov; +GRANT SELECT ON public.dissemination_federalaward to api_fac_gov; + +REVOKE SELECT ON public.dissemination_federalaward from api_fac_gov; +REVOKE SELECT ON public.dissemination_general from api_fac_gov; + +SET SESSION SESSION AUTHORIZATION 'api_fac_gov'; +select api_v1_1_0.get_federal_award_batch(2); + + +------------------- + +SELECT id, report_id FROM public.dissemination_general WHERE id < 3000; diff --git a/backend/dissemination/sql/api/api_v1_1_0/public_data.general b/backend/dissemination/sql/api/api_v1_1_0/public_data.general new file mode 100644 index 0000000000..8a8c8743ca --- /dev/null +++ b/backend/dissemination/sql/api/api_v1_1_0/public_data.general @@ -0,0 +1,314 @@ +id,report_id,auditee_certify_name,auditee_certify_title,auditee_contact_name,auditee_email,auditee_name,auditee_phone,auditee_contact_title,auditee_address_line_1,auditee_city,auditee_state,auditee_ein,auditee_uei,is_additional_ueis,auditee_zip,auditor_phone,auditor_state,auditor_city,auditor_contact_title,auditor_address_line_1,auditor_zip,auditor_country,auditor_contact_name,auditor_email,auditor_firm_name,auditor_foreign_address,auditor_ein,cognizant_agency,oversight_agency,date_created,ready_for_certification_date,auditor_certified_date,auditee_certified_date,submitted_date,fy_end_date,fy_start_date,audit_year,audit_type,gaap_results,sp_framework_basis,is_sp_framework_required,sp_framework_opinions,is_going_concern_included,is_internal_control_deficiency_disclosed,is_internal_control_material_weakness_disclosed,is_material_noncompliance_disclosed,is_aicpa_audit_guide_included,dollar_threshold,is_low_risk_auditee,agencies_with_prior_findings,entity_type,number_months,audit_period_covered,total_amount_expended,type_audit_code,is_public,data_source,fac_accepted_date,auditor_certify_name,auditor_certify_title +56000,2022-06-CENSUS-0000232294,"VISTA AT ENTRADA, SCHOOL OF PERFORMING ARTS AND TECHNOLOGY",FINANCE AND IT DIRECTOR,TROY BRADSHAW,TBRADSHAW@VISTAUTAH.COM,"VISTA AT ENTRADA, SCHOOL OF PERFORMING ARTS AND TECHNOLOGY",4356734110,FINANCE AND IT DIRECTOR,585 E CENTER ST,IVINS,UT,263368224,XXW1B6FYZKU7,No,84738,8012256900,UT,OREM,PARTNER,1329 SOUTH 800 EAST,84097,USA,M. PAUL WINWARD,PAULW@SQUIRE.COM,"SQUIRE & COMPANY, PC",,870343246,,84,2024-01-20,2022-11-02,2022-11-03,2022-11-03,2022-11-02,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"84, 10",local,,annual,1225211,UG,true,CENSUS,2022-11-02,M. PAUL WINWARD,PARTNER +57000,2022-06-CENSUS-0000235102,NORTHEAST EDUCATIONAL SERVICES COOPERATIVE,BUSINESS MANAGER,TIFFANY STORMO,TIFFANY.STORMO@K12.SD.US,NORTHEAST EDUCATIONAL SERVICES COOPERATIVE,6057833607,BUSINESS MANAGER,310 5TH STREET,HAYTI,SD,460360461,J3GGF5B55VP3,No,57241,6059967717,SD,MITCHELL,PARTNER,PO BOX 249,57301,USA,TERRI POST,TERRI.POST@ELOCPA.COM,ELO CPAS & ADVISORS,,460434947,,84,2024-01-20,2023-03-20,2023-04-12,2023-04-11,2023-03-20,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,Yes,84,local,,annual,2537584,UG,true,CENSUS,2023-03-20,TERRI POST,PARTNER +58000,2022-06-CENSUS-0000257833,"NATIONAL ASSOCIATION OF FAMILY DEVELOPMENT CENTERS, INC.",EXECUTIVE DIRECTOR,PAMELA KAPLAN,PAMLKAP3@GMAIL.COM,"NATIONAL ASSOCIATION OF FAMILY DEVELOPMENT CENTERS, INC.",7182587767,EXECUTIVE DIRECTOR,1114 AVENUE J,BROOKLYN,NY,112707362,ZL6RQ6TZJDH7,No,11230,7184681240,NY,QUEENS VILLAGE,OWNER,"221-10 JAMAICA AVENUE, SUITE 207",11428,USA,GRACE NKENKE,GRACECPA@AOL.COM,"GRACE NKENKE, CPA",,331112630,,93,2024-01-20,2023-03-30,2023-03-31,2023-03-31,2023-03-30,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 10, 84",non-profit,,annual,3535134,UG,true,CENSUS,2023-03-30,GRACE NKENKE,OWNER +59000,2022-06-CENSUS-0000231704,ROANOKE CITY PUBLIC SCHOOLS,DIRECTOR OF ACCOUNTING SERVICES,DONNA CALDWELL,DCALDWELL@RCPS.INFO,ROANOKE CITY PUBLIC SCHOOLS,5408532125,DIRECTOR OF ACCOUNTING SERVICES,"40 DOUGLAS AVENUE, N.W.",ROANOKE,VA,546001570,ZU3QCHKBMW13,No,24012,4349489000,VA,ROANOKE,DIRECTOR,3905 ELECTRIC ROAD,24018,USA,SUSAN CHAPMAN,SCHAPMAN@BECPAS.COM,"BROWN, EDWARDS & COMPANY LLP.",,540504608,84,,2024-01-20,2023-01-10,2022-12-16,2023-01-11,2023-01-10,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,1741814,Yes,"84, 10, 12, 93, 21",local,,annual,58060475,UG,true,CENSUS,2023-01-10,SUSAN CHAPMAN,DIRECTOR +60000,2022-06-CENSUS-0000085100,"CATHOLIC CHARITITES OF THE ARCHDIOCESE OF ATLANTA, INC.",CEO,VANESSA RUSSELL,VRUSSELL@CCATLANTA.ORG,"CATHOLIC CHARITITES OF THE ARCHDIOCESE OF ATLANTA, INC.",4049207729,CEO,"2401 LAKE PARK DRIVE, SE",SMYRNA,GA,581097003,SP3QV7HE24K4,No,30080,4042627920,GA,ATLANTA,SHAREHOLDER,"3475 PIEDMONT ROAD, SUITE 1500",30305,USA,LAURA HELLER,LHH@JONESKOLB.COM,"JONES AND KOLB, CPA'S",,581763570,,93,2024-01-20,2023-03-30,2023-03-24,2023-03-29,2023-03-30,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,Yes,"99, 14, 97, 10, 93, 19, 94, 16, 84",non-profit,,annual,2564967,UG,true,CENSUS,2023-03-30,LAURA HELLER,SHAREHOLDER +61000,2021-08-CENSUS-0000174864,AUSTIN COMMUNITY COLLEGE DISTRICT,"EVC, FINANCE AND ADMINISTRATION",NEIL VICKERS,NVICKERS@AUSTINCC.EDU,AUSTIN COMMUNITY COLLEGE DISTRICT,5122231098,"EVC, FINANCE AND ADMINISTRATION",5930 MIDDLE FISKVILLE ROAD,AUSTIN,TX,741742036,GSA_MIGRATION,No,78752,9724489205,TX,DALLAS,PARTNER,"2300 NORTH FIELD STREET, SUITE 1000",75201,USA,DALLAS DUGGER,DALLAS.DUGGER@WEAVER.COM,"WEAVER AND TIDWELL, LLP",,750786316,84,,2024-01-20,2022-01-27,2022-01-28,2022-01-28,2022-01-27,2021-08-31,2020-08-31,2021,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,2889917,No,"11, 84, 47, 10, 93, 17",higher-ed,,annual,96330573,UG,true,CENSUS,2022-01-27,DALLAS DUGGER,PARTNER +62000,2021-06-CENSUS-0000204454,CHAMBERLAIN'S YOUTH SERVICES,CHIEF FINANCIAL OFFICER,RENEE ROCHA,RROCHA@CHAMBERLAINSYOUTH.ORG,CHAMBERLAIN'S YOUTH SERVICES,8316362121,CHIEF FINANCIAL OFFICER,1850 SAN BENITO ST,HOLLISTER,CA,942357401,GSA_MIGRATION,No,95023,6264036801,CA,PASADENA,PARTNER,2698 MATARO STREET,91107,USA,CARLOS A. DAVIS,CARLOS@NPOCPAS.COM,HARRINGTON GROUP,,954557617,,93,2024-01-20,2022-04-07,2022-03-28,2022-04-08,2022-04-07,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"97, 93, 10",non-profit,,annual,1329595,UG,true,CENSUS,2022-04-07,CARLOS A. DAVIS,PARTNER +65000,2021-06-CENSUS-0000251712,"TOWN OF HULETT, WYOMING",MAYOR,TED PARSONS,HULETTMAYOR@GMAIL.COM,"TOWN OF HULETT, WYOMING",3074675771,MAYOR,POBOX 278,HULETT,WY,830211382,GSA_MIGRATION,No,82720,3073335093,WY,CASPER,OWNER,90 VALLEY DR,82604,USA,DEBRA KAY RAUGUTT,DEBRARCPA2016@CHARTER.NET,DEBRA RAUGUTT. LLC,,811317330,,20,2024-01-20,2021-12-12,2021-12-12,2021-12-13,2021-12-12,2021-06-30,2020-06-30,2021,single-audit,not_gaap,cash_basis,No,unmodified_opinion,No,No,No,No,Yes,750000,No,20,local,,annual,1071068,UG,true,CENSUS,2021-12-12,DEBRA KAY RAUGUTT,OWNER +63000,2021-06-CENSUS-0000181365,KENTUCKY HIGHER EDUCATION STUDENT LOAN CORPORATION,DIRECTOR OF INTERNAL AUDIT,RHONDA MANN,RMANN@KHESLC.COM,KENTUCKY HIGHER EDUCATION STUDENT LOAN CORPORATION,5026387435,DIRECTOR OF INTERNAL AUDIT,10180 LINN STATION ROAD,LOUISVILLE,KY,521294450,GSA_MIGRATION,No,402240266,5025851600,KY,LOUISVILLE,MANAGING PARTNER,"1600 WATERFRONT PLAZA, 325 WEST MAIN STREET",40202,USA,WILLIAM MEYER,BMEYER@STROTHMAN.COM,STROTHMAN AND CO.,,611191655,,84,2024-01-20,2021-10-14,2021-10-15,2021-10-15,2021-10-14,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,84,state,,annual,2069309,UG,true,CENSUS,2021-10-14,WILLIAM MEYER,MANAGING PARTNER +64000,2021-12-CENSUS-0000245950,TIMBER LAKE HOUSING CORPORATION,VICE PRESIDENT OF ACCOUNTING,TERRA PAWLICK-MCGIFFIN,TERRA.MCGIFFIN@IMPACTSEVEN.ORG,TIMBER LAKE HOUSING CORPORATION,7154341710,VICE PRESIDENT OF ACCOUNTING,2961 DECKER DR,RICE LAKE,WI,391984458,GSA_MIGRATION,No,54868,8506500125,FL,DESTIN,MANAGING PARTNER,543 HARBOR BLVD 201,32541,USA,CHRISTINE CARTER,CHRISTINE@CARTERCPA.NET,CARTER & COMPANY CPA LLC,,582646754,,14,2024-01-20,2022-03-13,2022-03-13,2022-03-14,2022-03-13,2021-12-31,2020-12-31,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,941026,UG,true,CENSUS,2022-03-13,CHRISTINE CARTER,MANAGING PARTNER +66000,2021-06-CENSUS-0000227760,PINNACLE ACADEMY,AUDIT MANAGER,JANET THATCHER,NHAAUDIT@NHASCHOOLS.COM,PINNACLE ACADEMY,6169291327,AUDIT MANAGER,860 EAST 22ND STREET,EUCLID,OH,201104751,GSA_MIGRATION,No,44123,7402894131,OH,PIKETON,PARTNER,528 SOUTH WEST STREET,45661,USA,BRENDEN BALESTRA,BRENDEN.BALESTRA@BHMCPAGROUP.COM,BHM CPA GROUP,,311413363,,84,2024-01-20,2022-01-31,2022-02-01,2022-02-01,2022-01-31,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"21, 10, 84",local,,annual,1281231,UG,true,CENSUS,2022-01-31,BRENDEN BALESTRA,PARTNER +67000,2021-09-CENSUS-0000234422,JUST 4 KIDS FOOD PROGRAM,EXECUTIVE DIRECTOR,YOLANDA LEE,JUST4KIDSFP@YAHOO.COM,JUST 4 KIDS FOOD PROGRAM,4053879455,EXECUTIVE DIRECTOR,PO BOX 1475,NEWCASTLE,OK,262344185,GSA_MIGRATION,No,73065,5803328548,OK,ADA,MANAGING MEMBER,PO BOX 1406,748211406,USA,G. B. SAUNDERS,GSAUNDERS@SAUNDERSCPAS.COM,"SAUNDERS & ASSOCIATES, PLLC",,208209116,,10,2024-01-20,2022-06-26,2022-06-27,2022-06-27,2022-06-26,2021-09-30,2020-09-30,2021,single-audit,not_gaap,cash_basis,No,unmodified_opinion,No,No,No,No,No,750000,No,10,non-profit,,annual,1076958,UG,true,CENSUS,2022-06-26,G. B. SAUNDERS,MANAGING MEMBER +68000,2021-12-CENSUS-0000010982,THE HEBREW HOME FOR THE AGED AT RIVERDALE,VP OF FINANCE,CARL WILLNER,CARL.WILLNER@RIVERSPRINGHEALTH.ORG,THE HEBREW HOME FOR THE AGED AT RIVERDALE,7185811313,VP OF FINANCE,5901 PALISADE AVENUE,BRONX,NY,131739971,GSA_MIGRATION,No,10471,2128674000,NY,NEW YORK,MANAGING DIRECTOR,1155 AVENUE OF THE AMERICAS,10036,USA,ERIC GOLDFARB,ERIC.GOLDFARB@FORVIS.COM,FORVIS LLP,,440160260,,97,2024-01-20,2022-09-28,2022-09-29,2022-09-29,2022-09-28,2021-12-31,2020-12-31,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"97, 93",non-profit,,annual,15841051,UG,true,CENSUS,2022-09-28,ERIC GOLDFARB,MANAGING DIRECTOR +69000,2021-01-CENSUS-0000029970,FAMILY CARE HEALTH CENTERS,CFO,KEVIN MADDOX,KMADDOX@FCHCSTL.ORG,FAMILY CARE HEALTH CENTERS,3144811615,CFO,401 HOLLY HILLS AVENUE,ST. LOUIS,MO,237076112,GSA_MIGRATION,No,63111,3142315544,MO,ST. LOUIS,MANAGING DIRECTOR,211 N. BROADWAY SUITE 600,63102,USA,BETH KNYSAK,BKNYSAK@BKD.COM,"BKD, LLP",,440160260,,93,2024-01-20,2021-08-30,2021-08-31,2021-08-31,2021-08-30,2021-01-31,2020-02-01,2021,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,Yes,"10, 93, 21",non-profit,,annual,7918734,UG,true,CENSUS,2021-08-30,BETH KNYSAK,MANAGING DIRECTOR +70000,2021-12-CENSUS-0000187458,"HOLLYBROOK HOMES, INC. (HOLLYBROOK HOMES)",MANAGING AGENT,"LYNN GRIFFIN, SR.",LYNNGRIFFINSR@180CARES.COM,"HOLLYBROOK HOMES, INC. (HOLLYBROOK HOMES)",9043891196,MANAGING AGENT,104 KING STREET,JACKSONVILLE,FL,591210087,GSA_MIGRATION,No,32204,2259263360,LA,BATON ROUGE,CPA,"5627 BANKERS AVENUE, BUILDING 2",708082610,USA,JEFF MADDOX,MAIL@MADDOXASSOCIATES.COM,"MADDOX & ASSOCIATES, APC",,721314069,,14,2024-01-20,2022-06-13,2022-06-14,2022-06-13,2022-06-13,2021-12-31,2020-12-31,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,2452646,UG,true,CENSUS,2022-06-13,JEFF MADDOX,CPA +71000,2021-06-CENSUS-0000242842,"COMMUNITY EDUCATION ASSOCIATION, INC",PRINCIPAL,SCOT HOOPER,SHOOPER@ODYSSEYCHARTERSCHOOL.NET,"COMMUNITY EDUCATION ASSOCIATION, INC",7702516111,PRINCIPAL,14 ST. JOHN CIRCLE,NEWNAN,GA,582429902,GSA_MIGRATION,No,30265,7709656455,GA,MARIETTA,MANAGING DIRECTOR,"707 WHITLOCK AVE SW, BLDG B STE 21",30064,USA,BAMBO SONAIKE,BAMBO@CPA-SERVICE.COM,"BAMBO SONAIKE CPA, LLC",,900226617,,84,2024-01-20,2021-08-30,2021-08-30,2021-08-31,2021-08-30,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,84,non-profit,,annual,1958209,UG,true,CENSUS,2021-08-30,BAMBO SONAIKE,MANAGING DIRECTOR +72000,2021-12-CENSUS-0000050324,ALLINA HEALTH SYSTEM,MANAGER RESEARCH GRANTS & FINANCE,MICHAEL FULCHER,MICHAEL.FULCHER@ALLINA.COM,ALLINA HEALTH SYSTEM,3302558060,MANAGER RESEARCH GRANTS & FINANCE,2925 CHICAGO AVE,MINNEAPOLIS,MN,363261413,GSA_MIGRATION,No,554071321,3126654266,MN,MINNEAPOLIS,PARTNER,90 SOUTH 7TH STREET SUITE 4200,55402,USA,JARED SILVER,JSSILVER@KPMG.COM,KPMG LLP,,135565207,93,,2024-01-20,2022-09-21,2022-09-22,2022-09-22,2022-09-21,2021-12-31,2020-12-31,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,3000000,Yes,"97, 93, 12",non-profit,,annual,159557744,UG,true,CENSUS,2022-09-21,JARED SILVER,PARTNER +73000,2021-06-CENSUS-0000058645,ALVERNO COLLEGE,CONTROLLER,JEAN V. NORTON,JEAN.POLCZYNSKI-NORTON@ALVERNO.EDU,ALVERNO COLLEGE,4143826188,CONTROLLER,3401 S 39TH STREET,MILWAUKEE,WI,390806263,GSA_MIGRATION,No,532343922,4147775423,WI,MILWAUKEE,PARTNER,"777 E WISCONSIN AVE, 32ND FLOOR",53202,USA,WENDI UNGER,WENDI.UNGER@BAKERTILLY.COM,"BAKER TILLY US, LLP",,390859910,,84,2024-01-20,2022-03-07,2022-03-08,2022-03-08,2022-03-07,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"21, 47, 84",higher-ed,,annual,26996479,UG,true,CENSUS,2022-03-07,WENDI UNGER,PARTNER +74000,2021-06-CENSUS-0000053634,SHAWNEE HEALTH SERVICE AND DEVELOPMENT CORPORATION,CHIEF FINANCIAL OFFICER,JEFF COOPER,JCOOPER@SHSDC.ORG,SHAWNEE HEALTH SERVICE AND DEVELOPMENT CORPORATION,6189859513,CHIEF FINANCIAL OFFICER,109 CALIFORNIA ST,CARTERVILLE,IL,370966854,GSA_MIGRATION,No,62918,6185291040,IL,MARION,PARTNER,3401 OFFICE PARK DRIVE,62959,USA,MARK DALLAS,MARKD@KEBCPA.COM,"KERBER, ECK & BRAECKEL",,430352985,,93,2024-01-20,2022-03-31,2022-04-01,2022-04-01,2022-03-31,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"21, 93",non-profit,,annual,8851122,UG,true,CENSUS,2022-03-31,MARK DALLAS,PARTNER +75000,2021-06-CENSUS-0000206536,SHOW BUS PUBLIC TRANSPORTATION NFP,DIRECTOR,LAURA DICK,LAURA.SHOWBUSNFP@GMAIL.COM,SHOW BUS PUBLIC TRANSPORTATION NFP,8159458500,DIRECTOR,510 HOSELTON DRIVE,CHENOA,IL,264530305,GSA_MIGRATION,No,61726,8152231095,IL,LASALLE,PRESIDENT,219 BUCKLIN STREET,61301,USA,MICHAEL WALKER,MIKE.WALKER@WALKERACCOUNTINGGROUP.COM,"WALKER ACCOUNTING GROUP, LTD.",,453552783,,20,2024-01-20,2022-04-27,2022-04-28,2022-04-28,2022-04-27,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,20,non-profit,,annual,2714828,UG,true,CENSUS,2022-04-27,MICHAEL WALKER,PRESIDENT +76000,2021-06-CENSUS-0000165493,NOBLE PUBLIC SCHOOLS,SUPERINTENDENT,FRANK SOLOMON,FSOLOMON@NOBLEPS.COM,NOBLE PUBLIC SCHOOLS,4058723452,SUPERINTENDENT,111 S. 4TH ST.,NOBLE,OK,736021050,GSA_MIGRATION,No,73068,9182508838,OK,BROKEN ARROW,CPA,2101 N WILLOW AVE.,74012,USA,KERRY PATTEN,KERRY.PATTEN@PATTENCPA.NET,"KERRY JOHN PATTEN, C.P.A.",,731408273,,84,2024-01-20,2022-03-10,2022-03-11,2022-03-11,2022-03-10,2021-06-30,2020-06-30,2021,single-audit,not_gaap,regulatory_basis,Yes,qualified_opinion,No,No,No,No,No,750000,No,"10, 84",local,,annual,4254210,UG,true,CENSUS,2022-03-10,KERRY PATTEN,CPA +77000,2021-06-CENSUS-0000209555,SCHOOL DISTRICT OF MISHICOT,BUSINESS MANAGER,CHRISTINE THELEN,CTHELEN@MISHICOT.K12.WI.US,SCHOOL DISTRICT OF MISHICOT,9207553159,BUSINESS MANAGER,660 WASHINGTON STREET,MISHICOT,WI,396023446,GSA_MIGRATION,No,54223,9204554305,WI,GREEN BAY,PRINCIPAL,2200 RIVERSIDE DRIVE,54305,USA,"SUSAN PABLE, CPA",SUE.PABLE@CLACONNECT.COM,CLIFTONLARSONALLEN LLP,,410746749,,84,2024-01-20,2021-12-19,2021-12-20,2021-12-20,2021-12-19,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,No,"84, 10, 93",local,,annual,1188691,UG,true,CENSUS,2021-12-19,"SUSAN PABLE, CPA",PRINCIPAL +78000,2021-06-CENSUS-0000231497,WATCHUNG HILLS REGIONAL HIGH SCHOOL DISTRICT,SCHOOL BUSINESS ADMINISTRATOR,"TIMOTHY STYS, CPA, SBA",TSTYS@WHRHS.ORG,WATCHUNG HILLS REGIONAL HIGH SCHOOL DISTRICT,9086474800,SCHOOL BUSINESS ADMINISTRATOR,108 STIRLING RD.,WARREN,NJ,226012324,GSA_MIGRATION,No,07059,2017917100,NJ,FAIR LAWN,PARTNER,17-17 ROUTE 208 NORTH,07410,USA,GARY J. VINCI,GVINCI@LVHCPA.COM,"LERCH, VINCI & HIGGINS, LLP",,223015339,,84,2024-01-20,2022-03-03,2022-03-04,2022-03-04,2022-03-03,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"84, 21",local,,annual,789736,UG,true,CENSUS,2022-03-03,GARY J. VINCI,PARTNER +79000,2021-06-CENSUS-0000246657,WESTERN MICHIGAN UNIVERSITY HOMER STRYKER M.D. SCHOOL OF MEDICINE,"DIRECTOR OF ACCOUNTING, CONTROLLER",AMY SMITCHOLS,AMY.SMITCHOLS@MED.WMICH.EDU,WESTERN MICHIGAN UNIVERSITY HOMER STRYKER M.D. SCHOOL OF MEDICINE,2693374704,"DIRECTOR OF ACCOUNTING, CONTROLLER",1000 OAKLAND DRIVE,KALAMAZOO,MI,454135256,GSA_MIGRATION,No,45202,5137444704,OH,CINCINNATI,PARTNER,"537 E PETE ROSE WAY, SUITE 300",45202,USA,STEVEN BISHOP,PM.SINGLEAUDIT@PLANTEMORAN.COM,"PLANTE & MORAN, PLLC",,381357951,,84,2024-01-20,2022-03-15,2022-03-12,2022-03-16,2022-03-15,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"16, 84, 45, 93",higher-ed,,annual,20824246,UG,true,CENSUS,2022-03-15,STEVEN BISHOP,PARTNER +80000,2021-06-CENSUS-0000166784,LANE COUNCIL OF GOVERNMENTS,SENIOR FINANCIAL ANALYST,DAVID JOYAL,DJOYAL@LCOG.ORG,LANE COUNCIL OF GOVERNMENTS,5426823384,SENIOR FINANCIAL ANALYST,"859 WILLAMETTE ST, STE 500",EUGENE,OR,936014373,GSA_MIGRATION,No,974012910,5413425161,OR,EUGENE,PARTNER,1976 GARDEN AVE,97403,USA,GATLIN HAWKINS,GHAWKINS@ISLERCPA.COM,ISLER CPA,,204723636,,93,2024-01-20,2022-02-21,2022-02-18,2022-02-22,2022-02-21,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"16, 10, 20, 66, 93, 14",local,,annual,4409905,UG,true,CENSUS,2022-02-21,GATLIN HAWKINS,PARTNER +81000,2021-06-CENSUS-0000190395,CITY OF ALAMOGORDO,FINANCE DIRECTOR,EVELYN HUFF,EHUFF@CI.ALAMOGORDO.NM.US,CITY OF ALAMOGORDO,5754394268,FINANCE DIRECTOR,1376 EAST 9TH STREET,ALAMOGORDO,NM,856000099,GSA_MIGRATION,No,88310,5058832727,NM,ALBUQUERQUE,PARTNER,2424 LOUISIANA BLVD NE SUITE 300,87110,USA,"ALAN D. BOWERS, JR.",ABOWERS@CRICPA.COM,"CARR, RIGGS & INGRAM, LLC",,721396621,,97,2024-01-20,2022-04-17,2022-04-15,2022-04-18,2022-04-17,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,No,"95, 94, 97, 20, 14, 93, 21, 16, 45",local,,annual,7793927,UG,true,CENSUS,2022-04-17,"ALAN D. BOWERS, JR.",PARTNER +82000,2021-06-CENSUS-0000151757,INDEPENDENT SCHOOL DISTRICT NO. 200,DIRECTOR OF BUSINESS,JENNIFER SEUBERT,JSEUBERT@HASTINGS.K12.MN.US,INDEPENDENT SCHOOL DISTRICT NO. 200,6514807005,DIRECTOR OF BUSINESS,1000 WEST 11TH STREET,HASTINGS,MN,416000810,GSA_MIGRATION,No,55033,9525450424,MN,MINNEAPOLIS,PRINCIPAL,"5353 WAYZATA BLVD, SUITE #410",55416,USA,AARON NIELSEN,ANIELSEN@MMKR.COM,"MALLOY, MONTAGUE, KARNOWSKI, RADOSEVICH & CO., P.A.",,411690382,,84,2024-01-20,2021-11-15,2021-11-15,2021-11-16,2021-11-15,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 10, 21",local,,annual,4645282,UG,true,CENSUS,2021-11-15,AARON NIELSEN,PRINCIPAL +83000,2021-06-CENSUS-0000135824,"TOWN OF TOLLAND, CONNECTICUT",DIRECTOR OF FINANCE AND RECORDS,LISA A HANCOCK,LHANCOCK@TOLLAND.ORG,"TOWN OF TOLLAND, CONNECTICUT",8608713652,DIRECTOR OF FINANCE AND RECORDS,21 TOLLAND GREEN,TOLLAND,CT,066002109,GSA_MIGRATION,No,06084,8605616824,CT,WEST HARTFORD,PRINICPAL,29 SOUTH MAIN STREET,06107,USA,VANESSA ROSSITTO,VANESSA.ROSSITTO@CLACONNECT.COM,CLIFTONLARSONALLEN LLP,,410746749,,84,2024-01-20,2021-12-28,2021-12-29,2021-12-29,2021-12-28,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"21, 97, 90, 93, 10, 84",local,,annual,1488200,UG,true,CENSUS,2021-12-28,VANESSA ROSSITTO,PRINICPAL +84000,2021-06-CENSUS-0000120093,VISIONS UNLIMITED,FISCAL MANAGER,JOSEPH MATLEN,JMATLEN@VUINC.ORG,VISIONS UNLIMITED,9163940800,FISCAL MANAGER,6833 STOCKTON BLVD,SACRAMENTO,CA,942500214,GSA_MIGRATION,No,95823,9165648727,CA,SACRAMENTO,MANAGING PARTNER,"550 HOWE AVENUE, SUITE 210",95825,USA,INGRID SHEIPLINE,ISHEIPLINE@RICHARDSONCPAS.COM,"RICHARDSON & COMPANY, LLP",,465577902,,93,2024-01-20,2021-09-28,2021-09-29,2021-09-29,2021-09-28,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,93,non-profit,,annual,2080537,UG,true,CENSUS,2021-09-28,INGRID SHEIPLINE,MANAGING PARTNER +85000,2021-06-CENSUS-0000160236,CHAUTAUQUA LAKE CENTRAL SCHOOL DISTRICT,SCHOOL BUSINESS EXECUTIVE,JACKSON GRAHAM,JGRAHAM@CLAKE.ORG,CHAUTAUQUA LAKE CENTRAL SCHOOL DISTRICT,7167535874,SCHOOL BUSINESS EXECUTIVE,100 NORTH ERIE STREET,MAYVILLE,NY,161502023,GSA_MIGRATION,No,14757,7163721620,NY,OLEAN,SHAREHOLDER,"130 SOUTH UNION STREET, SUITE 200",14760,USA,"DAVID V. DITANNA, CPA",DVD@BWBCPA.COM,"BUFFAMANTE WHIPPLE BUTTAFARO, P.C.",,161117932,,84,2024-01-20,2021-10-19,2021-10-20,2021-10-19,2021-10-19,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,Yes,Yes,750000,Yes,"84, 10",local,,annual,1261461,UG,true,CENSUS,2021-10-19,"DAVID V. DITANNA, CPA",SHAREHOLDER +86000,2021-12-CENSUS-0000203566,HOUSING AUTHORITY OF THE CITY OF WISCONSIN RAPIDS,EXECUTIVE DIRECTOR,MARY VANG,MARY@WISRAPHA.ORG,HOUSING AUTHORITY OF THE CITY OF WISCONSIN RAPIDS,7154237288,EXECUTIVE DIRECTOR,2521 10TH STREET S,WISCONSIN RAPIDS,WI,391149728,GSA_MIGRATION,No,54494,6083266456,WI,PRAIRIE DU CHIEN,CPA,516 S MARQUETTE RD,53821,USA,BRAD BERTHIAUME,REBECCASTROSCHEIN@CPACOLLINS.COM,"COLLINS & ASSOCIATES, S.C.",,391769340,,14,2024-01-20,2022-09-27,2022-09-28,2022-09-28,2022-09-27,2021-12-31,2020-12-31,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,local,,annual,1418248,UG,true,CENSUS,2022-09-27,BRAD BERTHIAUME,CPA +87000,2021-06-CENSUS-0000187622,GLENS FALLS CITY SCHOOL DISTRICT,ASSISTANT SUPERINTENDENT FOR BUSINESS,ROBERT D YUSKO JR,RYUSKO@GFSD.ORG,GLENS FALLS CITY SCHOOL DISTRICT,5187921451,ASSISTANT SUPERINTENDENT FOR BUSINESS,15 QUADE STREET,GLENS FALLS,NY,146001855,GSA_MIGRATION,No,12801,5188516650,NY,CLAVERACK,CPA,P.O. BOX 538,12513,USA,SCOTT PREUSSER,SCOTT@RGPREUSSER-CPA.COM,"RAYMOND G. PREUSSER, CPA, P.C.",,141821605,,84,2024-01-20,2022-03-02,2022-03-03,2022-03-03,2022-03-02,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 10",local,,annual,2450731,UG,true,CENSUS,2022-03-02,SCOTT PREUSSER,CPA +88000,2021-06-CENSUS-0000218262,"ST. BONIFACE GARDENS, INC. (HUD PROJECT NO. 066-EE074)",CHIEF FINANCIAL OFFICER,DAVID D'AMICO,DDAMICO@CHSFLA.COM,"ST. BONIFACE GARDENS, INC. (HUD PROJECT NO. 066-EE074)",9544841515,CHIEF FINANCIAL OFFICER,11410N. KENDALL DRIVE,MIAMI,FL,650984174,GSA_MIGRATION,No,33176,3054463177,FL,CORAL GABLES,PDEARMAS@VDTCPA.COM,"255 ALHAMBRA CIRCLE, 560",33134,USA,PEDRO M. DE ARMAS,PDEARMAS@VDTCPA.COM,"VERDEJA, DE ARMAS & TRUJILLO, LLP.",,204989621,,14,2024-01-20,2021-10-18,2021-10-19,2021-10-15,2021-10-18,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,7686821,UG,true,CENSUS,2021-10-18,PEDRO M. DE ARMAS,PDEARMAS@VDTCPA.COM +89000,2021-06-CENSUS-0000004757,"CHILDREN'S SERVICES OF ROXBURY, INC.",CHIEF FINANCIAL OFFICER,EDWARD D'AMATO,EDAMATO@CSROX.ORG,"CHILDREN'S SERVICES OF ROXBURY, INC.",6179899647,CHIEF FINANCIAL OFFICER,520 DUDLEY STREET,ROXBURY,MA,043082352,GSA_MIGRATION,No,02119,9787648966,MA,BOSTON,PARTNER,"4238 WASHINGTON ST, STE 307",02131,USA,RANDALL DAVIS,RDAVIS@RSDCPAS.COM,DAVISKELLY LLP,,463169148,,93,2024-01-20,2022-02-24,2022-02-25,2022-02-25,2022-02-24,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,93,non-profit,,annual,3703284,UG,true,CENSUS,2022-02-24,RANDALL DAVIS,PARTNER +90000,2021-12-CENSUS-0000252961,"NATIONAL COWBOY AND WESTERN HERITAGE MUSEUM, INC.",CHIEF FINANCIAL OFFICER,MICHAEL MYERS,MMYERS@NATIONALCOWBOYMUSEUM.ORG,"NATIONAL COWBOY AND WESTERN HERITAGE MUSEUM, INC.",4054782250,CHIEF FINANCIAL OFFICER,1700 NORTHEAST 63RD STREET,OKLAHOMA CITY,OK,300341029,GSA_MIGRATION,No,73111,4058482020,OK,OKLAHOMA CITY,ASSURANCE PARTNER,"1225 N BROADWAY AVENUE, SUITE 200",73103,USA,BRITTNEY WYCOFF,BWYCOFF@HOGANTAYLOR.COM,HOGANTAYLOR LLP,,731413977,,59,2024-01-20,2022-04-12,2022-04-12,2022-04-13,2022-04-12,2021-12-31,2020-12-31,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"59, 45",non-profit,,annual,1370968,UG,true,CENSUS,2022-04-12,BRITTNEY WYCOFF,ASSURANCE PARTNER +91000,2021-06-CENSUS-0000138427,SUGAR-SALEM JOINT SCHOOL DISTRICT NO 322,BUSINESS MANAGER,BECKY BATES,BBATES@SUGARSALEM.COM,SUGAR-SALEM JOINT SCHOOL DISTRICT NO 322,2083568802,BUSINESS MANAGER,P.O. BOX 150,SUGAR CITY,ID,826000836,GSA_MIGRATION,No,83448,2085299276,ID,IDAHO FALLS,CPA,725 S. WOODRUFF AVE.,83401,USA,SCOTT BOND,SBOND@RUDDCO.COM,"RUDD & COMPANY, PLLC",,820467399,,84,2024-01-20,2021-11-19,2021-11-20,2021-11-19,2021-11-19,2021-06-30,2020-06-30,2021,single-audit,qualified_opinion,,,,No,No,No,No,No,750000,No,"84, 93, 10, 45, 21",local,,annual,2541333,UG,true,CENSUS,2021-11-19,SCOTT BOND,CPA +92000,2022-06-CENSUS-0000093828,NASHVILLE CARES,CEO,AMNA OSMAN,AOSMAN@NASHVILLECARES.ORG,NASHVILLE CARES,6152594866,CEO,633 THOMPSON LANE,NASHVILLE,TN,621274532,QCDWWBKW26D8,No,37204,6152427351,TN,NASHVILLE,MEMBER,555 GREAT CIRCLE ROAD,37228,USA,KENNETH YOUNSTEAD,KYOUNGSTEAD@KRAFTCPAS.COM,KRAFTCPAS PLLC,,620713250,,93,2024-01-20,2023-03-30,2023-03-30,2023-03-30,2023-03-30,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,Yes,"14, 93",non-profit,,annual,5915672,UG,true,CENSUS,2023-03-30,KENNETH YOUNSTEAD,MEMBER +93000,2021-06-CENSUS-0000141779,BETHALTO COMMUNITY UNIT SCHOOL DISTRICT NO. 8,BOOKKEEPER,HEATHER PINTER,HPINTER@BETHALTO.ORG,BETHALTO COMMUNITY UNIT SCHOOL DISTRICT NO. 8,6183777200,BOOKKEEPER,101 SCHOOL ST,BETHALTO,IL,376006341,GSA_MIGRATION,No,62010,6184654999,IL,ALTON,PRINCIPAL,1904 STATE ST,62002,USA,DENNIS ROSE,DROSECPA@DRA-CPA.COM,"DENNIS ROSE & ASSOCIATES, P.C.",,371375191,,84,2024-01-20,2022-01-24,2022-01-25,2022-01-25,2022-01-24,2021-06-30,2020-06-30,2021,single-audit,not_gaap,regulatory_basis,Yes,qualified_opinion,No,No,No,No,No,750000,No,"93, 84, 10",local,,annual,2757038,UG,true,CENSUS,2022-01-24,DENNIS ROSE,PRINCIPAL +94000,2021-03-CENSUS-0000194595,HOUSING AUTHORITY OF THE TOWN OF YORK,EXECUTIVE DIRECTOR,RENEE PRINGLE,YORK1802@BELLSOUTH.NET,HOUSING AUTHORITY OF THE TOWN OF YORK,2053925071,EXECUTIVE DIRECTOR,P.O. BOX 9,YORK,AL,636005425,GSA_MIGRATION,No,36925,2059820992,AL,BIRMINGHAM,MEMBER,"200 CHASE PARK SOUTH, SUITE 220",35244,USA,ROY W. HENDERSON JR.,RHENDERSON@HENDERSONCPA.COM,"HENDERSON & PILLETERI, LLC",,263957106,,14,2024-01-20,2021-12-27,2021-12-28,2021-12-28,2021-12-27,2021-03-31,2020-03-31,2021,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,No,14,local,,annual,887057,UG,true,CENSUS,2021-12-27,ROY W. HENDERSON JR.,MEMBER +95000,2021-09-CENSUS-0000258327,NUVANCE HEALTH AND SUBSIDIARIES,"AVP, ACCOUNTING AND STRATEGIC PROJECTS",DEVIN MURPHY,DEVIN.MURPHY@NUVANCEHEALTH.ORG,NUVANCE HEALTH AND SUBSIDIARIES,4019323469,"AVP, ACCOUNTING AND STRATEGIC PROJECTS",100 RESERVE ROAD,DANBURY,CT,834214573,GSA_MIGRATION,No,06810,2127739592,NY,NEW YORK,PARTNER,ONE MANHATTAN WEST,100018604,USA,JOSEPH MORELLO,JOE.MORELLO@EY.COM,ERNST & YOUNG LLP,,346565596,93,,2024-01-20,2023-08-03,2023-08-04,2023-08-04,2023-08-03,2021-09-30,2020-09-30,2021,single-audit,unmodified_opinion,,,,No,Yes,Yes,No,No,3000000,No,"93, 16, 32, 21",non-profit,,annual,137318932,UG,true,CENSUS,2023-08-03,JOSEPH MORELLO,PARTNER +96000,2021-06-CENSUS-0000233469,WEST MARSHALL COMMUNITY SCHOOL DISTRICT,BUSINESS MANAGER,STEPHANIE EDLER,SEDLER@WMCSD.ORG,WEST MARSHALL COMMUNITY SCHOOL DISTRICT,6414832660,BUSINESS MANAGER,"P.O. BOX 670, 601 3RD ST NW",STATE CENTER,IA,420883206,GSA_MIGRATION,No,502471011,6416722541,IA,OSKALOOSA,CERTIFIED PUBLIC ACCOUNTANT,1201 HIGH AVE W,525771937,USA,MICHAEL STANLEY,MSTANLEY@HUNTANDASSOCIATESCPA.COM,"HUNT & ASSOCIATES, PC",,421395096,,10,2024-01-20,2022-02-13,2022-02-11,2022-02-14,2022-02-13,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"93, 84, 10",local,,annual,888006,UG,true,CENSUS,2022-02-13,MICHAEL STANLEY,CERTIFIED PUBLIC ACCOUNTANT +97000,2021-04-CENSUS-0000250304,CITY OF LAHARPE,MAYOR,KATHERINE HASTEN-REED,HASTENWARD3@GMAIL.COM,CITY OF LAHARPE,2176593918,MAYOR,207 E. MAIN ST. P.O. BOX 471,LAHARPE,IL,376000988,GSA_MIGRATION,No,61450,2172247500,IL,QUINCY,PARTNER,901 HAMPSHIRE ST.,62301,USA,ROBERT GRAY,RGRAY@ABNGPC.COM,"ARNOLD, BEHRENS, NESBIT, GRAY, P.C.",,371144423,,10,2024-01-20,2022-04-24,2022-04-25,2022-04-21,2022-04-24,2021-04-30,2020-04-30,2021,single-audit,"adverse_opinion, not_gaap",cash_basis,No,unmodified_opinion,No,No,No,No,Yes,750000,No,"21, 10",local,,annual,1001124,UG,true,CENSUS,2022-04-24,ROBERT GRAY,PARTNER +98000,2021-06-CENSUS-0000134337,LIVE OAK SCHOOL DISTRICT,CHIEF BUSINESS OFFICIAL,ALISON WARNER,AWARNER@LOSD.CA,LIVE OAK SCHOOL DISTRICT,8314756333,CHIEF BUSINESS OFFICIAL,984-I BOSTWICK LANE,SANTA CRUZ,CA,946002632,GSA_MIGRATION,No,95062,6502236104,CA,MENLO PARK,PARTNER,"4040 CAMPBELL AVE., STE. 200",940251053,USA,JOYCE PETERS,JPETERS@EIDEBAILLY.COM,EIDE BAILLY LLP,,450250958,,10,2024-01-20,2022-02-28,2022-02-23,2022-03-01,2022-02-28,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"21, 93, 84, 10",local,,annual,4512861,UG,true,CENSUS,2022-02-28,JOYCE PETERS,PARTNER +99000,2021-06-CENSUS-0000150361,WEST SHORE COMMUNITY COLLEGE,CHIEF FINANCIAL OFFICER,CONNY BAX,CBAX@WESTSHORE.EDU,WEST SHORE COMMUNITY COLLEGE,2318456211,CHIEF FINANCIAL OFFICER,300 NORTH STILES ROAD,SCOTTSVILLE,MI,381852447,GSA_MIGRATION,No,49454,6169754100,MI,GRAND RAPIDS,PRINCIPAL,2330 EAST PARIS AVENUE SE,49546,USA,"MICHELLE FOWLER, CPA",MICHELLE.FOWLER@REHMANN.COM,REHMANN ROBSON LLC,,383635706,,84,2024-01-20,2021-11-09,2021-11-10,2021-11-10,2021-11-09,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"21, 84",higher-ed,,annual,3138146,UG,true,CENSUS,2021-11-09,"MICHELLE FOWLER, CPA",PRINCIPAL +100000,2021-12-CENSUS-0000255288,"FONDOS UNIDOS DE PUERTO RICO, INC.",VP FINANCE AND ADMINISTRATION,HEIDI E. CORTES,H.CORTES@FONDOSUNIDOS.ORG,"FONDOS UNIDOS DE PUERTO RICO, INC.",7877288500,VP FINANCE AND ADMINISTRATION,"CALLE MARGINAL LOS ANGELES, ESQ. SAGRADO",SANTURCE,PR,660269222,GSA_MIGRATION,No,00909,7879934360,PR,SAN JUAN,ASSURANCE & QUALITY CONTROL PARTNER,1250 PONCE DE LEON AVE.,009073912,USA,FERNANDO E. ORTIZ RAMOS,FORTIZ@GTCPAPR.COM,"GONZALEZ TORRES & CO., CPA, PSC",,660528656,,21,2024-01-20,2022-10-04,2022-10-05,2022-10-05,2022-10-04,2021-12-31,2020-12-31,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,21,non-profit,,annual,19689286,UG,true,CENSUS,2022-10-04,FERNANDO E. ORTIZ RAMOS,ASSURANCE & QUALITY CONTROL PARTNER +101000,2021-12-CENSUS-0000168797,BUCKS COUNTY HOUSING AUTHORITY,DIRECTOR OF FINANCE,DONNA MURRAY,DMURRAYBCHA@HOTMAIL.COM,BUCKS COUNTY HOUSING AUTHORITY,2153489469,DIRECTOR OF FINANCE,PO BOX 1329,DOYLESTOWN,PA,231659922,GSA_MIGRATION,No,189010967,7326237003,NJ,TOMS RIVER,PARTNER,"1144 HOOPER AVE, SUITE 203",08753,USA,RICH LARSEN,RICH.LARSEN@NOVOCO.COM,"NOVOGRADAC & COMPANY, LLP",,943108253,,14,2024-01-20,2022-09-27,2022-09-28,2022-09-28,2022-09-27,2021-12-31,2020-12-31,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,906416,Yes,14,local,,annual,30213862,UG,true,CENSUS,2022-09-27,RICH LARSEN,PARTNER +102000,2021-03-CENSUS-0000193568,YWCA OF GREATER BATON ROUGE,EXECUTIVE DIRECTOR,DIANNA PAYTON,PAYTOND@YWCA-BR.ORG,YWCA OF GREATER BATON ROUGE,2253830681,EXECUTIVE DIRECTOR,11404 LAKE SHERWOOD AVE N. STE B,BATON ROUGE,LA,720650993,GSA_MIGRATION,No,70806,5045618820,LA,NEW ORLEANS,MANAGING PARTNER,1100 POYDRAS STREET SUITE 1225,70163,USA,LUTHER SPEIGHT,LUTHER@SPEIGHTCPA.COM,"LUTHER SPEIGHT & CO., LLC",,300156143,,93,2024-01-20,2022-07-25,2022-05-19,2022-07-26,2022-07-25,2021-03-31,2020-03-31,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"14, 93",non-profit,,annual,3236878,UG,true,CENSUS,2022-07-25,LUTHER SPEIGHT,MANAGING PARTNER +103000,2021-03-CENSUS-0000114051,BLUE MOUNTAIN ACTION COUNCIL,CFO,RICK CLARIDGE,RICKC@BMACWW.ORG,BLUE MOUNTAIN ACTION COUNCIL,5095294980,CFO,8 E CHERRY ST,WALLA WALLA,WA,910793597,GSA_MIGRATION,No,99362,5097472158,WA,SPOKANE,CPA,"422 W. RIVERSIDE AVE., #1420",992010395,USA,DARCY SKJOTHAUG,DSKJOTHAUG@SCHOEDEL.COM,SCHOEDEL & SCHOEDEL CPAS PLLC,,910614823,,21,2024-01-20,2021-10-13,2021-10-13,2021-10-14,2021-10-13,2021-03-31,2020-03-31,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"17, 21, 84, 97, 81, 93, 64, 14, 10",non-profit,,annual,8740117,UG,true,CENSUS,2021-10-13,DARCY SKJOTHAUG,CPA +104000,2021-08-CENSUS-0000173942,LACKLAND INDEPENDENT SCHOOL DISTRICT,CHIEF FINANCIAL OFFICER,CEDRIC MENCHION,MENCHION.C@LACKLANDISD.NET,LACKLAND INDEPENDENT SCHOOL DISTRICT,2103575005,CHIEF FINANCIAL OFFICER,"2460 KENLY AVENUE, BUILDING 8265",SAN ANTONIO,TX,746013766,GSA_MIGRATION,No,78236,8302786276,TX,UVALDE,PARTNER,400 E. NOPAL ST.,78801,USA,"DUSTY R. ROUTH, CPA",DUSTYROUTH@COLEMANHORTONCPA.COM,"COLEMAN, HORTON AND COMPANY, LLP",,742668537,,84,2024-01-20,2022-05-30,2022-05-05,2022-05-31,2022-05-30,2021-08-31,2020-08-31,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 12, 21, 10",local,,annual,10655747,UG,true,CENSUS,2022-05-30,"DUSTY R. ROUTH, CPA",PARTNER +105000,2021-06-CENSUS-0000214106,"AHEPA 310-VIII, INC.",VICE PRESIDENT OF FINANCE,TRACY STORTS,TSTORTS@AHEPAHOUSING.ORG,"AHEPA 310-VIII, INC.",3178453410,VICE PRESIDENT OF FINANCE,12680 PADGETT SWITCH ROAD,IRVINGTON,AL,631262819,GSA_MIGRATION,No,36544,9373992000,OH,SPRINGFIELD,PRINCIPAL,14 EAST MAIN STREET SUITE 500,45502,USA,ROBERT KITCHEN,RKITCHEN@CSHCO.COM,CLARK SCHAEFER HACKETT,,310800053,,14,2024-01-20,2021-11-09,2021-11-09,2021-11-09,2021-11-09,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,3519281,UG,true,CENSUS,2021-11-09,ROBERT KITCHEN,PRINCIPAL +106000,2020-06-CENSUS-0000118338,FELTON INSTITUTE-FKA FAMILY SERVICE,CFOO,MARVIN DAVIS,MDAVIS@FELTON.ORG,FELTON INSTITUTE-FKA FAMILY SERVICE,4154747310,CFOO,1005 ATLANTIC AVENUE,ALAMEDA,CA,941156530,GSA_MIGRATION,No,94501,6264036801,CA,PASADENA,PARTNER,"234 E. COLORADO BLVD., SUITE M150",91101,USA,TONETTA L. CONNER,TONETTA@NPOCPAS.COM,HARRINGTON GROUP,,954557617,,94,2024-01-21,2021-03-31,2021-03-31,2021-04-01,2021-03-31,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"94, 10, 93, 84, 17",non-profit,,annual,6587216,UG,true,CENSUS,2021-03-31,TONETTA L. CONNER,PARTNER +107000,2020-06-CENSUS-0000161038,GRAHAM COUNTY,FINANCE OFFICER,REBECCA GARLAND,BECKY.GARLAND@GRAHAMCOUNTY.ORG,GRAHAM COUNTY,8284797961,FINANCE OFFICER,12 NORTH MAIN STREET,ROBBINSVILLE,NC,566000302,GSA_MIGRATION,No,28771,8288378188,NC,MURPHY,CPA PARTNER,31 PEACHTREE STREET,28906,USA,HOLLY M. TURNER,HOLLY@MYTURNERCPA.COM,TURNER & COMPANY CPAS P.A.,,561225077,,93,2024-01-20,2021-06-15,2021-06-10,2021-06-15,2021-06-15,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"23, 97, 10, 93, 14, 20, 21",local,,annual,2113788,UG,true,CENSUS,2021-06-15,HOLLY M. TURNER,CPA PARTNER +108000,2020-06-CENSUS-0000194416,LA CA?ADA UNIFIED SCHOOL DISTRICT,ASSOCIATE SUPERINTENDENT OF BUSINESS,MARK EVANS,MEVANS@LCUSD.NET,LA CA?ADA UNIFIED SCHOOL DISTRICT,8189528380,ASSOCIATE SUPERINTENDENT OF BUSINESS,4490 CORNSHION AVE,LA CANADA,CA,956001790,GSA_MIGRATION,No,91011,9097552816,CA,RANCHO CUCAMONGA,PARTNER,10681 FOOTHILL BLVD STE 300,91730,USA,BRIAN RUFF,BRUFF@EIDEBAILLY.COM,EIDE BAILLY LLP,,450250958,,84,2024-01-21,2021-01-04,2021-01-05,2021-01-04,2021-01-04,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 10",local,,annual,1057396,UG,true,CENSUS,2021-01-04,BRIAN RUFF,PARTNER +109000,2020-06-CENSUS-0000118417,PENINSULA FAMILY SERVICE,CHIEF FINANCIAL OFFICER,INESSA SHISHMANYAN,ISHISHMANYAN@PENINSULAFAMILYSERVICE.ORG,PENINSULA FAMILY SERVICE,6504034300,CHIEF FINANCIAL OFFICER,24 SECOND AVENUE,SAN MATEO,CA,941186169,GSA_MIGRATION,No,944013828,4155683291,CA,SAN FRANCISCO,PARTNER,"44 MONTGOMERY STREET, SUITE 900",94104,USA,GRANT LAM,GRANT.LAM@ARMANINOLLP.COM,ARMANINO LLP,,946214841,,93,2024-01-21,2020-12-21,2020-12-21,2020-12-21,2020-12-21,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"93, 20, 17, 10",non-profit,,annual,3508377,UG,true,CENSUS,2020-12-21,GRANT LAM,PARTNER +110000,2020-06-CENSUS-0000135940,PUTNAM HOUSING AUTHORITY,EXECUTIVE DIRECTOR,KATHY CARTER,PUTNAMHOUSING@GMAIL.COM,PUTNAM HOUSING AUTHORITY,8609636829,EXECUTIVE DIRECTOR,123 LACONIA AVE,PUTNAM,CT,066009290,GSA_MIGRATION,No,06260,8043552808,VA,RICHMOND,PARTNER,21 S SHEPPARD ST,23221,USA,JAKE DOOLEY,JAKE@DVCPAS.COM,DOOLEY & VICARS L.L.P,,541950231,,14,2024-01-21,2021-04-07,2021-04-08,2021-04-08,2021-04-07,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,local,,annual,901313,UG,true,CENSUS,2021-04-07,JAKE DOOLEY,PARTNER +111000,2020-09-CENSUS-0000128011,GREEN MANOR,DIRECTOR OF FINANCE,MICHELE REDDY,MICHELEREDDY@FGA-NET.COM,GREEN MANOR,3232583512,DIRECTOR OF FINANCE,PO BOX 7070,PASADENA,CA,956205420,GSA_MIGRATION,No,91109,6192947200,CA,SAN DIEGO,PARTNER,"2810 CAMINO DEL RIO SOUTH, SUITE 200",92108,USA,MICHAEL J. ZIZZI,LEAFCOLEDATA@LEAF-COLE.COM,"LEAF & COLE, LLP",,952076568,,14,2024-01-21,2021-01-20,2021-01-21,2021-01-21,2021-01-20,2020-09-30,2019-10-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,5323675,UG,true,CENSUS,2021-01-20,MICHAEL J. ZIZZI,PARTNER +112000,2020-03-CENSUS-0000059346,"SIXTEENTH STREET COMMUNITY HEALTH CENTERS, INC.",VICE PRESIDENT OF FINANCE,SHERRI HUFF,SHERRI.HUFF@SSCHC.ORG,"SIXTEENTH STREET COMMUNITY HEALTH CENTERS, INC.",4148975156,VICE PRESIDENT OF FINANCE,"1337 SOUTH CESAR E. CHAVEZ DR., 2ND FL",MILWAUKEE,WI,391180475,GSA_MIGRATION,No,53204,4147775500,WI,MILWAUKEE,PARTNER,"777 E. WISCONSIN AVE, 32ND FLOOR",53202,USA,"KRISTA K. PANKOP, CPA",KRISTA.PANKOP@BAKERTILLY.COM,"BAKER TILLY US, LLP",,390859910,,93,2024-01-21,2020-08-06,2020-08-06,2020-08-06,2020-08-06,2020-03-31,2019-04-01,2020,single-audit,unmodified_opinion,,,,No,No,Yes,No,No,750000,Yes,"93, 10, 94",non-profit,,annual,6777350,UG,true,CENSUS,2020-08-06,"KRISTA K. PANKOP, CPA",PARTNER +113000,2020-06-CENSUS-0000250260,"NEW ORLEANS MILITARY AND MARITIME ACADEMY, INC.",FINANCE DIRECTOR,DENNIS LAFONT,DLAFONT@NOMMA.NET,"NEW ORLEANS MILITARY AND MARITIME ACADEMY, INC.",5048674610,FINANCE DIRECTOR,425 O'BANNON STREET,NEW ORLEANS,LA,270868305,GSA_MIGRATION,No,70114,5048375434,LA,METAIRIE,AUDIT MANAGER,"110 VETERANS MEMORIAL BLVD., SUITE 170",70005,USA,MIKE CAPAROTTA,MCAPAROTTA@HIENZMACALUSO.COM,"HIENZ & MACALUSO, LLC",,721473527,,84,2024-01-21,2021-03-15,2021-03-16,2021-03-16,2021-03-15,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"84, 10",non-profit,,annual,938918,UG,true,CENSUS,2021-03-15,MIKE CAPAROTTA,AUDIT MANAGER +114000,2020-06-CENSUS-0000247782,"NORTHERN MIDDLE TENNESSEE LOCAL WORKFORCE DEVELOPMENT BOARD, INC.",WIOA FISCAL DIRECTOR,GINGER FUSSELL,GFUSSELL@WORKFORCEESSENTIALS.COM,"NORTHERN MIDDLE TENNESSEE LOCAL WORKFORCE DEVELOPMENT BOARD, INC.",9319053559,WIOA FISCAL DIRECTOR,523 MADISON STREET STE A,CLARKSVILLE,TN,621812741,GSA_MIGRATION,No,37040,9315527474,TN,CLARKSVILLE,PARTNER,324 FRANKLIN STREET,37040,USA,JEFF PROCTOR,JPROCTOR@TCCPAS.COM,"THURMAN CAMPBELL GROUP, PLC",,263683574,,17,2024-01-21,2020-12-29,2020-12-30,2020-12-21,2020-12-29,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,17,non-profit,,annual,11901286,UG,true,CENSUS,2020-12-29,JEFF PROCTOR,PARTNER +115000,2020-06-CENSUS-0000181461,OHKAY OWINGEH BOARD OF EDUCATION,CFO,DOMINIC AGUINO,DOMINIC.AGUINO@OHKAY.ORG,OHKAY OWINGEH BOARD OF EDUCATION,5058520408,CFO,PO BOX 1119,OHKAY OWINGEH,NM,850442632,GSA_MIGRATION,No,875661119,5093443200,WA,LIBERTY LAKE,PARTNER,2501 N FAIRWAY ROAD,990198573,USA,SHANE COX,SCOX@STAUFFER-CPA.COM,STAUFFER & ASSOCIATES PLLC,,811196633,,15,2024-01-21,2021-06-01,2021-06-01,2021-06-02,2021-06-01,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"84, 15, 10",tribal,,annual,1522989,UG,true,CENSUS,2021-06-01,SHANE COX,PARTNER +116000,2020-12-CENSUS-0000238710,MISSISSIPPI COUNTY FEDERAL AWARDS,TREASURER,PEGGY MEATTE,TREASURER@MISSISSIPPICOUNTY.AR.ORG,MISSISSIPPI COUNTY FEDERAL AWARDS,8707622152,TREASURER,200 W WALNUT,BLYTHEVILLE,AR,716014331,GSA_MIGRATION,No,72315,8702387971,AR,WYNNE,CPA,P.O. BOX 1045,72396,USA,HODA WARD,HWARDCPA@HOTMAIL.COM,"MEYER & WARD, CPA",,710810734,,16,2024-01-21,2023-02-06,2023-01-16,2023-02-07,2023-02-06,2020-12-31,2020-01-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,No,"93, 21, 16",local,,annual,963087,UG,true,CENSUS,2023-02-06,HODA WARD,CPA +117000,2020-06-CENSUS-0000211765,CONTINUUM SUPPORTIVE HOUSING OF WEST HARTFORD HUD PROJ NO 017-HD016,CHIEF FINANCIAL OFFICER,"ANGELA WESTWOOD, CPA",AWESTWOOD@CONTINUUMCT.ORG,CONTINUUM SUPPORTIVE HOUSING OF WEST HARTFORD HUD PROJ NO 017-HD016,2035622264,CHIEF FINANCIAL OFFICER,109 LEGION AVENUE,NEW HAVEN,CT,061474439,GSA_MIGRATION,No,06519,8603475689,CT,MIDDLETOWN,PARTNER,505 MAIN STREET,06457,USA,"TERESA OPALACZ, CPA",TERESA@GDSCPAS.COM,"GUILMARTIN, DIPIRO & SOKOLOWSKI, LLC",,060971998,,14,2024-01-21,2021-01-21,2021-01-20,2021-01-22,2021-01-21,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,Yes,14,non-profit,,annual,1329953,UG,true,CENSUS,2021-01-21,"TERESA OPALACZ, CPA",PARTNER +118000,2020-06-CENSUS-0000235901,"OUR HOUSE, INC.",DIRECTOR OF OPERATIONS,MATT RUNGE,MATT.RUNGE@OURHOUSESHELTER.ORG,"OUR HOUSE, INC.",5013747383,DIRECTOR OF OPERATIONS,302 E. ROOSEVELT ROAD,LITTLE ROCK,AR,710653846,GSA_MIGRATION,No,72206,5012211000,AR,LITTLE ROCK,PRINCIPAL,11025 ANDERSON DR. SUITE 300,72212,USA,SARAH GENTRY,SGENTRY@HC-CPA.COM,HCJ CPAS & ADVISORS PLLC,,710650689,,14,2024-01-21,2021-01-19,2021-01-19,2021-01-20,2021-01-19,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"14, 97, 10, 17, 94, 93",non-profit,,annual,1372508,UG,true,CENSUS,2021-01-19,SARAH GENTRY,PRINCIPAL +119000,2020-09-CENSUS-0000228697,CITY OF LAKE WALES,FINANCE DIRECTOR,DOROTHY ABBOTT,DABBOTT@LAKEWALESFL.GOV,CITY OF LAKE WALES,8636784182,FINANCE DIRECTOR,201 W. CENTRAL AVE,LAKE WALES,FL,596000357,GSA_MIGRATION,No,33853,9412817212,FL,WINTER PARK,PARTNER,1031 W. MORSE BLVD,32789,USA,LORRI KIDDER,LKIDDER@CRICPA.COM,"CARR, RIGGS & INGRAM, LLC",,721396621,,20,2024-01-21,2021-05-02,2021-04-28,2021-05-03,2021-05-02,2020-09-30,2019-10-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"14, 45, 16, 20, 66, 21",local,,annual,3143823,UG,true,CENSUS,2021-05-02,LORRI KIDDER,PARTNER +120000,2020-06-CENSUS-0000199968,LA VERNA TERRACE HOUSING CORP DBA PROVENA LA VERNA TERRACE 073-EH057,HOUSING ACCOUNTING SUPPORT SUPERVISOR,ELIZABETH KING,NCRAUDITTEAM@NATIONALCHURCHRESIDENCES.ORG,LA VERNA TERRACE HOUSING CORP DBA PROVENA LA VERNA TERRACE 073-EH057,6142732179,HOUSING ACCOUNTING SUPPORT SUPERVISOR,2335 NORTH BANK DRIVE,COLUMBUS,OH,363438977,GSA_MIGRATION,No,43220,3178485700,IN,CARMEL,MEMBER,501 CONGRESSIONAL BLVD.,46032,USA,HEATHER PERRY,HPERRY@DOZ.NET,"DAUBY O'CONNOR & ZALESKI, LLC",,351750664,,14,2024-01-21,2020-09-24,2020-09-25,2020-09-25,2020-09-24,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,764033,UG,true,CENSUS,2020-09-24,HEATHER PERRY,MEMBER +121000,2020-06-CENSUS-0000151682,INDEPENDENT SCHOOL DISTRICT NO. 12,EXECUTIVE DIRECTOR OF BUSINESS SERVICES,PATRICK CHAFFEY,PCHAFFEY@ISD12.ORG,INDEPENDENT SCHOOL DISTRICT NO. 12,7637926001,EXECUTIVE DIRECTOR OF BUSINESS SERVICES,4707 NORTH ROAD,CIRCLE PINES,MN,416008811,GSA_MIGRATION,No,55014,9525450424,MN,MINNEAPOLIS,PRINCIPAL,"5353 WAYZATA BLVD, STE 410",55416,USA,AARON NIELSEN,ANIELSEN@MMKR.COM,"MALLOY, MONTAGUE, KARNOWSKI, RADOSEVICH & CO., P.A.",,411690382,,84,2024-01-21,2020-11-22,2020-11-21,2020-11-23,2020-11-22,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 84",local,,annual,3601530,UG,true,CENSUS,2020-11-22,AARON NIELSEN,PRINCIPAL +122000,2020-06-CENSUS-0000215794,SCIOTOVILLE COMMUNITY SCHOOL,TREASURER,KAREN BALLENGEE,KAREN.BALLENGEE@TARTAN.K12.OH.US,SCIOTOVILLE COMMUNITY SCHOOL,7407766777,TREASURER,224 MARSHALL STREET,SCIOTOVILLE,OH,311758579,GSA_MIGRATION,No,456625549,7402894131,OH,PIKETON,AUDIT DIRECTOR,528 SOUTH WEST STREET,456618042,USA,BRENDEN BALESTRA,BRENDEN.BALESTRA@BHMCPAGROUP.COM,"BHM CPA GROUP, INC.",,311413363,,84,2024-01-21,2020-12-12,2020-12-11,2020-12-13,2020-12-12,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 10",local,,annual,796416,UG,true,CENSUS,2020-12-12,BRENDEN BALESTRA,AUDIT DIRECTOR +123000,2020-06-CENSUS-0000169916,CAREER TECHNOLOGY CENTER OF LACKAWANNA COUNTY,CHIEF FINANCIAL OFFICER,JAMES MCHALE,JMCHALE@CTCLC.EDU,CAREER TECHNOLOGY CENTER OF LACKAWANNA COUNTY,5703468471,CHIEF FINANCIAL OFFICER,3201 ROCKWELL AVENUE,SCRANTON,PA,231709530,GSA_MIGRATION,No,18508,5708762300,PA,OLYPHANT,PARTNER,299 MAIN STREET - 2ND FLOOR,18447,USA,SEAN GRASSI,SGRASSI@RRCO.NET,ROBERT ROSSI AND COMPANY,,232385520,,84,2024-01-21,2021-03-03,2021-03-04,2021-03-03,2021-03-03,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"64, 84, 10, 17",local,,annual,1806528,UG,true,CENSUS,2021-03-03,SEAN GRASSI,PARTNER +124000,2019-06-CENSUS-0000248695,"MOUNTAIN POINTE APARTMENTS, LP DBA MOUNTAIN POINTE APTS 123-11269",CONTROLLER,QING CHEN,QING.CHEN@CPLC.ORG,"MOUNTAIN POINTE APARTMENTS, LP DBA MOUNTAIN POINTE APTS 123-11269",6022576788,CONTROLLER,1112 EAST BUCKEYE ROAD,PHOENIX,AZ,860971578,GSA_MIGRATION,No,85034,3178485700,IN,CARMEL,PRINCIPAL,501 CONGRESSIONAL BLVD.,46032,USA,KRISTEN KELLEHER,KKELLEHER@DOZ.NET,"DAUBY O'CONNOR & ZALESKI, LLC",,351750664,,14,2024-01-21,2019-10-02,2019-10-03,2019-10-03,2019-10-02,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,14,non-profit,,other,3360600,UG,true,CENSUS,2019-10-02,KRISTEN KELLEHER,PRINCIPAL +125000,2020-06-CENSUS-0000223716,REMINGTON COLLEGE,PRESIDENT,PAM BELL,PAM.BELL@REMINGTONCOLLEGE.EDU,REMINGTON COLLEGE,4075625501,PRESIDENT,7131 BUSINESS PARK LANE,LAKE MARY,FL,273339369,GSA_MIGRATION,No,32746,9122381001,GA,SAVANNAH,PARTNER,118 PARK OF COMMERCE DRIVE SUITE 200,31405,USA,DONYA J. FOWLER,DONYA.FOWLER@TJSDD.COM,TJS DEEMER DANA LLP,,582663273,84,,2024-01-21,2021-03-07,2021-03-08,2021-03-04,2021-03-07,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,84,higher-ed,,annual,74576483,UG,true,CENSUS,2021-03-07,DONYA J. FOWLER,PARTNER +126000,2020-03-CENSUS-0000194592,ONEIDA HOUSING AUTHORITY,EXECUTIVE DIRECTOR,ROBERT WALTERS,ONEIDAHOUSING@CNYMAIL.COM,ONEIDA HOUSING AUTHORITY,3153638450,EXECUTIVE DIRECTOR,226 FARRIER AVENUE,ONEIDA,NY,161036082,GSA_MIGRATION,No,134211667,5188277499,NY,GILBOA,CPA,126 MAYBIE ROAD,12076,USA,PATRICK BUEL,PBUEL@BUELCPA.COM,"BUEL CPA, P.C.",,200348081,,14,2024-01-21,2021-06-10,2021-06-11,2021-06-10,2021-06-10,2020-03-31,2019-04-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,local,,annual,1589036,UG,true,CENSUS,2021-06-10,PATRICK BUEL,CPA +127000,2020-11-CENSUS-0000246830,GRAMERCY PARK COOPERATIVE OF ST. PAUL,MANAGEMENT AGENT,CONNIE BEDNAREK,CBEDNAR2@FAIRVIEW.ORG,GRAMERCY PARK COOPERATIVE OF ST. PAUL,6128743460,MANAGEMENT AGENT,5688 BRENT AVENUE,INVER GROVE HEIGHTS,MN,411819879,GSA_MIGRATION,No,55076,9523451573,MN,EDINA,CPA,"7900 WEST 78TH STREET, SUITE 450",55439,USA,PAULA M MEIDL,PAULA@CASEYMENDEN.COM,"CASEY, MENDEN, FAUST & NELSON, PA",,411535741,,14,2024-01-21,2021-03-30,2021-03-30,2021-03-30,2021-03-30,2020-11-30,2019-12-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,6658025,UG,true,CENSUS,2021-03-30,PAULA M MEIDL,CPA +128000,2020-12-CENSUS-0000177899,"JUNEAU COUNTY, WISCONSIN",CPA - FINANCE DIRECTOR,LORI CHIPMAN,LCHIPMAN@CO.JUNEAU.WI.US,"JUNEAU COUNTY, WISCONSIN",6088479310,CPA - FINANCE DIRECTOR,220 EAST STATE STREET - ROOM 203,MAUSTON,WI,396005706,GSA_MIGRATION,No,53948,8883088281,WI,VIROQUA,AUDIT PARTNER,1315 BAD AXE COURT; PO BOX 271,546650271,USA,CARRIE LEONARD,CLEONARD@JOHNSONBLOCK.COM,"JOHNSON BLOCK & COMPANY, INC.",,391628949,,16,2024-01-21,2021-10-18,2021-10-15,2021-10-19,2021-10-18,2020-12-31,2020-01-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,No,"14, 93, 90, 16, 84, 21, 20, 10, 97",local,,annual,9382362,UG,true,CENSUS,2021-10-18,CARRIE LEONARD,AUDIT PARTNER +129000,2020-12-CENSUS-0000207657,ELLIS HOSPITAL (D/B/A ELLIS MEDICINE,CONTROLLER,ERIC BERGIN,BERGINE@ELLISMEDICINE.ORG,ELLIS HOSPITAL (D/B/A ELLIS MEDICINE,5186128645,CONTROLLER,1101 NOTT STREET,SCHENECTADY,NY,141338428,GSA_MIGRATION,No,12308,3154463600,NY,SYRCAUSE,PARTNER,5784 WIDEWATERS PARKWAY,13214,USA,SHANNON FLOOD,SFLOOD@FCC-CPA.COM,FUST CHARLES CHAMBERS LLP,,161226221,14,,2024-01-21,2021-10-13,2021-10-14,2021-10-13,2021-10-13,2020-12-31,2020-01-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"14, 10, 93, 16, 32, 84",non-profit,,annual,71511530,UG,true,CENSUS,2021-10-13,SHANNON FLOOD,PARTNER +130000,2020-12-CENSUS-0000217586,NIAGARA LUTHERAN DEVELOPMENT DBA GREENFIELD HEALTH AND REHABILITATION,PRESIDENT/CEO,CHRISTOPHER E KOENIG,PRESIDENT@NIAGARALUTHERAN.ORG,NIAGARA LUTHERAN DEVELOPMENT DBA GREENFIELD HEALTH AND REHABILITATION,7166840202,PRESIDENT/CEO,5949 BROADWAY STREET,LANCASTER,NY,161502150,GSA_MIGRATION,No,140869523,7164176877,NY,WHEATFIELD,PRINCIPAL,800 LIBERTY BUILDING,14120,USA,JOHN PETERSON,JOHN.PETERSON@FREEDMAXICK.COM,FREED MAXICK CPAS PC,,454051133,,14,2024-01-21,2021-04-22,2021-04-22,2021-04-19,2021-04-22,2020-12-31,2020-01-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"14, 93",non-profit,,annual,9142968,UG,true,CENSUS,2021-04-22,JOHN PETERSON,PRINCIPAL +131000,2020-06-CENSUS-0000204706,WESTMORELAND-FAYETTE WORKFORCE INVESTMENT BOARD,CHIEF FINANCIAL OFFICER,THOMAS WHETSEL,TWHETSEL@WESTFAYWIB.ORG,WESTMORELAND-FAYETTE WORKFORCE INVESTMENT BOARD,7247552145,CHIEF FINANCIAL OFFICER,145 PAVILION LANE,YOUNGWOOD,PA,251860049,GSA_MIGRATION,No,15697,7243123409,PA,NEW BRIGHTON,AUDIT MANAGER,1000 3RD AVENUE,15066,USA,KIMBERLY TURNLEY,K.TURNLEY@COMCAST.NET,MARK C TURNLEY CPA,,251570881,,17,2024-01-21,2021-03-29,2021-03-29,2021-03-30,2021-03-29,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 17, 93",non-profit,,annual,5679001,UG,true,CENSUS,2021-03-29,KIMBERLY TURNLEY,AUDIT MANAGER +132000,2020-06-CENSUS-0000161071,IREDELL-STATESVILLE BOARD OF EDUCATION,FINANCE DIRECTOR,MELISSA WIKE,MELISSA_WIKE@ISS.K12.NC.US,IREDELL-STATESVILLE BOARD OF EDUCATION,7048728931,FINANCE DIRECTOR,549 RACE STREET,STATESVILLE,NC,561744267,GSA_MIGRATION,No,18677,7048711477,NC,STATESVILLE,PARTNER,113 N CENTER STREET,28677,USA,JOHN ROSENTAL,JROSENTAL@CLCPLLC.COM,"COFFEY, LOVINS, AND COMPANY PLLC",,561929162,,84,2024-01-21,2020-11-02,2020-10-30,2020-11-02,2020-11-02,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 21, 84, 12",local,,annual,18625259,UG,true,CENSUS,2020-11-02,JOHN ROSENTAL,PARTNER +133000,2020-12-CENSUS-0000249241,"GOLDEN TERRACE VILLAGE IV, INC.",MANAGER,SHAYLA CHAPPELL,GOLDENTERRACE@SBCGLOBAL.NET,"GOLDEN TERRACE VILLAGE IV, INC.",3255731416,MANAGER,1607 52ND STREET,SNYDER,TX,752753824,GSA_MIGRATION,No,79549,4327583226,TX,SEMINOLE,AUDIT PARTNER,P.O. BOX 309,79360,USA,DEREK RAY,DEREK@DAVIS-RAY.COM,"DAVIS, RAY & CO., PC",,752057245,,14,2024-01-21,2021-03-29,2021-03-26,2021-03-30,2021-03-29,2020-12-31,2020-01-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,1634682,UG,true,CENSUS,2021-03-29,DEREK RAY,AUDIT PARTNER +134000,2020-06-CENSUS-0000214191,LAC COURTE OREILLES OJIBWE SCHOOL,ACCOUNTING DIRECTOR,DAWN DOYLE,DAWN.DOYLE@LCOOSK12.ORG,LAC COURTE OREILLES OJIBWE SCHOOL,7156348924,ACCOUNTING DIRECTOR,8575 N TREPANIA RD,HAYWARD,WI,391763080,GSA_MIGRATION,No,54843,9897321156,MI,GAYLORD,SENIOR AUDIT PARTNER,215 S COURT AVE,49735,USA,KURT TUCKER,KURT@MWP.US.COM,"MIDWEST PROFESSIONALS, PLLC",,383571340,,15,2024-01-21,2021-09-28,2021-09-29,2021-09-29,2021-09-28,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,Yes,Yes,No,Yes,750000,No,"15, 10, 93, 84",tribal,,annual,9102660,UG,true,CENSUS,2021-09-28,KURT TUCKER,SENIOR AUDIT PARTNER +135000,2020-06-CENSUS-0000133949,ROUND VALLEY UNIFIED SCHOOL DISTRICT,BUSINESS MANAGER,JESSICA WILLIAMSON,JWILLIAMSON@RVUSD.US,ROUND VALLEY UNIFIED SCHOOL DISTRICT,7079836171,BUSINESS MANAGER,76280 HIGH SCHOOL STREET,COVELO,CA,946002711,GSA_MIGRATION,No,95428,7079836171,CA,SAN DIEGO,PARTNER,348 OLIVE STREET,92103,USA,MICHAEL ASH,MASH@CHRISTYWHITE.COM,CHRISTY WHITE INC,,272956198,,84,2024-01-21,2021-03-31,2021-04-01,2021-03-31,2021-03-31,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 84",local,,annual,2618781,UG,true,CENSUS,2021-03-31,MICHAEL ASH,PARTNER +136000,2020-12-CENSUS-0000241605,HOUSING AUTHORITY OF THE BOROUGH OF GLASSBORO,EXECUTIVE DIRECTOR,KIMBERLY GOBER,KGOBER@HAGC.ORG,HOUSING AUTHORITY OF THE BOROUGH OF GLASSBORO,8568454959,EXECUTIVE DIRECTOR,181 DELSEA MANOR DRIVE,GLASSBORO,NJ,216008418,GSA_MIGRATION,No,08028,8564356200,NJ,WOODBURY,PARTNER,"6 NORTH BROAD STREET, SUITE 201",08096,USA,NINA SORELLE,NSORELLE@BOWMAN.CPA,"BOWMAN & COMPANY, LLP",,210658561,,14,2024-01-21,2021-07-27,2021-07-21,2021-07-28,2021-07-27,2020-12-31,2020-01-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 14",local,,annual,2490531,UG,true,CENSUS,2021-07-27,NINA SORELLE,PARTNER +137000,2020-06-CENSUS-0000192522,CITY OF MILES CITY,CITY CLERK,MARY ROWE,CITYCLERK@MILESCITY-MT.ORG,CITY OF MILES CITY,4068748609,CITY CLERK,PO BOX 910,MILES CITY,MT,816001292,GSA_MIGRATION,No,593010910,4066980022,MT,BILLINGS,SHAREHOLDER,2810 CENTRAL AVE SUITE B,59102,USA,CURTIS D WYSS,CURT@OLNESSCPA.COM,"OLNESS & ASSOCIATES, PC",,810485423,,20,2024-01-21,2021-02-10,2021-02-11,2021-02-11,2021-02-10,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,Yes,Yes,No,No,750000,No,"21, 93, 20, 97, 94",local,,annual,1393438,UG,true,CENSUS,2021-02-10,CURTIS D WYSS,SHAREHOLDER +138000,2020-06-CENSUS-0000157881,PENNS GROVE CARNEYS POINT REGIONAL SCHOOL DISTRICT,BUSINESS ADMINISTRATOR,BRIAN FERGUSON,BFERGUSON@PGCPSCHOOLS.ORG,PENNS GROVE CARNEYS POINT REGIONAL SCHOOL DISTRICT,8563583094,BUSINESS ADMINISTRATOR,100 IONA AVENUE,PENNS GROVE,NJ,216000282,GSA_MIGRATION,No,08069,8566293111,NJ,WILLIAMSTOWN,PARTNER,991 SOUTH BLACK HORSE PIKE,08094,USA,RAYMOND COLAVITA,ANTHONY@COLAVITA.NET,"NIGHTLINGER, COLAVITA, & VOLPA",,222617821,,93,2024-01-21,2021-01-24,2021-01-25,2021-01-25,2021-01-24,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,Yes,Yes,No,750000,No,"84, 10, 93",local,,annual,3700498,UG,true,CENSUS,2021-01-24,RAYMOND COLAVITA,PARTNER +139000,2020-08-CENSUS-0000174376,PHARR-SAN JUAN-ALAMO I.S.D.,CHIEF FINANCIAL OFFICER,JANET ROBLES,JANET.ROBLES@PSJAISD.US,PHARR-SAN JUAN-ALAMO I.S.D.,9563542019,CHIEF FINANCIAL OFFICER,601 E KELLY,PHARR,TX,746001876,GSA_MIGRATION,No,78577,9566863701,TX,MCALLEN,PARTNER,4100 N 23RD ST,78504,USA,AARON RIOS,ARIOS@CRICPA.COM,"CARR RIGGS & INGRAM, LLC",,721396621,84,,2024-01-21,2021-03-09,2021-03-10,2021-03-09,2021-03-09,2020-08-31,2019-09-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,2099022,Yes,"84, 93, 12, 17, 10",local,,annual,69967392,UG,true,CENSUS,2021-03-09,AARON RIOS,PARTNER +140000,2020-12-CENSUS-0000241399,ELICA HEALTH CENTERS,CONTROLLER,JAMES MOHRHERR,JMOHRHERR@ELICAHEALTH.ORG,ELICA HEALTH CENTERS,9165698484,CONTROLLER,"1860 HOWE AVENUE, SUITE 455",SACRAMENTO,CA,371424390,GSA_MIGRATION,No,95825,9169290540,CA,SACRAMENTO,PARTNER,"1760 CREEKSIDE OAKS DRIVE, SUITE 160",95833,USA,BRADLEY BARTELLS,BJB@MUNCPAS.COM,"MANN, URRUTIA, NELSON CPAS & ASSOCIATES, LLP",,200276349,,93,2024-01-21,2021-07-28,2021-07-29,2021-07-28,2021-07-28,2020-12-31,2020-01-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,93,non-profit,,annual,4347431,UG,true,CENSUS,2021-07-28,BRADLEY BARTELLS,PARTNER +141000,2020-12-CENSUS-0000193149,"SENIOR CITIZENS' Y-HOUSE OF BEAUMONT, INC.",MANAGEMENT AGENT,"JOSHUA W. ALLEN, SR.",RASHANDA@JALLENMGMT.COM,"SENIOR CITIZENS' Y-HOUSE OF BEAUMONT, INC.",4098338947,MANAGEMENT AGENT,1390 BROADWAY,BEAUMONT,TX,760512267,GSA_MIGRATION,No,77701,4098383755,TX,BEAUMONT,TWHIDDON@FMWCPAS.COM,1150 NORTH ELEVENTH ST.,77702,USA,TERRY S. WHIDDON,TWHIDDON@FMWCPAS.COM,"FMW, PC",,760154479,,14,2024-01-21,2021-04-05,2021-04-05,2021-04-06,2021-04-05,2020-12-31,2020-01-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,14,non-profit,,annual,2505978,UG,true,CENSUS,2021-04-05,TERRY S. WHIDDON,TWHIDDON@FMWCPAS.COM +142000,2019-06-CENSUS-0000160541,HOLLAND PATENT CENTRAL SCHOOL DISTRICT,SUPERINTENDENT,JASON EVANGELIST,JEVANGELIST@HPSCHOOLS.ORG,HOLLAND PATENT CENTRAL SCHOOL DISTRICT,3158657200,SUPERINTENDENT,9601 MAIN STREET,HOLLAND PATENT,NY,156002239,GSA_MIGRATION,No,13354,3153369220,NY,ROME,PARTNER,200 EAST GARDEN STREET,13440,USA,ROY J. CLARK,CCLARK@DARCANGELO-CNY.COM,"D'ARCANGELO & CO., LLP",,132550103,,84,2024-01-21,2019-11-18,2019-11-15,2019-11-18,2019-11-18,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 84",local,,annual,1065188,UG,true,CENSUS,2019-11-18,ROY J. CLARK,PARTNER +143000,2019-06-CENSUS-0000004952,"THE BOSTON HEALTHCARE FOR THE HOMELESS PROGRAM, INC.",CHIEF FINANCIAL OFFICER,AGNES LEUNG,ALEUNG@BHCHP.ORG,"THE BOSTON HEALTHCARE FOR THE HOMELESS PROGRAM, INC.",8576541100,CHIEF FINANCIAL OFFICER,780 ALBANY STREET,BOSTON,MA,043160480,GSA_MIGRATION,No,01581,5083669789,MA,WESTBOROUGH,VICE PRESIDENT,50 WASHINGTON STREET,01581,USA,SORIE M. KABA,SKABA@AAFCPA.COM,"ALEXANDER, ARONSON, FINNING & CO., P.C.",,042571780,,93,2024-01-21,2019-11-19,2019-12-16,2019-12-17,2019-11-19,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 94, 93",non-profit,,annual,5777532,UG,true,CENSUS,2019-11-19,SORIE M. KABA,VICE PRESIDENT +144000,2020-06-CENSUS-0000249289,FORT TOTTEN PUBLIC SCHOOL DISTRICT #30,BUSINESS MANAGER,MEGAN A CALLAHAN,MEGAN.A.CALLAHAN@K12.ND.US,FORT TOTTEN PUBLIC SCHOOL DISTRICT #30,7017661400,BUSINESS MANAGER,HWY 57 S,FORT TOTTEN,ND,456000113,GSA_MIGRATION,No,58301,7012525422,ND,JAMESTOWN,CPA,111 9TH ST SW,58401,USA,DUANE R DUNN,D.DUNN@SCHAUERCPA.COM,"SCHAUER & ASSOCIATES, P.C.",,450350118,,84,2024-01-21,2021-03-31,2021-03-30,2021-03-31,2021-03-31,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,Yes,84,local,,annual,2418519,UG,true,CENSUS,2021-03-31,DUANE R DUNN,CPA +145000,2020-09-CENSUS-0000142159,SOUTH CENTRAL ILLINOIS REGIONAL PLANNING & DEVELOPMENT COMMISSION,EXECUTIVE DIRECTOR,JAMES PATRICK,JPATRICK@SCIRPDC.COM,SOUTH CENTRAL ILLINOIS REGIONAL PLANNING & DEVELOPMENT COMMISSION,6185484234,EXECUTIVE DIRECTOR,120 DELMAR AVE.,SALEM,IL,370974030,GSA_MIGRATION,No,62882,6182673213,IL,ODIN,PRESIDENT,5859 US HWY 51,62870,USA,SUSAN J LYONS,SLYONS1007@GMAIL.COM,SUSAN J LYONS CPA PC,,364149296,,10,2024-01-21,2021-01-28,2021-01-29,2021-01-28,2021-01-28,2020-09-30,2019-10-01,2020,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,No,"20, 11, 10, 59",local,,annual,2131397,UG,true,CENSUS,2021-01-28,SUSAN J LYONS,PRESIDENT +146000,2019-12-CENSUS-0000181260,ST MATTHEW HOUSING DEVELOPMENT INC T/A ST MATTHEW MANOR,MANAGING AGENT,CHARLES MENSCH,CMENSCH@COMMUNITYREALTYMANAGEMENT.COM,ST MATTHEW HOUSING DEVELOPMENT INC T/A ST MATTHEW MANOR,6096468861,MANAGING AGENT,230-242 NORTH 57TH STREET,PHILADELPHIA,PA,222813779,GSA_MIGRATION,No,19139,6096414000,NJ,NORTHFIELD,PARTNER,332 TILTON ROAD,08225,USA,JEFFREY WILSON,JAWILSON@CAPALDIREYNOLDS.COM,"CAPALDI REYNOLDS & PELOSI, P.A.",,222097516,,14,2024-01-21,2020-04-23,2020-04-24,2020-04-24,2020-04-23,2019-12-31,2018-12-31,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,5511201,UG,true,CENSUS,2020-04-23,JEFFREY WILSON,PARTNER +147000,2019-06-CENSUS-0000222692,"PORT CITY VOA HOUSING, INC. 062-HD050",SENIOR ACCOUNTING MANAGER,CINDY FULDORD,CFULFORD@VOASE.ORG,"PORT CITY VOA HOUSING, INC. 062-HD050",2513003500,SENIOR ACCOUNTING MANAGER,1204 HILLCREST ROAD,MOBILE,AL,753114572,GSA_MIGRATION,No,36695,2513431200,AL,MOBILE,PARTNER,3800 AIRPORT BLVD. STE 101,36608,USA,ZACHARY WOLF,ZACK@SMITHDUKES.COM,"SMITH, DUKES & BUCKALEW, LLP",,630191630,,14,2024-01-21,2019-10-18,2019-10-18,2019-10-19,2019-10-18,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,1009643,UG,true,CENSUS,2019-10-18,ZACHARY WOLF,PARTNER +148000,2019-12-CENSUS-0000213472,"MEEKER SENIOR HOUSING, INC.",MANAGEMENT AGENT,"LYNCO, INC.",DEBRA@LYNCOPROPERTIES.COM,"MEEKER SENIOR HOUSING, INC.",9184888485,MANAGEMENT AGENT,209 W GREEN ST,MEEKER,OK,731396847,GSA_MIGRATION,No,74855,9182612758,OK,TULSA,CPA,6851 S CANTON,74136,USA,JOHN FLUSCHE,CPA@OKLAHOMA.NET,"JOHN FLUSCHE, CPA",,731513650,,14,2024-01-21,2020-04-27,2020-04-28,2020-04-28,2020-04-27,2019-12-31,2018-12-31,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,1173058,UG,true,CENSUS,2020-04-27,JOHN FLUSCHE,CPA +149000,2019-12-CENSUS-0000249807,LOUP VALLEYS RURAL PUBLIC POWER DISTRICT,OFFICE MANAGER,JOSH ROSBERG,JOSH@LOUPVALLEYSPOWER.COM,LOUP VALLEYS RURAL PUBLIC POWER DISTRICT,3087283633,OFFICE MANAGER,606 S STREET,ORD,NE,476000977,GSA_MIGRATION,No,68862,4023915336,NE,OMAHA,CPA,2711 SOUTH 88TH STREET,68124,USA,LAURA RIGGS,FREEMNRIGG@AOL.COM,FREEMAN & RIGGS,,470784935,,97,2024-01-21,2020-10-08,2020-10-09,2020-10-08,2020-10-08,2019-12-31,2018-12-31,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,97,local,,annual,9332446,UG,true,CENSUS,2020-10-08,LAURA RIGGS,CPA +150000,2019-06-CENSUS-0000158282,BERNALILLO PUBLIC SCHOOL DISTRICT,FINANCE DIRECTOR,VICKIE GRACIA,VGARCIA@BPS.K12.NM.US,BERNALILLO PUBLIC SCHOOL DISTRICT,5054045689,FINANCE DIRECTOR,224 N. CAMINO DEL PUEBLO,BERNALLIO,NM,856000791,GSA_MIGRATION,No,87004,5059984468,NM,ALBUQUERQUE,AUDIT SENIOR,1030 18TH STREET NW,87104,USA,ESTHER ALEJO,ESTHER.ALEJO@RICCICPA.COM,RICCI & COMPANY LLC CPA,,205949532,,84,2024-01-21,2020-03-29,2020-03-26,2020-03-30,2020-03-29,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,Yes,No,No,Yes,750000,Yes,"10, 93, 84",state,,annual,13041709,UG,true,CENSUS,2020-03-29,ESTHER ALEJO,AUDIT SENIOR +151000,2019-06-CENSUS-0000133772,RICHLAND UNION SCHOOL DISTRICT,CHIEF BUSINESS OFFICER,MARTIN RODRIGUEZ,MRODRIGUEZ@RSDSHAFTER.ORG,RICHLAND UNION SCHOOL DISTRICT,6617468610,CHIEF BUSINESS OFFICER,331 SHAFTER AVE,SHAFTER,CA,956002546,GSA_MIGRATION,No,93263,5104525051,CA,SAN DIEGO,MANAGING PARTNER,"16870 W BERNARDO DR #250, , CA 92127",92127,USA,SANWAR HARSHWAL,SANWAR@HARSHWAL.COM,"HARSHWAL& COMPANY, LLP",,270741376,,10,2024-01-21,2020-03-29,2020-03-30,2020-03-30,2020-03-29,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 10, 84",local,,annual,5559395,UG,true,CENSUS,2020-03-29,SANWAR HARSHWAL,MANAGING PARTNER +152000,2019-06-CENSUS-0000189571,CITY OF PITTSBURG,FINANCIAL REPORTING MANAGER,LAURA MENDEZ,LMENDEZ@CI.PITTSBURG.CA.US,CITY OF PITTSBURG,9252524872,FINANCIAL REPORTING MANAGER,65 CIVIC AVENUE,PITTSBURG,CA,946000395,GSA_MIGRATION,No,94565,9259300902,CA,PLEASANT HILL,PARTNER,"3478 BUSKIRK AVENUE, SUITE 215",94523,USA,VIKKI RODRIGUEZ,VIKR@MAZEASSOCIATES.COM,MAZE & ASSOCIATES ACCOUNTANCY,,942590179,,14,2024-01-21,2020-02-16,2020-02-17,2020-02-14,2020-02-16,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"17, 16, 66, 14, 20",local,,annual,19729027,UG,true,CENSUS,2020-02-16,VIKKI RODRIGUEZ,PARTNER +153000,2019-12-CENSUS-0000244861,LORETTO APTS AT O'BRIEN ROAD HDFC INC (O'BRIEN RD SR APPTS 2) 014EE287,V.P. OF FINANCE (MANAGING AGENT),DAVID R. FILIPSKI,DFILIPSKI@CHRISTOPHER-COMMUNITY.ORG,LORETTO APTS AT O'BRIEN ROAD HDFC INC (O'BRIEN RD SR APPTS 2) 014EE287,3154241821,V.P. OF FINANCE (MANAGING AGENT),7174 O'BRIEN ROAD,SYRACUSE,NY,462450456,GSA_MIGRATION,No,13209,3154719171,NY,SYRACUSE,MEMBER,443 N FRANKLIN STREET,13204,USA,DAVID J. DE SILVA,DJD@DBBLLC.COM,"DERMODY, BURKE & BROWN, CPAS, LLC",,010723685,,14,2024-01-21,2020-03-15,2020-03-14,2020-03-16,2020-03-15,2019-12-31,2018-12-31,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,3422426,UG,true,CENSUS,2020-03-15,DAVID J. DE SILVA,MEMBER +154000,2019-06-CENSUS-0000160714,GREATER SOUTHERN TIER BOCES,ASSISTANT SUPERINTENDENT FOR FINANCE,T. GREGORY DALE,GDALE@GSTBOCES.ORG,GREATER SOUTHERN TIER BOCES,6077393581,ASSISTANT SUPERINTENDENT FOR FINANCE,459 PHILO ROAD,ELMIRA,NY,760814144,GSA_MIGRATION,No,14903,6072724444,NY,ITHACA,PARTNER,"401 EAST STATE STREET, SUITE 500",14850,USA,LESLIE SPURGIN,DCF@INSEROCPA.COM,"INSERO & CO. CPAS, LLP",,475324570,,84,2024-01-21,2019-10-21,2019-10-22,2019-10-22,2019-10-21,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 10",local,,annual,2065783,UG,true,CENSUS,2019-10-21,LESLIE SPURGIN,PARTNER +155000,2019-06-CENSUS-0000227008,"POSITIVELY THIRD STREET HOUSING DEVELOPMENT FUND CO.,INC. 012-EE287",VICE PRESIDENT OF FINANCE,MICHAEL KLIDAS,MKLIDAS@JASA.ORG,"POSITIVELY THIRD STREET HOUSING DEVELOPMENT FUND CO.,INC. 012-EE287",2122735257,VICE PRESIDENT OF FINANCE,"247 WEST 37TH STREET, 9TH FLOOR",NEW YORK,NY,300054211,GSA_MIGRATION,No,10018,3019615546,NY,NEW YORK,PARTNER,1301 AVENUE OF THE AMERICAS,10019,USA,WINELL BELFONTE,WINELL.BELFONTE@COHNREZNICK.COM,COHNREZNICK LLP,,221478099,,14,2024-01-21,2019-11-19,2019-12-23,2019-12-23,2019-11-19,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,8815693,UG,true,CENSUS,2019-11-19,WINELL BELFONTE,PARTNER +156000,2019-09-CENSUS-0000221617,"ADAMS COUNTY HEALTH CENTER, INC.",CFO,MARY ANN DOMECQ,MARYANND@ACHCID.ORG,"ADAMS COUNTY HEALTH CENTER, INC.",2082534242,CFO,PO BOX 428,COUNCIL,ID,208341138,GSA_MIGRATION,No,83612,5092420874,WA,SPOKANE VALLEY,OWNER,12015 EAST MAIN AVENUE,99206,USA,SHAUN JOHNSON,SJOHNSON@DZACPA.COM,"DINGUS, ZARECOR AND ASSOCIATES PLLC",,200079326,,93,2024-01-21,2020-01-12,2020-01-10,2020-01-13,2020-01-12,2019-09-30,2018-09-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,93,non-profit,,annual,1333589,UG,true,CENSUS,2020-01-12,SHAUN JOHNSON,OWNER +157000,2019-09-CENSUS-0000056721,MICHIGAN PEER REVIEW ORGANIZATION,SR. V-P AND CHIEF ADMINISTRATIVE OFFICER,FRANK FOX,FFOX@MPRO.ORG,MICHIGAN PEER REVIEW ORGANIZATION,2484657300,SR. V-P AND CHIEF ADMINISTRATIVE OFFICER,"22670 HAGGERTY ROAD, SUITE 100",FARMINGTON HILLS,MI,382536610,GSA_MIGRATION,No,483352611,3139652655,MI,DETROIT,PRINCIPAL,"1200 BUHL BUILDING, 535 GRISWOLD",482263689,USA,"MICHAEL R. NICHOLAS, C.P.A.",MNICHOLAS@GJC-CPA.COM,GEORGE JOHNSON & COMPANY,,382029668,,93,2024-01-21,2020-02-09,2020-02-10,2020-02-10,2020-02-09,2019-09-30,2018-09-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,93,non-profit,,annual,3521000,UG,true,CENSUS,2020-02-09,"MICHAEL R. NICHOLAS, C.P.A.",PRINCIPAL +158000,2019-12-CENSUS-0000227812,ALLIANCE GOOD SAMARITAN HSG DBA WILD FLOWER TERRACE 103-EE035,VICE PRESIDENT,ERIC VANDEN HULL,SBEAMAN@GOOD-SAM.COM,ALLIANCE GOOD SAMARITAN HSG DBA WILD FLOWER TERRACE 103-EE035,6053230921,VICE PRESIDENT,4800 W 57TH ST,SIOUX FALLS,SD,204714573,GSA_MIGRATION,No,57108,6053391999,SD,SIOUX FALLS,PARTNER,"200 E 10TH ST, STE 500",57104,USA,ANGIE HILLESTAD,AHILLESTAD@EIDEBAILLY.COM,EIDE BAILLY LLP,,450250958,,14,2024-01-21,2020-04-22,2020-04-23,2020-04-23,2020-04-22,2019-12-31,2018-12-31,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,2206024,UG,true,CENSUS,2020-04-22,ANGIE HILLESTAD,PARTNER +159000,2019-09-CENSUS-0000053662,"EAST CENTRAL ILLINOIS AREA AGENCY ON AGING, INC.",DEPUTY DIRECTOR,SUSAN REDMAN,SREDMAN@ECIAAA.ORG,"EAST CENTRAL ILLINOIS AREA AGENCY ON AGING, INC.",3098292065,DEPUTY DIRECTOR,1003 MAPLE HILL ROAD,BLOOMINGTON,IL,370982325,GSA_MIGRATION,No,617059327,2177933363,IL,SPRINGFIELD,DIRECTOR,"3201 WEST WHITE OAKS DRIVE, SUITE 102",62704,USA,"TERI TAYLOR, CPA",TERI.TAYLOR@SIKICH.COM,SIKICH LLP,,363168081,,93,2024-01-21,2020-04-02,2020-04-02,2020-04-02,2020-04-02,2019-09-30,2018-09-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,93,non-profit,,annual,3692515,UG,true,CENSUS,2020-04-02,"TERI TAYLOR, CPA",DIRECTOR +160000,2019-12-CENSUS-0000202475,CITY OF ALBERT LEA,FINANCE DIRECTOR,KRISTI BRUTLAG,KBRUTLAG@CI.ALBERTLEA.MN.US,CITY OF ALBERT LEA,5073774305,FINANCE DIRECTOR,221 E. CLARK ST,ALBERT LEA,MN,416004922,GSA_MIGRATION,No,56007,9525636873,MN,BLOOMINGTON,PARTNER,3800 AMERICAN BLVD W,55431,USA,MATT MAYER,MATT.MAYER@BERGANKDV.COM,"BERGANKDV, LTD",,411431613,,66,2024-01-21,2020-06-17,2020-06-18,2020-06-18,2020-06-17,2019-12-31,2018-12-31,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"15, 20, 66, 14, 16",local,,annual,1591450,UG,true,CENSUS,2020-06-17,MATT MAYER,PARTNER +161000,2019-12-CENSUS-0000139132,"CITY OF GALESBURG, ILLINOIS",DIRECTOR OF FINANCE AND INFORMATION SYST,GLORIA OSBORN,GOSBORN@CI.GALESBURG.IL.US,"CITY OF GALESBURG, ILLINOIS",3093453677,DIRECTOR OF FINANCE AND INFORMATION SYST,55 WEST TOMPKINS STREET,GALESBURG,IL,376001160,GSA_MIGRATION,No,61401,6306456205,IL,OAK BROOK,PARTNER,"1301 W .22ND STREET, STE 400",60523,USA,JASON COYLE,JASON.COYLE@BAKERTILLY.COM,BAKER TILLY VIRCHOW KRAUSE LLP,,390859910,,16,2024-01-21,2020-08-13,2020-08-14,2020-08-14,2020-08-13,2019-12-31,2018-12-31,2019,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,No,"14, 97, 16, 20, 66, 10",local,,annual,2544113,UG,true,CENSUS,2020-08-13,JASON COYLE,PARTNER +162000,2019-06-CENSUS-0000165855,MCLOUD PUBLIC SCHOOLS,SUPERINTENDENT,STEVE STANLEY,SSTANLEY@MCLOUDSCHOOLS.US,MCLOUD PUBLIC SCHOOLS,4059643314,SUPERINTENDENT,PO BOX 240,MCLOUD,OK,730939118,GSA_MIGRATION,No,74851,9183664440,OK,BIXBY,PARTNER,116 W BRECKENRIDGE,74008,USA,JACK JENKINS,JHJCPA@JACKHJENKINSCPA.COM,"JENKINS & KEMPER, CPAS, P.C.",,731514558,,84,2024-01-21,2020-03-30,2020-03-30,2020-03-31,2020-03-30,2019-06-30,2018-06-30,2019,single-audit,not_gaap,regulatory_basis,Yes,qualified_opinion,No,No,No,No,No,750000,No,"10, 84",state,,annual,1945179,UG,true,CENSUS,2020-03-30,JACK JENKINS,PARTNER +163000,2019-06-CENSUS-0000131980,CITY OF SANGER,ADMINISTRATIVE SERVICE DIRECTOR,BRET HARMON,BHARMON@CI.SANGER.CA.US,CITY OF SANGER,5598766300,ADMINISTRATIVE SERVICE DIRECTOR,1700 7TH STREET,SANGER,CA,946000425,GSA_MIGRATION,No,93657,9169290540,CA,SACRAMENTO,PARTNER,"1760 CREEKSIDE OAKS DRIVE, SUITE 160",95833,USA,JUSTIN WILLIAMS,JJW@MUNCPAS.COM,"MANN, URRUTIA, NELSON CPAS & ASSOCIATES",,200276349,,20,2024-01-21,2020-03-04,2020-07-22,2020-07-24,2020-03-04,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,No,"16, 20, 93, 15, 14",local,,annual,1206157,UG,true,CENSUS,2020-03-04,JUSTIN WILLIAMS,PARTNER +164000,2019-06-CENSUS-0000134527,EXETER UNIFIED SCHOOL DISTRICT,BUSINESS MANAGER,KRISTEN KIRK,KKIRK@EXETER.K12.CA.US,EXETER UNIFIED SCHOOL DISTRICT,5595929421,BUSINESS MANAGER,215 N. CRESPI AVE,EXETER,CA,462351710,GSA_MIGRATION,No,93221,5595584112,CA,FRESNO,PARTNER,"6051 N. FRESNO STREET, SUITE 101",93710,USA,BILL WILLIAMS,BCWILLIAMS@EIDEBAILLY.COM,EIDE BAILLY LLP,,450250958,,84,2024-01-21,2019-11-19,2019-12-23,2019-12-30,2019-11-19,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 10",local,,annual,2992667,UG,true,CENSUS,2019-11-19,BILL WILLIAMS,PARTNER +165000,2019-12-CENSUS-0000248475,NATIVE VILLAGE OF TAZLINA,TRIBAL ADMINISTRATOR,MARCIE SIMEON,TAZLINA.TRIBAL@GMAIL.COM,NATIVE VILLAGE OF TAZLINA,9078224375,TRIBAL ADMINISTRATOR,P.O. BOX 87,GLENALLEN,AK,920068390,GSA_MIGRATION,No,99588,9072742992,AK,ANCHORAGE,PRINCIPAL,"3000 C ST. N., SUITE 201",99503,USA,JOSEPH V. BERGENE,FEDCENSUS@ALTROGCO.COM,"ALTMAN, ROGERS & CO.",,920143182,,12,2024-01-21,2020-07-23,2020-07-24,2020-07-24,2020-07-23,2019-12-31,2018-12-31,2019,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,No,"66, 20, 39, 15, 12",tribal,,annual,933053,UG,true,CENSUS,2020-07-23,JOSEPH V. BERGENE,PRINCIPAL +166000,2019-06-CENSUS-0000099289,SOUTHEASTERN LOUISIANA AREA HEALTH EDUCATION CENTER,CFO,CHERYL TANNER,CHERYL.TANNER@SELAHEC.ORG,SOUTHEASTERN LOUISIANA AREA HEALTH EDUCATION CENTER,9853451119,CFO,1302 JW DAVIS DRIVE,HAMMOND,LA,721155014,GSA_MIGRATION,No,70403,5048355522,LA,METAIRIE,DIRECTOR,"111 VETERANS MEMORIAL BLVD, SUITE 600",70005,USA,"LANCE MORAN, CPA",LMORAN@LAPORTE.COM,"LAPORTE, APAC",,721088864,,84,2024-01-21,2020-02-01,2020-01-30,2020-02-02,2020-02-01,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 84",non-profit,,annual,3489073,UG,true,CENSUS,2020-02-01,"LANCE MORAN, CPA",DIRECTOR +167000,2019-11-CENSUS-0000138661,"MCDONOUGH COUNTY, ILLINOIS",COUNTY CLERK,GRETCHEN DEJAYNES,GDEJAYNES@MCDONOUGHCOUNTYCLERK.ORG,"MCDONOUGH COUNTY, ILLINOIS",3098332474,COUNTY CLERK,ONE COURTHOUSE SQUARE #7,MACOMB,IL,376001537,GSA_MIGRATION,No,61455,2177933363,IL,SPRINGFIELD,DIRECTOR,"3201 WEST WHITE OAKS DRIVE, SUITE 102",62704,USA,"MEGAN COCHRAN, CPA",MEGAN.COCHRAN@SIKICH.COM,SIKICH LLP,,363168081,,16,2024-01-21,2020-10-26,2020-10-22,2020-10-27,2020-10-26,2019-11-30,2018-11-30,2019,single-audit,unmodified_opinion,,,,No,Yes,No,No,Yes,750000,No,"93, 97, 20, 16, 10, 90, 66",local,,annual,856084,UG,true,CENSUS,2020-10-26,"MEGAN COCHRAN, CPA",DIRECTOR +168000,2019-09-CENSUS-0000192518,MARTIN HOUSING AUTHORITY,EXECUTIVE DIRECTOR,PAM BRATCHER,PAM@MARTINHOUSING.ORG,MARTIN HOUSING AUTHORITY,7315873186,EXECUTIVE DIRECTOR,134 EAST HEIGHTS DRIVE,MARTIN,TN,620719631,GSA_MIGRATION,No,38237,2059820992,AL,BIRMINGHAM,MEMBER,"200 CHASE PARK SOUTH, SUITE 220",35244,USA,"ROY W. HENDERSON, JR.",RHENDERSON@HENDERSONCPA.COM,"HENDERSON & PILLETERI, LLC",,263957106,,14,2024-01-21,2020-03-30,2020-03-31,2020-03-31,2020-03-30,2019-09-30,2018-09-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 97, 10, 14",local,,annual,1759625,UG,true,CENSUS,2020-03-30,"ROY W. HENDERSON, JR.",MEMBER +169000,2019-12-CENSUS-0000248376,"WILDBERRY MANOR PROJECT OF COMMUNITY HOUSING, INC.",MANAGER,BRENDA NABHOLZ,BNABHOLZ@HOTMAIL.COM,"WILDBERRY MANOR PROJECT OF COMMUNITY HOUSING, INC.",3193347110,MANAGER,302 1ST ST E,INDEPENDENCE,IA,420993731,GSA_MIGRATION,No,50644,5152258400,IA,WEST DES MOINES,PARTNER,"1601 22ND STREET, SUITE 400",50266,USA,MANDI HOLCOMB,MHOLCOMB@DENMAN-CPA.COM,"DENMAN & COMPANY, LLP",,420794029,,10,2024-01-21,2021-05-09,2020-08-04,2021-05-10,2021-05-09,2019-12-31,2018-12-31,2019,single-audit,unmodified_opinion,,,,No,Yes,Yes,No,No,750000,No,10,non-profit,,annual,984678,UG,true,CENSUS,2021-05-09,MANDI HOLCOMB,PARTNER +170000,2019-06-CENSUS-0000200348,"VICTIM OUTREACH INTERVENTION CENTER, INC.",EXECUTIVE DIRECTOR,LINDA STRACHAN,LINDAS@VOICEFORVICTIMS.COM,"VICTIM OUTREACH INTERVENTION CENTER, INC.",7242838700,EXECUTIVE DIRECTOR,"111 SOUTH CLIFF STREET, SUITE 1-A",BUTLER,PA,251364362,GSA_MIGRATION,No,16001,4126359088,PA,PITTSBURGH,CPA,"8150 PERRY HIGHWAY, SUITE 105",15237,USA,JAMES R. HUNE,JHUNE@CREESESMITH.COM,"CREESE, SMITH & CO. LLC",,251801538,,14,2024-01-21,2019-11-19,2019-11-19,2019-11-18,2019-11-19,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 16, 14",non-profit,,annual,1288547,UG,true,CENSUS,2019-11-19,JAMES R. HUNE,CPA +171000,2019-06-CENSUS-0000195886,LAKE VILLA COMMUNITY CONSOLIDATED SCHOOL DISTRICT NO 41,BUSINESS MANAGER,ANNA KASPRZYK,ANNA.KASPRZYK@DISTRICT41.ORG,LAKE VILLA COMMUNITY CONSOLIDATED SCHOOL DISTRICT NO 41,8472458010,BUSINESS MANAGER,131 MCKINLEY AVENUE,LAKE VILLA,IL,366004850,GSA_MIGRATION,No,60046,8153441300,IL,MCHENRY,PARTNER,"5400 WEST ELM STREET, SUITE 203",60050,USA,CHERYDEN JUERGENSEN,CPAS@EDERCASELLA.COM,"EDER, CASELLA & CO",,363614997,,84,2024-01-21,2019-10-07,2019-10-08,2019-10-07,2019-10-07,2019-06-30,2018-06-30,2019,single-audit,"adverse_opinion, not_gaap",regulatory_basis,No,unmodified_opinion,No,No,No,No,No,750000,No,"93, 10, 84",local,,annual,1183418,UG,true,CENSUS,2019-10-07,CHERYDEN JUERGENSEN,PARTNER +172000,2019-06-CENSUS-0000211402,"PRO-YOUTH, INC.",CHIEF FINANCIAL OFFICER,DOLORES TAYLOR,DTAYLOR@PYHEART.ORG,"PRO-YOUTH, INC.",5597989135,CHIEF FINANCIAL OFFICER,505 N. COURT STREET,VISALIA,CA,770337714,GSA_MIGRATION,No,93291,6618347411,CA,BAKERSFIELD,PARTNER,300 NEW STINE RD,93308,USA,NANCY BELTON,NANCY@DPVB.COM,DANIELLS PHILLIPS VAUGHAN AND BOCK,,952972229,,84,2024-01-21,2020-03-19,2020-03-19,2020-03-20,2020-03-19,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 10",non-profit,,annual,5362587,UG,true,CENSUS,2020-03-19,NANCY BELTON,PARTNER +173000,2019-06-CENSUS-0000170093,CONEMAUGH TOWNSHIP AREA SCHOOL DISTRICT,BUSINESS MANAGER,REGINA REMBOLD,REGINA.REMBOLD@CTASD.ORG,CONEMAUGH TOWNSHIP AREA SCHOOL DISTRICT,8144797431,BUSINESS MANAGER,300 WEST CAMPUS AVENUE,DAVIDSVILLE,PA,256004241,GSA_MIGRATION,No,159280407,8142694912,PA,JOHNSTOWN,CERTIFIED PUBLIC ACCOUNTANT,334 BUDFIELD STREET SUITE 180,159043214,USA,CAROLYN MUSSELMAN,CMUSSELMAN@KOTZANCPA.COM,"KOTZAN CPA & ASSOCIATES, P.C.",,262055351,,84,2024-01-21,2020-01-07,2020-01-08,2020-01-08,2020-01-07,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 10, 84",local,,annual,763390,UG,true,CENSUS,2020-01-07,CAROLYN MUSSELMAN,CERTIFIED PUBLIC ACCOUNTANT +174000,2019-12-CENSUS-0000043477,"LABOR COUNCIL SENIOR HOUSING, INC. 042-11164",MANAGING AGENT,STEVEN BOTNICK,SBOTNICK@BOTNICKREALTY.COM,"LABOR COUNCIL SENIOR HOUSING, INC. 042-11164",3308674836,MANAGING AGENT,1653 MERRIMAN ROAD,AKRON,OK,341399884,GSA_MIGRATION,No,44313,3308677350,OH,AKRON,SENIOR MANAGER,1540 WEST MARKET STREET,44313,USA,CHRISTINE KLAIBER,CKLAIBER@APPLEGROWTH.COM,APPLE GROWTH PARTNERS,,341082617,,14,2024-01-21,2020-05-18,2020-05-19,2020-05-19,2020-05-18,2019-12-31,2018-12-31,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,2349626,UG,true,CENSUS,2020-05-18,CHRISTINE KLAIBER,SENIOR MANAGER +175000,2019-06-CENSUS-0000111765,"SER - JOBS FOR PROGRESS OF SOUTHERN ARIZONA, INC.",EXECUTIVE DIRECTOR,ERNEST URIAS,SER7@QWESTOFFICE.NET,"SER - JOBS FOR PROGRESS OF SOUTHERN ARIZONA, INC.",5206248629,EXECUTIVE DIRECTOR,40 WEST 28TH ST.,TUCSON,AZ,860251307,GSA_MIGRATION,No,85713,6022643077,AZ,SCOTTSDALE,PARTNER,9019 E BAHIA DRIVE SUITE 100,85260,USA,KEVIN CAMBERG,KCAMBERG@F-CPC.COM,"FESTER & CHAPMAN, PLLC",,821455657,,17,2024-01-21,2020-03-15,2020-03-16,2020-03-10,2020-03-15,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"17, 93, 14",non-profit,,annual,1680310,UG,true,CENSUS,2020-03-15,KEVIN CAMBERG,PARTNER +176000,2019-06-CENSUS-0000201775,"CITY OF SOUTH BURLINGTON, VERMONT",FINANCE OFFICER,SUE DOREY,SDOREY@SBURL.COM,"CITY OF SOUTH BURLINGTON, VERMONT",8028464104,FINANCE OFFICER,575 DORSET STREET,SOUTH BURLINGTON,VT,036002712,GSA_MIGRATION,No,05403,2079294606,ME,BUXTON,QUALITY CONTROL SPECIALIST,3 OLD ORCHARD ROAD,04093,USA,CATHY MARKAVICH,CMARKAVICH@RHRSMITH.COM,RHR SMITH & COMPANY,,043383155,,20,2024-01-21,2020-04-14,2021-01-18,2021-01-18,2020-04-14,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"16, 14, 20",local,,annual,4273532,UG,true,CENSUS,2020-04-14,CATHY MARKAVICH,QUALITY CONTROL SPECIALIST +177000,2019-12-CENSUS-0000228105,AIRPORT AUTHORITY FOR AIPORT DISTRICT #1 OF CALCASIEU PARISH,EXECUTIVE DIRECTOR,HEATH ALLEN,HALLEN@FLYLAKECHARLES.COM,AIRPORT AUTHORITY FOR AIPORT DISTRICT #1 OF CALCASIEU PARISH,3374776051,EXECUTIVE DIRECTOR,P O DRAWER 5820,LAKE CHARLES,LA,720551224,GSA_MIGRATION,No,70606,3375134915,LA,LAKE CHARLES,CPA,"2720 RUE DE JARDIN, SUITE 300",70605,USA,STEVEN M DEROUEN,STEVE@SDEROUENCPA.COM,STEVEN M DEROUEN & ASSOCIATES,,800488055,,20,2024-01-21,2020-07-26,2020-07-27,2020-07-27,2020-07-26,2019-12-31,2018-12-31,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"97, 20",local,,annual,2880293,UG,true,CENSUS,2020-07-26,STEVEN M DEROUEN,CPA +178000,2020-06-CENSUS-0000152122,HEADWATERS REGIONAL DEVELOPMENT COMMISSION,FINANCE DIRECTOR,NICOLE MCKINNON,NMCKINNON@HRDC.ORG,HEADWATERS REGIONAL DEVELOPMENT COMMISSION,2184444732,FINANCE DIRECTOR,1320 NIELSON AVE SE,BEMIDJI,MN,410983661,GSA_MIGRATION,No,56601,2187516300,MN,BEMIDJI,PARTNER,513 BELTRAMI AVENUE,56601,USA,JOSH SANDEN,JSANDEN@MILLERMCDONALD.COM,"MILLER MCDONALD, INC.",,411281737,,11,2024-01-22,2022-04-24,2022-04-25,2022-04-13,2022-04-24,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,Yes,No,No,Yes,750000,No,"20, 10, 11",local,,annual,3119109,UG,true,CENSUS,2022-04-24,JOSH SANDEN,PARTNER +179000,2019-06-CENSUS-0000145513,UNIFIED SCHOOL DISTRICT NO. 428,SUPERINTENDENT,KHRIS THEXTON,KHRIS.THEXTON@USD428.NET,UNIFIED SCHOOL DISTRICT NO. 428,6207931500,SUPERINTENDENT,201 S PATTON,GREAT BEND,KS,480721637,GSA_MIGRATION,No,67530,6207922428,KS,GREAT BEND,CPA-PARTNER,2006 BROADWAY SUITE 2A,67530,USA,VICKIE DREILING,VDREILING@ABBB.COM,"ADAMS, BROWN, BERAN & BALL, CHTD.",,481040139,,84,2024-01-22,2020-01-26,2020-01-27,2020-01-27,2020-01-26,2019-06-30,2018-06-30,2019,single-audit,not_gaap,regulatory_basis,No,unmodified_opinion,No,No,Yes,No,No,750000,No,"10, 93, 84",local,,annual,4126613,UG,true,CENSUS,2020-01-26,VICKIE DREILING,CPA-PARTNER +180000,2017-09-CENSUS-0000056213,"TURNING POINT, INC.",CFO,JULIA BINGHAM,JBINGHAM@TURNINGPOINTMACOMB.ORG,"TURNING POINT, INC.",5864634430,CFO,158 S. MAIN STREET,MOUNT CLEMENS,MI,382292020,GSA_MIGRATION,No,480432309,2483688868,MI,SOUTHFIELD,PARTNER,"2000 TOWN CENTER, SUITE 900",480751103,USA,LAURIE L. HORVATH,LAURIE.HORVATH@BAKERTILLY.COM,"BAKER TILLY VIRCHOW KRAUSE, LLP",,380859910,,16,2024-01-27,2018-04-03,2018-04-04,2018-03-29,2018-04-03,2017-09-30,2016-09-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 97, 14, 16, 10",non-profit,,annual,1410903,UG,true,CENSUS,2018-04-03,LAURIE L. HORVATH,PARTNER +181000,2019-09-CENSUS-0000216900,COMMUNITY APARTMENTS CORPORATION OF FORSYTH COUNTY #3,DIRECTOR OF HOUSING SERVICES,LISA PARDUE,GSA_MIGRATION,COMMUNITY APARTMENTS CORPORATION OF FORSYTH COUNTY #3,3362734404,DIRECTOR OF HOUSING SERVICES,353 E. SIX FORKS ROAD SUITE 300,RALEIGH,NC,562277204,GSA_MIGRATION,No,27609,3368544277,NC,GREENSBORO,SHAREHOLDER,445 DOLLEY MADISON ROAD SUITE 202,27410,USA,BRETT KOCEJA,BRETT@AKACPA.COM,"APPLE, KOCEJA & ASSOCIATES, PA",,561793406,,14,2024-01-23,2020-03-22,2020-03-22,2020-03-17,2020-03-22,2019-09-30,2018-09-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,771829,UG,true,CENSUS,2020-03-22,BRETT KOCEJA,SHAREHOLDER +182000,2019-06-CENSUS-0000244276,"THE ARC OF CAPE MAY COUNTY, INC AND AFFILIATE",DIRECTOR OF FINANCE,BARBARA MARCHIANO,BMARCHIANO@ARCOFCAPEMAY.ORG,"THE ARC OF CAPE MAY COUNTY, INC AND AFFILIATE",6098617100,DIRECTOR OF FINANCE,"822 ROUTE 47, P.O. BOX 255",SOUTH DENNIS,NJ,222271013,GSA_MIGRATION,No,08245,6096414000,NJ,NORTHFIELD,PARTNER,332 TILTON ROAD,08225,USA,DONNA BUZBY,DHBUZBY@CAPALDIREYNOLDS.COM,"CAPALDI REYNOLDS & PELOSI, CPA PA",,222097516,,14,2024-01-23,2020-02-25,2020-02-26,2020-01-15,2020-02-25,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,942033,UG,true,CENSUS,2020-02-25,DONNA BUZBY,PARTNER +183000,2018-06-CENSUS-0000031006,"NEW BRIDGE FOUNDATION, INC.",DIRECTOR OF ADMINISTRATION,NATHALIE GIREAUD FERKO,NATHALIE@NEWBRIDGEFOUNDATION.ORG,"NEW BRIDGE FOUNDATION, INC.",8007852400,DIRECTOR OF ADMINISTRATION,2323 HEARST AVENUE,BERKELEY,CA,237131209,GSA_MIGRATION,No,94709,5104525051,CA,OAKLAND,MANAGING PARTNER,7677 OAKPORT STREET,94621,USA,SANWAR HARSHWAL,SANWAR@HARSHWAL.COM,"HARSHWAL & COMPANY, LLP",,270741376,,93,2024-01-26,2019-03-24,2019-03-24,2019-03-22,2019-03-24,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,93,non-profit,,annual,1791503,UG,true,CENSUS,2019-03-24,SANWAR HARSHWAL,MANAGING PARTNER +184000,2018-03-CENSUS-0000224763,MAGNOLIA HEIGHTS HOUSING CORPORATION 084-HD051 M016Q041005,CHIEF FINANCIAL OFFICER,NATHAN MORDICA,NMMORDICA@FREEMANHEALTH.COM,MAGNOLIA HEIGHTS HOUSING CORPORATION 084-HD051 M016Q041005,4173477606,CHIEF FINANCIAL OFFICER,1105 EAST 32ND STREET,JOPLIN,MO,470950622,GSA_MIGRATION,No,64804,4178658701,MO,SPRINGFIELD,PARTNER,"910 E. SAINT LOUIS STREET, P.O. BOX 1190",658011190,USA,KRISTEN BRIGHT,KBRIGHT@BKD.COM,"BKD, LLP",,440160260,,14,2024-01-26,2018-07-02,2018-07-03,2018-06-28,2018-07-02,2018-03-31,2017-03-31,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,2422017,UG,true,CENSUS,2018-07-02,KRISTEN BRIGHT,PARTNER +185000,2018-06-CENSUS-0000091952,"BLUE GRASS COMMUNITY ACTION PARTNERSHIP, INC.",EXECUTIVE DIRECTOR,TROY ROBERTSQ,TROY.ROBERTS@BGCAP.ORG,"BLUE GRASS COMMUNITY ACTION PARTNERSHIP, INC.",5026954290,EXECUTIVE DIRECTOR,111 PROFESSIONAL COURT,FRANKFORT,KY,610659583,GSA_MIGRATION,No,40601,8592544427,KY,LEXINGTON,PARTNER,190 MARKET STREET,40507,USA,ANDREW DIERUF,ADIERUF@BANDDCPA.COM,"BESTEN & DIERUF, PLLC",,453048812,,93,2024-01-26,2018-12-11,2018-12-11,2018-12-12,2018-12-11,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 81, 97, 17, 94, 20, 93",non-profit,,annual,9674474,UG,true,CENSUS,2018-12-11,ANDREW DIERUF,PARTNER +186000,2018-06-CENSUS-0000191547,STARR KING SCHOOL FOR THE MINISTRY,VP OF FINANCE AND ADMINISTRATION,R. KELLEY GIBBS,KGIBBS@SKSM.EDU,STARR KING SCHOOL FOR THE MINISTRY,5105494702,VP OF FINANCE AND ADMINISTRATION,2441 LE CONTE AVENUE,BERKELEY,CA,941196217,GSA_MIGRATION,No,94709,4158196718,CA,SAN FRANCISCO,OWNER,PO BOX 641773,941641773,USA,KENNETH MIERZWINSKI,KEN@KPMCPAS.COM,KPM ACCOUNTING & MANAGEMENT SOLUTIONS,,453220824,,84,2024-01-26,2019-03-24,2019-03-24,2019-03-25,2019-03-24,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,84,non-profit,,annual,1010464,UG,true,CENSUS,2019-03-24,KENNETH MIERZWINSKI,OWNER +188000,2018-06-CENSUS-0000247990,"VOICES FOR CHILDREN FOUNDATION, INC.",CONTROLLER,ALAIN GARCIA,AGARCIA@BEAVOICE.ORG,"VOICES FOR CHILDREN FOUNDATION, INC.",7864693863,CONTROLLER,"601 NW 1ST CT, 10TH FL",MIAMI,FL,592746076,GSA_MIGRATION,No,33136,3058585800,FL,MIAMI,CERTIFIED PUBLIC ACCOUNTANT,"3225 AVIATION AVENUE, SUITE 500",33133,USA,DARWISH KAIYAL,GSA_MIGRATION,PINCHASIK YELEN MUSKAT STEIN,,274256127,,16,2024-01-26,2019-02-27,2019-02-28,2019-02-28,2019-02-27,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,16,non-profit,,annual,949658,UG,true,CENSUS,2019-02-27,DARWISH KAIYAL,CERTIFIED PUBLIC ACCOUNTANT +189000,2018-12-CENSUS-0000246173,"SAINT JAMES HEALTH, INC.",CFO,MARCUS FREEMAN,MFREEMAN@SJHEALTHNEWARK.COM,"SAINT JAMES HEALTH, INC.",9737898111,CFO,228 LAFAYETTE STREET,NEWARK,NJ,465221991,GSA_MIGRATION,No,07105,9734726250,NJ,CLIFTON,PARTNER,855 VALLEY ROAD,07013,USA,JOSH CHANANIE,JCHANANIE@SAXLLP.COM,SAX LLP,,818950760,,93,2024-01-26,2019-07-31,2019-08-01,2019-07-25,2019-07-31,2018-12-31,2017-12-31,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,93,non-profit,,annual,1079809,UG,true,CENSUS,2019-07-31,JOSH CHANANIE,PARTNER +190000,2018-12-CENSUS-0000192752,"ROCK VALLEY COMMUNITY PROGRAMS, INC.",EXECUTIVE DIRECTOR,ANGEL EGGERS,AEGGERS@RVCP.ORG,"ROCK VALLEY COMMUNITY PROGRAMS, INC.",6087414500,EXECUTIVE DIRECTOR,203 W. SUNNY LANE ROAD,JANESVILLE,WI,391438843,GSA_MIGRATION,No,535120932,8153162375,IL,ROCKFORD,PARTNER,6815 WEAVER ROAD SUITE SUITE 300,611148060,USA,CURTIS D. KLECKLER,CKLECKLER@BENNINGGROUP.COM,"BENNING GROUP, LLC",,300003878,,64,2024-01-26,2019-07-25,2019-07-25,2019-07-26,2019-07-25,2018-12-31,2017-12-31,2018,single-audit,unmodified_opinion,,,,Yes,No,No,No,Yes,750000,No,"93, 16, 64",non-profit,,annual,4451969,UG,true,CENSUS,2019-07-25,CURTIS D. KLECKLER,PARTNER +191000,2018-12-CENSUS-0000187800,TEXARKANA AIRPORT AUTHORITY,DIRECTOR,MARK MELLINGER,MARK.MELLINGER@TXKAIRPORT.COM,TEXARKANA AIRPORT AUTHORITY,8707742171,DIRECTOR,201 AIRPORT DRIVE,TEXARKANA,AR,716045823,GSA_MIGRATION,No,71854,9037935646,TX,TEXARKANA,SHAREHOLDER,1810 GALLERIA OAKS,75503,USA,"KELLY BIRDWELL, CPA",KELLY@WILHEN.COM,"WILF & HENDERSON, P.C.",,751726749,,20,2024-01-26,2019-07-28,2019-07-29,2019-07-29,2019-07-28,2018-12-31,2017-12-31,2018,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,No,20,local,,annual,759045,UG,true,CENSUS,2019-07-28,"KELLY BIRDWELL, CPA",SHAREHOLDER +192000,2018-06-CENSUS-0000211066,CLEVELAND HEIGHTS-UNIVERSITY HEIGHTS CITY SCHOOL DISTRICT,TREASURER,A. SCOTT GAINER,S_GAINER@CHUH.ORG,CLEVELAND HEIGHTS-UNIVERSITY HEIGHTS CITY SCHOOL DISTRICT,2163717171,TREASURER,2155 MIRAMAR BOULEVARD,UNIVERSITY HEIGHTS,OH,346000687,GSA_MIGRATION,No,44118,2167873665,OH,COLUMBUS,SENIOR AUDIT MANAGER,88 EAST BROAD STREET,43215,USA,MATTHEW GOLDMAN,MAGOLDMAN@OHIOAUDITOR.GOV,"DAVE YOST, AUDITOR OF STATE",,311334820,,84,2024-01-26,2019-01-27,2019-01-28,2019-01-26,2019-01-27,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,Yes,No,750000,Yes,"10, 84",local,,annual,8195043,UG,true,CENSUS,2019-01-27,MATTHEW GOLDMAN,SENIOR AUDIT MANAGER +193000,2017-10-CENSUS-0000182894,OBLATES RESIDENCES 042-11193-PM,VICE PRESIDENT,KATHLEEN CROWLEY,KCROWLEY@VMC.ORG,OBLATES RESIDENCES 042-11193-PM,4192422300,VICE PRESIDENT,1931 SCOTTWOOD AVENUE SUITE 700,TOLEDO,OH,341320478,GSA_MIGRATION,No,43620,3178485700,IN,CARMEL,PRINCIPAL,"501 CONGRESSIONAL BLVD, SUITE 300",46032,USA,KRISTEN KELLEHER,KKELLEHER@DOZ.NET,"DAUBY O'CONNOR & ZALESKI, LLC",,351750664,,14,2024-01-27,2018-01-23,2018-01-24,2018-01-24,2018-01-23,2017-10-31,2016-10-31,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,14,non-profit,,annual,3889656,UG,true,CENSUS,2018-01-23,KRISTEN KELLEHER,PRINCIPAL +194000,2018-06-CENSUS-0000187947,"GRAND PRAIRIE SERVICES, INC.",PRESIDENT & CEO,SHARRONNE WARD,SWARD@GPSBH.ORG,"GRAND PRAIRIE SERVICES, INC.",7084441012,PRESIDENT & CEO,17746 OAK PARK AVENUE,TINLEY PARK,IL,362362364,GSA_MIGRATION,No,60477,6304064490,IL,BATAVIA,PRESIDENT,201 HOUSTON STREET SUITE 301,60510,USA,PAUL WIELAND,PAUL@WIELANDCPAS.NET,"WIELAND & COMPANY, INC.",,364025026,,93,2024-01-26,2019-03-20,2019-03-21,2019-03-21,2019-03-20,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,93,non-profit,,annual,1213940,UG,true,CENSUS,2019-03-20,PAUL WIELAND,PRESIDENT +195000,2018-06-CENSUS-0000189752,"CHEROKEE FAMILY VIOLENCE CENTER, INC.",EXECUTIVE DIRECTOR,MEG ROGERS,MEG@CFVC.ORG,"CHEROKEE FAMILY VIOLENCE CENTER, INC.",7705477346,EXECUTIVE DIRECTOR,100 HEARTHSTONE LANDING DRIVE,CANTON,GA,581650925,GSA_MIGRATION,No,30114,7067782154,GA,CLARKESVILLE,AUDIT PARTNER,854 WASHINGTON STREET SUITE 200,30523,USA,JOELY MIXON,JOELY@MIXONCPA.COM,"MIXON, MIXON, BROWN & TENCH CPAS",,823104745,,16,2024-01-26,2018-12-25,2018-12-26,2018-12-26,2018-12-25,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"16, 93, 97, 14",non-profit,,annual,1076770,UG,true,CENSUS,2018-12-25,JOELY MIXON,AUDIT PARTNER +196000,2018-06-CENSUS-0000145974,POWELL COUNTY FISCAL COURT,COUNTY TREASURER,CONNIE CRABTREE,RCRABTREE33@HOTMAIL.COM,POWELL COUNTY FISCAL COURT,6066632834,COUNTY TREASURER,525 WASHINGTON STREET,STANTON,KY,616016937,GSA_MIGRATION,No,40380,5025645841,KY,FRANKFORT,ASSISTANT AUDITOR OF PUBLIC ACCOUNTS,209 ST. CLAIR STREET,40601,USA,FARRAH PETTER,FARRAH.PETTER@KY.GOV,"MIKE HARMON, KY AUDITOR OF PUBLIC ACCOUNTS",,610600439,,97,2024-01-26,2019-10-17,2019-10-18,2019-10-14,2019-10-17,2018-06-30,2017-06-30,2018,single-audit,not_gaap,regulatory_basis,No,unmodified_opinion,No,Yes,No,Yes,No,750000,No,"14, 20, 97",local,,annual,851352,UG,true,CENSUS,2019-10-17,FARRAH PETTER,ASSISTANT AUDITOR OF PUBLIC ACCOUNTS +197000,2018-06-CENSUS-0000083404,LIMESTONE COLLEGE,VP FOR FINANCIAL AFFAIRS,WAYDE DAWSON,LDAWSON@LIMESTONE.EDU,LIMESTONE COLLEGE,8644884522,VP FOR FINANCIAL AFFAIRS,1115 COLLEGE DRIVE,GAFFNEY,SC,570314402,GSA_MIGRATION,No,29340,8642405166,SC,GREENVILLE,DIRECTOR,"110 EAST COURT STREET, SUITE 500",29601,USA,ROSELLE BONNOITT,RBONNOITT@CBH.COM,CHERRY BEKAERT LLP,,560574444,,84,2024-01-26,2018-10-10,2018-10-11,2018-10-01,2018-10-10,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,84,non-profit,,annual,26788199,UG,true,CENSUS,2018-10-10,ROSELLE BONNOITT,DIRECTOR +198000,2018-09-CENSUS-0000172888,"CITY OF EULESS, TEXAS",DIRECTOR OF FINANCE,JANINA JEWELL,JJEWELL@EULESSTX.GOV,"CITY OF EULESS, TEXAS",8176851444,DIRECTOR OF FINANCE,201 N. ECTOR DRIVE,EULESS,TX,756004644,GSA_MIGRATION,No,76039,9724901970,TX,DALLAS,PARTNER,"2300 NORTH FIELD STREET, SUITE 1000",75201,USA,SARA DEMPSEY,SARA.DEMPSEY@WEAVER.COM,"WEAVER AND TIDWELL, L.L.P.",,750786316,,16,2024-01-26,2019-03-11,2019-03-12,2019-03-12,2019-03-11,2018-09-30,2017-09-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"16, 97, 66, 45, 20",local,,annual,801977,UG,true,CENSUS,2019-03-11,SARA DEMPSEY,PARTNER +199000,2018-06-CENSUS-0000166560,DOUGLAS EDUCATION SERVICE DISTRICT,FINANCE DIRECTOR,BARBARA TAYLOR,BARBARA.TAYLOR@DOUGLASESD.K12.OR.US,DOUGLAS EDUCATION SERVICE DISTRICT,5414404752,FINANCE DIRECTOR,1871 NE STEPHENS ST,ROSEBURG,OR,936000405,GSA_MIGRATION,No,974701433,5416724886,OR,ROSEBURG,CERTIFIED PUBLIC ACCOUNTANT,2500 W. HARVARD AVE,974712507,USA,JEFFREY R. COOLEY,JEFF@NEUNERDAVIDSON.COM,NEUNER DAVIDSON & CO,,930764156,,84,2024-01-26,2018-12-27,2018-12-26,2018-12-21,2018-12-27,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 93",local,,annual,2521742,UG,true,CENSUS,2018-12-27,JEFFREY R. COOLEY,CERTIFIED PUBLIC ACCOUNTANT +200000,2018-06-CENSUS-0000226783,MARLBOROUGH COMMUNITY DEVELOPMENT AUTHORITY,EXECUTIVE DIRECTOR,DOUGLAS BUSHMAN,DBUSHMAN@MARLBOROUGH-MA.GOV,MARLBOROUGH COMMUNITY DEVELOPMENT AUTHORITY,5086240980,EXECUTIVE DIRECTOR,"255 MAIN STREET, SUITE 212",MARLBOROUGH,MA,046001399,GSA_MIGRATION,No,01752,6173766226,MA,BRAINTREE,DIRECTOR,"100 GRANDVIEW ROAD, SUITE 320",02184,USA,MICHAEL GUYDER,MGUYDER@GUYDER-HURLEY.COM,"GUYDER HURLEY, PC",,043520245,,14,2024-01-26,2019-01-06,2019-01-04,2019-01-07,2019-01-06,2018-06-30,2017-06-30,2018,program-specific,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,local,,annual,1580926,UG,true,CENSUS,2019-01-06,MICHAEL GUYDER,DIRECTOR +201000,2018-06-CENSUS-0000233189,RICHMOND HEIGHTS LOCAL SCHOOL DISTRICT,TREASURER,COOPER MARTIN,MARTIN.COOPER@RICHMONDHEIGHTSSCHOOLS.ORG,RICHMOND HEIGHTS LOCAL SCHOOL DISTRICT,2166920089,TREASURER,447 RICHMOND ROAD,RICHMOND HEIGHTS,OH,346002300,GSA_MIGRATION,No,44143,2167873665,OH,COLUMBUS,SENIOR AUDIT MANAGER,88 EAST BROAD STREET,43215,USA,WILLIAM WARD,WJWARD@OHIOAUDITOR.GOV,"KEITH FABER, AUDITOR OF STATE",,311334820,,84,2024-01-26,2019-03-25,2019-03-26,2019-03-26,2019-03-25,2018-06-30,2017-06-30,2018,single-audit,"unmodified_opinion, qualified_opinion",,,,No,Yes,No,Yes,No,750000,Yes,"10, 84",local,,annual,1058244,UG,true,CENSUS,2019-03-25,WILLIAM WARD,SENIOR AUDIT MANAGER +202000,2017-06-CENSUS-0000183467,UNION COUNTY SCHOOL DISTRICT NO. 1,BUSINESS MANAGER,CHRIS PANIKE,CHRIS.PANIKE@LAGRANDESD.ORG,UNION COUNTY SCHOOL DISTRICT NO. 1,5416633202,BUSINESS MANAGER,1305 N WILLOW STREET,LA GRANDE,OR,936000972,GSA_MIGRATION,No,97850,5419634191,OR,LA GRANDE,OWNER/MEMBER,PO BOX 1024,97850,USA,"CHELSEA HERRON, CPA",CHERRON@LAGRANDECPA.COM,"LEWIS, POE, MOELLER, GUNDERSON & ROBERTS, LLC",,931245066,,84,2024-01-27,2017-12-19,2017-12-20,2017-12-19,2017-12-19,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,Yes,"84, 10",local,,annual,2080149,UG,true,CENSUS,2017-12-19,"CHELSEA HERRON, CPA",OWNER/MEMBER +203000,2018-06-CENSUS-0000232362,ARLINGTON STREET PEOPLE'S ASSISTANCE NETWORK AND AFFILIATE,EXECUTIVE DIRECTOR,KATHLEEN SIBERT,KSIBERT@A-SPAN.ORG,ARLINGTON STREET PEOPLE'S ASSISTANCE NETWORK AND AFFILIATE,7032287807,EXECUTIVE DIRECTOR,2020 14TH STREET NORTH,ARLINGTON,VA,541615993,GSA_MIGRATION,No,22201,7038930026,VA,VIENNA,PRINCIPAL,"8300 BOONE BLVD, SUITE 600",22182,USA,JORGE DIAZ,JDIAZ@ROGERSPLLC.COM,ROGERS & COMPANY PLLC,,585676261,,14,2024-01-26,2018-12-02,2018-12-03,2018-12-03,2018-12-02,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,1428871,UG,true,CENSUS,2018-12-02,JORGE DIAZ,PRINCIPAL +204000,2018-06-CENSUS-0000185583,"GRANVILLE PLAZA, INC. 053-EE001",ASSISTANT VICE PRESIDENT,JOHN NICHOLS,TVERNON@NCHSM.ORG,"GRANVILLE PLAZA, INC. 053-EE001",3367653906,ASSISTANT VICE PRESIDENT,750 BETHESDA ROAD,WINSTON-SALEM,NC,582004370,GSA_MIGRATION,No,27103,3362944494,NC,GREENSBORO,PARTNER,"1501 HIGHWOODS BLVD., SUITE 300",27410,USA,ERICA VERNON,EVERNON@BRCCPA.COM,"BERNARD ROBINSON & COMPANY, L.L.P.",,560571159,,14,2024-01-26,2019-01-02,2019-01-02,2019-01-03,2019-01-02,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,2368407,UG,true,CENSUS,2019-01-02,ERICA VERNON,PARTNER +205000,2018-06-CENSUS-0000146445,BOONE COUNTY BOARD OF EDUCATION,FINANCE DIRECTOR,LINDA SCHILD,LINDA.SCHILD@BOONE.KYSCHOOLS.US,BOONE COUNTY BOARD OF EDUCATION,8592822938,FINANCE DIRECTOR,8330 U.S. HIGHWAY 42,FLORENCE,KY,616001252,GSA_MIGRATION,No,41042,8593446400,KY,CRESTVIEW HILLS,DIRECTOR,2617 LEGENDS WAY,41017,USA,"GEORGE S. SPARKS, JR.",GSPARKS@BARNESDENNIG.COM,BARNES DENNIG,,311119890,,84,2024-01-26,2018-12-11,2018-12-12,2018-12-12,2018-12-11,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 10, 93",state,,annual,12054169,UG,true,CENSUS,2018-12-11,"GEORGE S. SPARKS, JR.",DIRECTOR +206000,2018-09-CENSUS-0000070523,"NORTHWEST KANSAS AREA AGENCY ON AGING, INC.",EXECUTIVE DIRECTOR,MICHELLE MORGAN,MMORGAN@NWKAAA.ORG,"NORTHWEST KANSAS AREA AGENCY ON AGING, INC.",7856288204,EXECUTIVE DIRECTOR,510 WEST 29TH STREET,HAYS,KS,480874448,GSA_MIGRATION,No,67601,7856283046,KS,HAYS,PARTNER,PO BOX 1186,67601,USA,MEAGAN M WELLBROCK,MWELLBROCK@ABBB.COM,"ADAMS, BROWN, BERAN & BALL CHTD.",,481040139,,93,2024-01-26,2019-01-02,2019-01-03,2019-01-03,2019-01-02,2018-09-30,2017-09-30,2018,single-audit,not_gaap,other_basis,No,unmodified_opinion,No,Yes,No,No,No,750000,No,"93, 10",non-profit,,annual,1383333,UG,true,CENSUS,2019-01-02,MEAGAN M WELLBROCK,PARTNER +207000,2018-09-CENSUS-0000220164,"MONTGOMERY COUNTY, MISSISSIPPI",CHANCERY CLERK,RYAN WOOD,RYANWOOD@MONTGOMERYCOUNTYMS.COM,"MONTGOMERY COUNTY, MISSISSIPPI",6622832233,CHANCERY CLERK,P.O. BOX 71,WINONA,MS,646000851,GSA_MIGRATION,No,38967,6015762800,MS,JACKSON,"DIRECTOR, COUNTY AUDIT SECTION",P.O. BOX 956,39205,USA,JOE MCKNIGHT,JOE.MCKNIGHT@OSA.MS.GOV,OFFICE OF THE STATE AUDITOR,,646000742,,23,2024-01-27,2020-04-20,2020-04-21,2020-04-21,2020-04-20,2018-09-30,2017-09-30,2018,single-audit,"unmodified_opinion, adverse_opinion",,,,No,Yes,No,No,No,750000,No,"23, 20, 97, 16, 14",local,,annual,1140725,UG,true,CENSUS,2020-04-20,JOE MCKNIGHT,"DIRECTOR, COUNTY AUDIT SECTION" +208000,2018-06-CENSUS-0000134269,SAN MATEO UNION HIGH SCHOOL DISTRICT,DIRECTOR OF BUDGET AND FISCAL SERVICES,VALERIE MILLER,VMILLER@SMUHSD.ORG,SAN MATEO UNION HIGH SCHOOL DISTRICT,6505582223,DIRECTOR OF BUDGET AND FISCAL SERVICES,650 NORTH DELAWARE STREET,SAN MATEO,CA,942700919,GSA_MIGRATION,No,94401,9169939494,CA,SACRAMENTO,PARTNER,701 HOWE AVE STE E3,95825,USA,DAVID BECKER,DBECKER@JPMCPA.COM,JAMES MARTA & COMPANY LLP,,271682261,,84,2024-01-27,2019-01-02,2018-12-17,2019-01-03,2019-01-02,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 84",local,,annual,4123164,UG,true,CENSUS,2019-01-02,DAVID BECKER,PARTNER +209000,2018-12-CENSUS-0000242324,HAWAII RURAL WATER ASSOCIATION,EXECUTIVE DIRECTOR,MAHANA GOMES,MAHANA@HAWAIIRWA.ORG,HAWAII RURAL WATER ASSOCIATION,8083158925,EXECUTIVE DIRECTOR,"65-1158 MAMALAHOA HWY, STE 2D",KAMUELA,HI,270711752,GSA_MIGRATION,No,96743,8089615532,HI,HILO,PRINCIPAL,PO BOX 6691,96720,USA,ANN FUKUHARA,ANN.FUKUHARA@HAWAIIANTEL.NET,"ANN FUKUHARA CPA MBA, AN ACCOUNTING CORPORATION",,990350770,,10,2024-01-27,2019-05-28,2019-05-27,2019-05-28,2019-05-28,2018-12-31,2017-12-31,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"66, 10",non-profit,,annual,905543,UG,true,CENSUS,2019-05-28,ANN FUKUHARA,PRINCIPAL +210000,2018-06-CENSUS-0000135408,LAKE COUNTY SCHOOL DISTRICT R-1,ACCOUNTING MANAGER,RENA SANCHEZ,RSANCHEZ@LAKECOUNTYSCHOOLS.NET,LAKE COUNTY SCHOOL DISTRICT R-1,7194866812,ACCOUNTING MANAGER,328 WEST 5TH STREET,LEADVILLE,CO,846011994,GSA_MIGRATION,No,804613661,9702453000,CO,GRAND JUNCTION,PRINCIPAL,2499 HWY 6&50,815051106,USA,LISA HEMANN,LISAH@CSDCPA.COM,"CHADWICK, STEINKIRCHNER, DAVIS & CO., P.C.",,840865725,,93,2024-01-27,2019-02-03,2019-02-04,2019-02-04,2019-02-03,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,No,"10, 93, 84",local,,annual,2024613,UG,true,CENSUS,2019-02-03,LISA HEMANN,PRINCIPAL +211000,2018-12-CENSUS-0000209247,ATLANTIC INSTITUTE OF ORIENTAL MEDICINE,FINANCIAL OFFICER,CELIA MUNOZ,CELIAMUNOZ@ATOM.EDU,ATLANTIC INSTITUTE OF ORIENTAL MEDICINE,9547639840,FINANCIAL OFFICER,100 EAST BORWARD BLVD #100,FORT LAUDERDALE,FL,650512087,GSA_MIGRATION,No,33301,3057613132,FL,MIAMI,PARTNER,10250 SW 56 STREET # B201,33165,USA,DOMIMGO ALVAREZ,DOMINGO@ALVAREZANDMENDOSA.COM,ALVAREZ & MENDOZA PA,,203162550,,84,2024-01-27,2019-06-20,2019-06-20,2019-06-20,2019-06-20,2018-12-31,2017-12-31,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,84,non-profit,,annual,3012801,UG,true,CENSUS,2019-06-20,DOMIMGO ALVAREZ,PARTNER +212000,2018-12-CENSUS-0000245306,"NEW YORK GENOME CENTER, INC.","SENIOR VICE PRESIDENT, FINANCE AND ADMIN",ANA BASTIANI-POSNER,ABASTIANI@NYGENOME.ORG,"NEW YORK GENOME CENTER, INC.",6469777343,"SENIOR VICE PRESIDENT, FINANCE AND ADMIN",101 AVENUE OF THE AMERICAS,NEW YORK,NY,800631734,GSA_MIGRATION,No,10013,6314256053,NY,MELVILLE,PARTNER,1305 WALT WHITMAN,11747,USA,EDWARD LEE,ENLEE@KPMG.COM,KPMG LLP,,135565207,,93,2024-01-27,2019-09-26,2019-09-27,2019-09-27,2019-09-26,2018-12-31,2017-12-31,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"47, 93, 12",non-profit,,annual,17318302,UG,true,CENSUS,2019-09-26,EDWARD LEE,PARTNER +213000,2018-06-CENSUS-0000133843,EAST WHITTIER CITY SCHOOL DISTRICT,"ASSISTANT SUPERINTENDENT, BUSINESS SERVI",RUBEN HERNANDEZ,RHERNANDEZ@EWCSD.ORG,EAST WHITTIER CITY SCHOOL DISTRICT,5629075900,"ASSISTANT SUPERINTENDENT, BUSINESS SERVI",14535 E WHITTIER BLVD,WHITTIER,CA,956001046,GSA_MIGRATION,No,90605,6192708222,CA,SAN DIEGO,PARTNER,348 OLIVE STREET,92103,USA,CHRISTY WHITE,CWHITE@CHRISTYWHITE.COM,CHRISTY WHITE ASSOCIATES,,272956198,,10,2024-01-27,2019-01-24,2019-01-25,2019-01-04,2019-01-24,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 10, 93",local,,annual,6741671,UG,true,CENSUS,2019-01-24,CHRISTY WHITE,PARTNER +214000,2018-12-CENSUS-0000157275,CAMDEN COUNTY MUNICIPAL UTILITIES AUTHORITY,CHIEF FINANCIAL OFFICER,WAYNE PLANAMENTO,WAYNE@CCMUA.ORG,CAMDEN COUNTY MUNICIPAL UTILITIES AUTHORITY,8565413700,CHIEF FINANCIAL OFFICER,1645 FERRY AVENUE,CAMDEN,NJ,222003702,GSA_MIGRATION,No,08104,8564356200,NJ,VOORHEES,PARTNER,601 WHITE HORSE ROAD,08043,USA,CAROL A. MCALLISTER,CMCALLISTER@BOWMANLLP.COM,BOWMAN & COMPANY LLP,,210658561,,66,2024-01-27,2019-11-19,2019-11-18,2019-12-06,2019-11-19,2018-12-31,2017-12-31,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,66,local,,annual,27337045,UG,true,CENSUS,2019-11-19,CAROL A. MCALLISTER,PARTNER +215000,2017-08-CENSUS-0000174370,HIDALGO INDEPENDENT SCHOOL DISTRICT,CFO,NANCY FLORES,NASANCHEZ@HIDALGO-ISD.ORG,HIDALGO INDEPENDENT SCHOOL DISTRICT,9568434413,CFO,324 E FLORA ST,HIDALGO,TX,746001085,GSA_MIGRATION,No,78557,9565447778,TX,BROWNSVILLE,ENGAGEMENT PARTNER,765 E 7TH STREET,78520,USA,CARLOS CASCOS,CCASCOS@PBHCPA.COM,PATILLO BROWN HILL LLP,,742751720,,12,2024-01-27,2018-02-08,2018-02-09,2018-02-09,2018-02-08,2017-08-31,2016-08-31,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 10, 12, 84",local,,annual,6055809,UG,true,CENSUS,2018-02-08,CARLOS CASCOS,ENGAGEMENT PARTNER +216000,2017-12-CENSUS-0000128718,"MUTUAL HOUSING ASSOCIATION OF HAWAII, INC.",EXECUTIVE DIRECTOR,DAVID NAKAMURA,DNAKAMURA@MUTUAL-HOUSING.ORG,"MUTUAL HOUSING ASSOCIATION OF HAWAII, INC.",8085500804,EXECUTIVE DIRECTOR,"900 FORT STREET, SUITE 1690",HONOLULU,HI,990308739,GSA_MIGRATION,No,96813,8085242255,HI,HONOLULU,AUDIT SENIOR MANAGER,"1001 BISHOP STREET, ASB TOWER, SUITE 1700",96813,USA,JOHN BAUTISTA,JBAUTISTA@NKCPA.COM,"N&K CPAS, INC.",,990169131,,14,2024-01-27,2018-06-19,2018-06-20,2018-06-20,2018-06-19,2017-12-31,2016-12-31,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"21, 14",non-profit,,annual,5308000,UG,true,CENSUS,2018-06-19,JOHN BAUTISTA,AUDIT SENIOR MANAGER +217000,2017-06-CENSUS-0000238317,FRIENDS OUTSIDE IN LOS ANGELES COUNTY,BOOKKEEPER,MARILU CLARIDGE,MCLARIDGE@FRIENDSOUTSIDELA.ORG,FRIENDS OUTSIDE IN LOS ANGELES COUNTY,6267957607,BOOKKEEPER,"261 E. COLORADO BLVD., SUITE 217",PASADENA,CA,953557032,GSA_MIGRATION,No,91101,3102884213,CA,LOS ANGELES,PRINCIPAL,"1925 CENTURY PARK EAST, 16TH FLOOR",90067,USA,TAYIIKA M. DENNIS,TAYIIKA.DENNIS@CLACONNECT.COM,CLIFFTONLARSONALLEN LLP,,410746749,,93,2024-01-27,2018-03-25,2018-03-24,2018-03-26,2018-03-25,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"17, 93",non-profit,,annual,1759579,UG,true,CENSUS,2018-03-25,TAYIIKA M. DENNIS,PRINCIPAL +218000,2017-12-CENSUS-0000200376,CREATIVE ALTERNATIVES SENIOR CITIZEN OPPORTUNITES 136-EH103-NP-L8-WAH,CONTROLLER,CADE SCHOLL,CSCHOLL@MERCYHOUSING.ORG,CREATIVE ALTERNATIVES SENIOR CITIZEN OPPORTUNITES 136-EH103-NP-L8-WAH,3038303300,CONTROLLER,"2512 RIVER PLAZA DRIVE, SUITE 200",SACRAMENTO,CA,770030850,GSA_MIGRATION,No,95833,9168045367,CA,EL DORADO HILLS,SOLE PROPRIETOR,3811 TILDEN DRIVE,95762,USA,THOMAS TOMASZEWSKI,TOMCPA@DIRECTCON.NET,THOMAS TOMASZEWSKI CPA,,680466584,,14,2024-01-27,2018-04-24,2018-04-18,2018-04-23,2018-04-24,2017-12-31,2016-12-31,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,state,,annual,853035,UG,true,CENSUS,2018-04-24,THOMAS TOMASZEWSKI,SOLE PROPRIETOR +219000,2017-06-CENSUS-0000161106,MONTGOMERY COUNTY BOARD OF EDUCATION,CHIEF FINANCIAL OFFICER,MITCH TAYLOR,MITCH.TAYLOR@MONTGOMERY.K12.NC.US,MONTGOMERY COUNTY BOARD OF EDUCATION,9105766511,CHIEF FINANCIAL OFFICER,441 PAGE STREET,TROY,NC,566001076,GSA_MIGRATION,No,27371,9109971418,NC,ROCKINGHAM,MEMBER,116 GREYLYN ROAD,28379,USA,"DALE SMITH, CPA",DSMITH@ASW-CPA.COM,ANDERSON SMITH & WIKE PLLC,,271473864,,84,2024-01-27,2017-11-12,2017-11-13,2017-11-13,2017-11-12,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 93, 84, 12",state,,annual,6456704,UG,true,CENSUS,2017-11-12,"DALE SMITH, CPA",MEMBER +220000,2017-09-CENSUS-0000194114,CHARLOTTE COUNTY AIRPORT AUTHORITY,DIRECTOR OF FINANCE,SANDY CAULEY,SCAULEY@FLYPGD.COM,CHARLOTTE COUNTY AIRPORT AUTHORITY,9416391101,DIRECTOR OF FINANCE,28000 A-1 AIRPORT ROAD,PUNTA GORDA,FL,591082365,GSA_MIGRATION,No,33982,2392222090,FL,FORT MYERS,PARTNER,"12621 WORLD PLAZA LANE, BLDG 55",33907,USA,JEFFREY M. TUSCAN,JTUSCAN@TUSCANCPA.COM,"TUSCAN & COMPANY, PA",,260254161,,20,2024-01-27,2018-04-25,2018-04-26,2018-04-25,2018-04-25,2017-09-30,2016-09-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"97, 20",local,,annual,1963905,UG,true,CENSUS,2018-04-25,JEFFREY M. TUSCAN,PARTNER +221000,2017-12-CENSUS-0000243537,SC THRIVE,CHIEF FINANCIAL OFFICER,ALLIE BOYKIN,ABOYKIN@SCTHRIVE.ORG,SC THRIVE,8037279965,CHIEF FINANCIAL OFFICER,2211 ALPINE ROAD EXTENSION,COLUMBIA,SC,901011409,GSA_MIGRATION,No,29223,8037990555,SC,COLUMBIA,SHAREHOLDER,1704 LAUREL STREET,29201,USA,CHRISTINA KELLY,CHRISTINA@HOBBSCPA.COM,"THE HOBBS GROUP, P.A.",,570957419,,10,2024-01-27,2018-04-29,2018-04-20,2018-04-30,2018-04-29,2017-12-31,2016-12-31,2017,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,Yes,"93, 10",non-profit,,annual,1333188,UG,true,CENSUS,2018-04-29,CHRISTINA KELLY,SHAREHOLDER +222000,2017-03-CENSUS-0000111191,"HOMEWISE, INC.",CONTROLLER,ROBIN COLE,RCOLE@HOMEWISE.ORG,"HOMEWISE, INC.",5059557014,CONTROLLER,1301 SILER RD. BLDG. D,SANTA FE,NM,850346325,GSA_MIGRATION,No,87505,5053380800,NM,ALBUQUERQUE,AUDIT MANAGER,"6200 UPTOWN BLVD NE, SUITE 400",87110,USA,LEE BALDWIN,LEE.BALDWIN@RICCICPA.COM,"RICCI & COMPANY, LLC",,205949532,,21,2024-01-27,2017-08-10,2017-08-11,2017-08-11,2017-08-10,2017-03-31,2016-03-31,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"14, 21",non-profit,,annual,2065432,UG,true,CENSUS,2017-08-10,LEE BALDWIN,AUDIT MANAGER +223000,2017-06-CENSUS-0000222752,GREENFIELD MANOR 122-HD144-WDD-NP,EXECUTIVE DIRECTOR,NORMA DELGADO,NORMAD@HOMEOPENINGDOORS.ORG,GREENFIELD MANOR 122-HD144-WDD-NP,3109884000,EXECUTIVE DIRECTOR,"5601 W SLAUSON AVE., SUITE 180",CULVER CITY,CA,731628868,GSA_MIGRATION,No,90230,8188485585,CA,VALENCIA,CPA,"27200 TOURNEY ROAD, SUITE 475",91355,USA,THOMAS ENGMAN,TOM@KKAJCPA.COM,"KKAJ, LLP",,463462919,,14,2024-01-27,2018-02-20,2018-02-21,2018-02-21,2018-02-20,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,911062,UG,true,CENSUS,2018-02-20,THOMAS ENGMAN,CPA +224000,2017-06-CENSUS-0000156554,MIDDLESEX COUNTY VOCATIONAL AND TECHNICAL HIGH SCHOOLS,BUSINESS ADMINISTRATOR,KARL J. KNEHR,KNEHRK@MCVTS.NET,MIDDLESEX COUNTY VOCATIONAL AND TECHNICAL HIGH SCHOOLS,7322573300,BUSINESS ADMINISTRATOR,112 RUES LANE,EAST BRUNSWICK,NJ,226002455,GSA_MIGRATION,No,08816,9089676855,NJ,CRANFORD,PARTNER,20 COMMERCE DRIVE SUITE 301,07016,USA,DAVID GANNON,DGANNON@PKFOD.COM,PKF O'CONNOR DAVIES LLP,,271728945,,84,2024-01-27,2017-12-21,2017-12-11,2017-12-22,2017-12-21,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 93, 84",local,,annual,2585883,UG,true,CENSUS,2017-12-21,DAVID GANNON,PARTNER +225000,2017-06-CENSUS-0000150272,SPARTA AREA SCHOOLS,FINANCE DIRECTOR,DEBORAH CARNES,DEB.CARNES@SPARTASCHOOLS.ORG,SPARTA AREA SCHOOLS,6168878253,FINANCE DIRECTOR,465 SOUTH UNION STREET,SPARTA,MI,386002095,GSA_MIGRATION,No,49345,6169493200,MI,GRAND RAPIDS,SHAREHOLDER,2910 LUCERNE DRIVE SE,495467175,USA,"ADRIANE SCHRAUBEN, CPA, MBA",ASCHRAUBEN@HUNGERFORDNICHOLS.COM,HUNGERFORD NICHOLS,,382184825,,84,2024-01-27,2017-11-05,2017-11-06,2017-11-06,2017-11-05,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 10, 84",local,,annual,2144687,UG,true,CENSUS,2017-11-05,"ADRIANE SCHRAUBEN, CPA, MBA",SHAREHOLDER +226000,2017-09-CENSUS-0000241244,CITRUS RESEARCH BOARD,MANAGER OF FINANCE & ACCOUNTING,KELLY SALINAS,KELLY@CITRUSRESEARCH.ORG,CITRUS RESEARCH BOARD,5597380246,MANAGER OF FINANCE & ACCOUNTING,PO BOX 230,VISALIA,CA,953777492,GSA_MIGRATION,No,93279,5592256891,CA,FRESNO,PARTNER,1180 E. SHAW AVE. SUITE 110,93710,USA,SCOTT FAETH,SCOTT@BCFZ-CPA.COM,"BORCHARDT, CORONA, FAETH & ZAKARIAN",,770314847,,10,2024-01-27,2018-05-01,2018-05-02,2018-05-02,2018-05-01,2017-09-30,2016-09-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,10,local,,annual,1086375,UG,true,CENSUS,2018-05-01,SCOTT FAETH,PARTNER +227000,2017-12-CENSUS-0000245491,THE GOOD SAMARITAN HEALTH CENTER OF COBB,DIRECTOR OF STEWARDSHIP,KACIE MCDONNELL,KACIE.MCDONNELL@GOODSAMHC.ORG,THE GOOD SAMARITAN HEALTH CENTER OF COBB,4047832533,DIRECTOR OF STEWARDSHIP,1605 ROBERTA DRIVE,MARIETTA,GA,320045238,GSA_MIGRATION,No,30008,6782905100,GA,MARIETTA,CPA,POST AND ASSOCIATES 3475 DALLAS HWY #125,30064,USA,JIM CASTLE,JIM@POSTCPAS.COM,DAN POST/POST AND ASSOCIATES,,813068280,,93,2024-01-27,2018-09-24,2018-09-24,2018-09-25,2018-09-24,2017-12-31,2016-12-31,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 14",non-profit,,annual,1052368,UG,true,CENSUS,2018-09-24,JIM CASTLE,CPA +228000,2017-06-CENSUS-0000068545,"HIT, INC. (HOUSING INDUSTRY TRAINING, INC.)",EXECUTIVE DIRECTOR,MIKE REMBOLDT,MREMBOLDT@HITINC.ORG,"HIT, INC. (HOUSING INDUSTRY TRAINING, INC.)",7016330379,EXECUTIVE DIRECTOR,1007 18TH STREET NW,MANDAN,ND,450353818,GSA_MIGRATION,No,585549369,7012558488,ND,BISMARCK,AUDIT PARTNER,1730 BURNT BOAT DR,585021914,USA,KATIE WILLIAMS,KWILLIAMS@EIDEBAILLY.COM,EIDE BAILLY LLP,,450250958,,93,2024-01-27,2017-10-16,2017-10-16,2017-10-16,2017-10-16,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 10",non-profit,,annual,1582738,UG,true,CENSUS,2017-10-16,KATIE WILLIAMS,AUDIT PARTNER +229000,2016-06-CENSUS-0000169847,"SCHOOL DISTRICT OF THE CITY OF ERIE, PENNSYLVANIA",CHIEF FINANCIAL OFFICER,BRIAN POLITO,BPOLITO@ERIESD.ORG,"SCHOOL DISTRICT OF THE CITY OF ERIE, PENNSYLVANIA",8148746137,CHIEF FINANCIAL OFFICER,148 WEST 21ST STREET,ERIE,PA,256001265,GSA_MIGRATION,No,16502,4123677102,PA,PITTSBURGH,PARTNER,"3800 MCKNIGHT EAST DRIVE, SUITE 3805",15237,USA,CORY JOHNSON,CJOHNSON@ZALLC.ORG,ZELENKOFSKE AXELROD LLC,,233022352,,84,2024-01-29,2017-03-16,2017-03-20,2017-03-20,2017-03-16,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,Yes,Yes,No,No,No,872151,Yes,"84, 93, 10",local,,annual,29200682,UG,true,CENSUS,2017-03-16,CORY JOHNSON,PARTNER +230000,2017-06-CENSUS-0000235032,EDISON TERRACE SOUTH 171-HD022,DEPARTMENT MANAGER,KIM CONWAY,KCONWAY@G-B.COM,EDISON TERRACE SOUTH 171-HD022,5094596102,DEPARTMENT MANAGER,818 W RIVERSIDE AVE#300,SPOKANE,WA,264362933,GSA_MIGRATION,No,99201,5097472158,WA,SPOKANE,CPA,422 W RIVERSIDE AVE #1420,99201,USA,DARCY SKJOTHAUG,DSKJOTHAUG@SCHOEDEL.COM,SCHOEDEL & SCHOEDEL CPAS PLLC,,910614823,,14,2024-01-27,2017-10-22,2017-10-16,2017-10-23,2017-10-22,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,1702419,UG,true,CENSUS,2017-10-22,DARCY SKJOTHAUG,CPA +231000,2017-12-CENSUS-0000233543,DOUGLAS POINTE III ASSOCIATES LLC DBA DOUGLAS POINTE III APTS 07311392,CONTROLLER,ROB ELSBY,RELSBY@FLCO.COM,DOUGLAS POINTE III ASSOCIATES LLC DBA DOUGLAS POINTE III APTS 07311392,3178169300,CONTROLLER,"ONE INDIANA SQUARE, SUITE 3000",INDIANAPOLIS,IN,351946082,GSA_MIGRATION,No,46204,3178485700,IN,CARMEL,PRINCIPAL,"501 CONGRESSIONAL BLVD, SUITE 300",46032,USA,RYAN STRUTZ,RSTRUTZ@DOZ.NET,"DAUBY O'CONNOR & ZALESKI, LLC",,351750664,,14,2024-01-27,2018-04-10,2018-04-11,2018-04-09,2018-04-10,2017-12-31,2016-12-31,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,2605252,UG,true,CENSUS,2018-04-10,RYAN STRUTZ,PRINCIPAL +232000,2017-09-CENSUS-0000188538,"CITY OF ROLLA, MISSOURI",FINANCE DIRECTOR,STEFFANIE ROGERS,SROGERS@ROLLACITY.ORG,"CITY OF ROLLA, MISSOURI",5734266980,FINANCE DIRECTOR,901 NORTH ELM STREET,ROLLA,MO,436003049,GSA_MIGRATION,No,65402,6365329525,MO,CHESTERFIELD,PARTNER,"15450 SOUTH OUTER ROAD FORTY, SUITE 135",630172066,USA,"TAMBER M. ALSOP, CPA",TAMMY@HBCLP.COM,"HOCHSCHILD, BLOOM & COMPANY LLP",,430673920,,16,2024-01-27,2018-10-24,2018-10-25,2018-10-25,2018-10-24,2017-09-30,2016-09-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"97, 20, 16",local,,annual,1130787,UG,true,CENSUS,2018-10-24,"TAMBER M. ALSOP, CPA",PARTNER +233000,2017-05-CENSUS-0000220905,"PENSDALE HOUSING, INC. 034-EE100-WAH",CONTROLLER,CHARMAINE KENT,CHARMAINE.KENT@PRESBYSINSPIREDLIFE.ORG,"PENSDALE HOUSING, INC. 034-EE100-WAH",6102601119,CONTROLLER,2000 JOSHUA ROAD,LAFAYETTE HILL,PA,311719838,GSA_MIGRATION,No,194442430,7177617910,PA,CAMP HILL,PARTNER,415 FALLOWFIELD ROAD,17011,USA,GARY DUBAS,GDUBAS@MACPAS.COM,"MCKONLY & ASBURY, LLP",,231909723,,14,2024-01-27,2017-08-29,2017-08-30,2017-08-30,2017-08-29,2017-05-31,2016-05-31,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,7436685,UG,true,CENSUS,2017-08-29,GARY DUBAS,PARTNER +234000,2017-12-CENSUS-0000230805,"GEORGIA MENTAL HEALTH CONSUMER NETWORK, INC.",FINANCIAL MANAGER,LYNN THOGERSEN,LYNN@GMHCN.ORG,"GEORGIA MENTAL HEALTH CONSUMER NETWORK, INC.",4046879487,FINANCIAL MANAGER,246 SYCAMORE ST. SUITE 260,DECATUR,GA,581981093,GSA_MIGRATION,No,300303434,7068864343,GA,TOCCOA,CPA,3438 HIGHWAY 17 ALT,30577,USA,DAVID BENFIELD,DAVID@DAVIDBENFIELDCPA.COM,"DAVID BENFIELD, CPA, PC",,582363670,,93,2024-01-27,2018-07-23,2018-07-24,2018-07-24,2018-07-23,2017-12-31,2016-12-31,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,93,non-profit,,annual,4425348,UG,true,CENSUS,2018-07-23,DAVID BENFIELD,CPA +235000,2017-06-CENSUS-0000144750,IOWA CENTRAL COMMUNITY COLLEGE,"VICE PRESIDENT, BUSINESS AFFAIRS",ANGIE MARTIN,MARTIN_A@IOWACENTRAL.EDU,IOWA CENTRAL COMMUNITY COLLEGE,5155741064,"VICE PRESIDENT, BUSINESS AFFAIRS",1 TRITON CIRCLE,FORT DODGE,IA,410906391,GSA_MIGRATION,No,50501,5155767136,IA,FORT DODGE,PARTNER,822 CENTRAL AVE,50501,USA,MURRAY STANLEY,MURRAY@SCHNURRLLP.COM,"SCHNURR & COMPANY, LLP",,421282836,,84,2024-01-27,2018-03-28,2018-03-19,2018-03-28,2018-03-28,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,Yes,"10, 84, 17",local,,annual,28805867,UG,true,CENSUS,2018-03-28,MURRAY STANLEY,PARTNER +236000,2017-12-CENSUS-0000234455,COLE STREET DEVELOPMENT CORPORATION,DIRECTOR OF FINANCE,MARK WRIGHT,MWRIGHT@MARIETTAHOUSINGAUTHORITY.ORG,COLE STREET DEVELOPMENT CORPORATION,7704193200,DIRECTOR OF FINANCE,95 COLE STREET,MARIETTA,GA,264536445,GSA_MIGRATION,No,30061,7708798411,GA,LAWRENCEVILLE,PRINCIPAL,"1255 LAKES PARKWAY, STE 375",30043,USA,DALE R. RECTOR,DRECTOR@CPA-RRL.COM,"RECTOR, REEDER & LOFTON, P.C.",,263670494,,14,2024-01-27,2018-07-11,2018-07-12,2018-05-17,2018-07-11,2017-12-31,2016-12-31,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,14,non-profit,,annual,3320075,UG,true,CENSUS,2018-07-11,DALE R. RECTOR,PRINCIPAL +237000,2017-06-CENSUS-0000247375,LAWS LOUDOUN CITIZENS FOR SOCIAL JUSTICE,EXECUTIVE DIRECTOR,"JUDY HANLEY, PHD",JUDY.HANLEY@LCSJ.ORG,LAWS LOUDOUN CITIZENS FOR SOCIAL JUSTICE,7037713398,EXECUTIVE DIRECTOR,105 EAST MARKET STREET,LEESBURG,VA,541282756,GSA_MIGRATION,No,201760000,7037774900,VA,LEESBURG,CPA,110 EAST MARKET STREET SUITE 200,201760000,USA,KARA SANTMYER,KARA@MCOCPA.COM,"MITCHELL & CO., P.C.",,541853459,,16,2024-01-27,2018-08-12,2018-08-06,2018-08-13,2018-08-12,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"93, 14, 16",non-profit,,annual,919623,UG,true,CENSUS,2018-08-12,KARA SANTMYER,CPA +238000,2017-06-CENSUS-0000136775,MONROE COUNTY DISTRICT SCHOOL BOARD,EXE DIRECTOR OF FINANCE AND PERFORMANCE,JAMES DRAKE,JAMES.DRAKE@KEYSSCHOOLS.COM,MONROE COUNTY DISTRICT SCHOOL BOARD,3052931400,EXE DIRECTOR OF FINANCE AND PERFORMANCE,241 TRUMBO ROAD,KEY WEST,FL,596000750,GSA_MIGRATION,No,330406684,8504122905,FL,TALLAHASSEE,AUDIT MANAGER,111 WEST MADISON STREET,323991450,USA,MICAH RODGERS,MICAHRODGERS@AUD.STATE.FL.US,STATE OF FLORIDA AUDITOR GENERAL,,596001874,,93,2024-01-27,2018-03-26,2018-03-27,2018-03-26,2018-03-26,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 93, 10",local,,annual,8979530,UG,true,CENSUS,2018-03-26,MICAH RODGERS,AUDIT MANAGER +239000,2017-12-CENSUS-0000241203,URBAN RESOURCE INSTITUTE AND AFFILIATES,SEVIOR VICE PRESIDENT OF FINANCE,DONOVAN MURRAY,DMURRAY@URINYC.ORG,URBAN RESOURCE INSTITUTE AND AFFILIATES,6465880040,SEVIOR VICE PRESIDENT OF FINANCE,75 BROAD STREET SUITE 505,NEW YORK,NY,112561648,GSA_MIGRATION,No,10004,7329649359,NJ,EAST BRUNSWICK,PARTNER,ONE TOWER CENTER BLVD,08816,USA,JAMES MULROY,JMULROY@WITHUM.COM,WTIHUMSMITH+BROWN,,222027092,,93,2024-01-27,2018-06-27,2018-06-28,2018-06-25,2018-06-27,2017-12-31,2016-12-31,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 14",non-profit,,annual,17891820,UG,true,CENSUS,2018-06-27,JAMES MULROY,PARTNER +240000,2017-12-CENSUS-0000209375,GENESEE VALLEY PRESBYTERIAN NURSING CTR D/B/A KIRKHAVEN 014-43199,"CONTROLLER, CMSWNY",SCOTT MAAR,SMAAR@CMSWNY.ORG,GENESEE VALLEY PRESBYTERIAN NURSING CTR D/B/A KIRKHAVEN 014-43199,5854424315,"CONTROLLER, CMSWNY",254 ALEXANDER STREET,ROCHESTER,NY,222291346,GSA_MIGRATION,No,14607,5853811000,NY,PITTSFORD,PARTNER,171 SULLY'S TRAIL,14534,USA,KELLE DEMONTE,KDEMONTE@BONADIO.COM,"BONADIO & CO., LLP",,161131146,,14,2024-01-27,2018-07-09,2018-07-10,2018-05-30,2018-07-09,2017-12-31,2016-12-31,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,4902508,UG,true,CENSUS,2018-07-09,KELLE DEMONTE,PARTNER +241000,2017-12-CENSUS-0000220835,"KENAI PENINSULA HOUSING INITIATIVES, INC.",EXECUTIVE DIRECTOR,STEVEN ROUSE,STEVEN@KPHI.NET,"KENAI PENINSULA HOUSING INITIATIVES, INC.",9072354357,EXECUTIVE DIRECTOR,3751 STERLING HIGHWAY,HOMER,AK,911823180,GSA_MIGRATION,No,99603,9072788878,AK,ANCHORAGE,MANAGING DIRECTOR,"3601 C STREET, SUITE 600",99503,USA,ALEX BECKMAN,ABECKMAN@BDO.COM,"BDO USA, LLP",,135381590,,14,2024-01-27,2018-09-29,2018-09-30,2018-09-30,2018-09-29,2017-12-31,2016-12-31,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,14,non-profit,,annual,2510066,UG,true,CENSUS,2018-09-29,ALEX BECKMAN,MANAGING DIRECTOR +242000,2016-06-CENSUS-0000111911,NORTHLAND FAMILY HELP CENTER,EXECUTIVE DIRECTOR,AILEEN FITZ,AFITZ@NORTHLANDFAMILY.ORG,NORTHLAND FAMILY HELP CENTER,9282332205,EXECUTIVE DIRECTOR,"2532 NORTH FOURTH STREET, SUITE 506",FLAGSTAFF,AZ,860351566,GSA_MIGRATION,No,86004,6022643077,AZ,SCOTTSDALE,DIRECTOR,"9019 EAST BAHIA DRIVE, SUITE 100",852601553,USA,OLIVIA BRASHER,OBRASHER@F-CPC.COM,FESTER & CHAPMAN P.C.,,860494040,,93,2024-01-27,2017-01-11,2017-01-12,2017-01-12,2017-01-11,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 16, 93",non-profit,,annual,954937,UG,true,CENSUS,2017-01-11,OLIVIA BRASHER,DIRECTOR +243000,2016-06-CENSUS-0000187474,CITY OF SANTA FE,ASSISTANT FINANCE DIRECTOR,TERESITA GARCIA,TMGARCIA@CI.SANTA-FE.NM.US,CITY OF SANTA FE,5059556532,ASSISTANT FINANCE DIRECTOR,200 LINCOLN AVENUE,SANTA FE,NM,856000168,GSA_MIGRATION,No,87504,5058832727,NM,ALBUQUERQUE,PARTNER,2424 LOUISIANA BLVD SUITE 300,87110,USA,ROBERT CORDOVA,RCORDOVA@ACGSW.COM,RPC CPAS + CONSULTANTS,,850454285,,14,2024-01-27,2017-01-22,2017-01-23,2017-01-22,2017-01-22,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"20, 14, 93, 94, 16, 15, 97",local,,annual,5201021,UG,true,CENSUS,2017-01-22,ROBERT CORDOVA,PARTNER +244000,2016-12-CENSUS-0000219565,PROMESA BEHAVIORAL HEALTH & SUBSIDIARY,CONTROLLER,SUSANNE PRUETT,SPRUETT@PROMESABEHAVIORAL.ORG,PROMESA BEHAVIORAL HEALTH & SUBSIDIARY,5594373596,CONTROLLER,7120 NORTH MARKS AVE STE 110,FRESNO,CA,770174896,GSA_MIGRATION,No,93711,5592999540,CA,CLOVIS,AUDIT MANAGER,677 SCOTT AVENUE,93612,USA,HENRY OUM,HENRY@PPCPAS.COM,PRICE PAIGE AND COMPANY,,770203007,,93,2024-01-27,2017-09-27,2017-09-27,2017-09-27,2017-09-27,2016-12-31,2016-01-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 93",non-profit,,annual,5904670,UG,true,CENSUS,2017-09-27,HENRY OUM,AUDIT MANAGER +245000,2016-06-CENSUS-0000187278,"COUNTY OF NELSON, VIRGINIA",COUNTY ADMINISTRATOR,STEPHEN CARTER,SCARTER@NELSONCOUNTY.ORG,"COUNTY OF NELSON, VIRGINIA",4342637001,COUNTY ADMINISTRATOR,PO BOX 336,LOVINGSTON,VA,546001441,GSA_MIGRATION,No,22949,4349738314,VA,CHARLOTTESVILLE,MEMBER,PO BOX 6580,22906,USA,DAVID FOLEY,DFOLEY@RFCA.COM,"ROBINSON, FARMER, COX ASSOCIATES",,541896113,,84,2024-01-27,2017-03-01,2017-03-01,2017-03-02,2017-03-01,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 14, 93, 84, 16, 20, 97",local,,annual,2601211,UG,true,CENSUS,2017-03-01,DAVID FOLEY,MEMBER +246000,2016-09-CENSUS-0000226040,"WIYN CONSORTIUM, INC.",SENIOR FINANCIAL COMPLIANCE MANAGER,JOSE MARTENS,JMARTENS@AURA-ASTRONOMY.ORG,"WIYN CONSORTIUM, INC.",5203188520,SENIOR FINANCIAL COMPLIANCE MANAGER,P.O. BOX 26732,TUCSON,AZ,860669456,GSA_MIGRATION,No,857266732,5207903500,AZ,TUCSON,PRINCIPAL,"5255 EAST WILLIAMS CIRCLE, SUITE 5000",85711,USA,DAVE PHILLIPS,DAVE.PHILLIPS@CLACONNECT.COM,CLIFTONLARSONALLEN,,410746749,,47,2024-01-27,2017-05-25,2017-05-26,2017-05-26,2017-05-25,2016-09-30,2015-10-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,47,non-profit,,annual,1248623,UG,true,CENSUS,2017-05-25,DAVE PHILLIPS,PRINCIPAL +247000,2016-12-CENSUS-0000246429,GOODWILL INDUSTRIES OF CENTRAL EAST TEXAS INC.,TIFFANY.RHODES@LUFKINGOODWILL.ORG,TIFFANY RHODES,TIFFANY.RHODES@LUFKINGOODWILL.ORG,GOODWILL INDUSTRIES OF CENTRAL EAST TEXAS INC.,9366328838,TIFFANY.RHODES@LUFKINGOODWILL.ORG,301 HILL STREET,LUFKIN,TX,752204594,GSA_MIGRATION,No,75904,9366346621,TX,LUFKIN,PARTNER,1307 SOUTH 1ST STREET,75901,USA,MOLLY ABELE,MABELE@AXLEYRODE.COM,AXLEY & RODE LLP,,750767305,,93,2024-01-27,2017-09-28,2018-01-05,2018-01-08,2017-09-28,2016-12-31,2016-01-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,93,non-profit,,annual,4942777,UG,true,CENSUS,2017-09-28,MOLLY ABELE,PARTNER +248000,2016-11-CENSUS-0000089384,"RURAL HEALTH CARE, INC., DBA AZALEA HEALTH",CHIEF FINANCIAL OFFICER,JIM REID,JREID@AZAHEALTH.ORG,"RURAL HEALTH CARE, INC., DBA AZALEA HEALTH",3863251086,CHIEF FINANCIAL OFFICER,613 ST. JOHNS AVENUE; 3RD FLOOR,PALATKA,FL,591792958,GSA_MIGRATION,No,32177,3523781331,FL,GAINESVILLE,PARTNER,5931 NW 1ST PLACE,32607,USA,JENNIFER FORRESTER,JENNIFER.FORRESTER@JMCO.COM,"JAMES MOORE & CO., P.L.",,593204548,,93,2024-01-27,2017-09-28,2017-09-29,2017-09-29,2017-09-28,2016-11-30,2015-12-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,93,non-profit,,annual,6092141,UG,true,CENSUS,2017-09-28,JENNIFER FORRESTER,PARTNER +249000,2016-12-CENSUS-0000039731,"CINCINNATI HEALTH NETWORK, INC.",CEO,KATHLEEN BENNETT,KBENNETT@CINTIHEALTHNETWORK.ORG,"CINCINNATI HEALTH NETWORK, INC.",5139610600,CEO,2825 BURNET AVE. #232,CINCINNATI,OH,311182378,GSA_MIGRATION,No,45219,8593315622,KY,FT. WRIGHT,AUDIT MANAGER,909 WRIGHT'S SUMMIT PARKWAY SUITE 120,41011,USA,VICKI SELIGMAN,VSELIGMAN@CPAWIN.COM,"ANNEKEN, HUEY & MOSER PLLC",,113799330,,93,2024-01-27,2017-09-27,2017-09-26,2017-09-28,2017-09-27,2016-12-31,2016-01-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,93,non-profit,,annual,3488002,UG,true,CENSUS,2017-09-27,VICKI SELIGMAN,AUDIT MANAGER +250000,2016-06-CENSUS-0000148123,"TOWN OF SOUTH HADLEY, MASSACHUSETTS",TOWN ACCOUNTANT,WILLIAM SUTTON,WSUTTON@SOUTHHADLEYMA.GOV,"TOWN OF SOUTH HADLEY, MASSACHUSETTS",4135385017,TOWN ACCOUNTANT,116 MAIN ST,SOUTH HADLEY,MA,046001303,GSA_MIGRATION,No,01075,4136654001,MA,SOUTH DEERFIELD,PARTNER,8 TINA DRIVE,01373,USA,"THOMAS J. SCANLON, JR.",TSCANLONJR@COMCAST.NET,"SCANLON & ASSOCIATES, LLC",,270371224,,93,2024-01-27,2017-06-28,2017-06-29,2017-06-12,2017-06-28,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"84, 93, 14, 97, 20, 10",local,,annual,1735743,UG,true,CENSUS,2017-06-28,"THOMAS J. SCANLON, JR.",PARTNER +251000,2016-08-CENSUS-0000085650,"THE CARTER CENTER, INC.","VICE PRESIDENT, FINANCE AND TREASURER",CHRISTOPHER BROWN,CHRISTOPHER.BROWN@CARTERCENTER.ORG,"THE CARTER CENTER, INC.",4044203429,"VICE PRESIDENT, FINANCE AND TREASURER","ONE COPENHILL, 453 FREEDOM PARKWAY",ATLANTA,GA,581454716,GSA_MIGRATION,No,30307,4042223258,GA,ATLANTA,PARTNER,"303 PEACHTREE ST NE, SUITE 2000",30308,USA,DASHA WALKER,DASHAWALKER@KPMG.COM,KPMG LLP,,135565207,,98,2024-01-27,2018-08-30,2018-08-31,2018-08-31,2018-08-30,2016-08-31,2015-09-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"98, 93, 19",non-profit,,annual,10395332,UG,true,CENSUS,2018-08-30,DASHA WALKER,PARTNER +252000,2016-12-CENSUS-0000194168,"STEVENS COUNTY ECONOMIC IMPROVEMENT COMMISSION, INC.",EXECUTIVE DIRECTOR,CHERYL KUHN,CHERYLSCEIC@FEDTEL.NET,"STEVENS COUNTY ECONOMIC IMPROVEMENT COMMISSION, INC.",3205852609,EXECUTIVE DIRECTOR,"215 ATLANTIC AVENUE, SUITE 110",MORRIS,MN,411383188,GSA_MIGRATION,No,56267,2187393267,MN,FERGUS FALLS,PARTNER,2450 COLLEGE WAY,56537,USA,DEAN BIRKELAND,DEANB@CARLSONHIGHLANDCPA.COM,CARLSON & COMPANY LLP,,411562398,,10,2024-01-29,2017-08-01,2017-08-02,2017-08-01,2017-08-01,2016-12-31,2016-01-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,Yes,"10, 14",non-profit,,annual,1525896,UG,true,CENSUS,2017-08-01,DEAN BIRKELAND,PARTNER +253000,2017-06-CENSUS-0000116605,OREGON COAST COMMUNITY ACTION,FINANCE DIRECTOR,ANDREW SANTOS,ASANTOS@ORCCA.US,OREGON COAST COMMUNITY ACTION,5414357755,FINANCE DIRECTOR,1855 THOMAS AVE,COOS BAY,OR,930547036,GSA_MIGRATION,No,97420,3605333370,WA,ABERDEEN,PRINCIPAL,343 W WISHKAH ST,98520,USA,LONNIE RICH,LONNIE@AIKEN-SANDERS.COM,AIKEN & SANDERS INC. PS,,910870697,,93,2024-01-28,2017-11-06,2017-11-06,2017-11-07,2017-11-06,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 97, 93, 10, 64, 14, 81",non-profit,,annual,5444526,UG,true,CENSUS,2017-11-06,LONNIE RICH,PRINCIPAL +254000,2017-09-CENSUS-0000102556,ST GEORGE'S SENIOR HOUSING INC. 115-EH057,MANAGEMENT AGENT,CONNIE QUILLEN,CQUILLEN@ALPHA-BARNES.COM,ST GEORGE'S SENIOR HOUSING INC. 115-EH057,5125813611,MANAGEMENT AGENT,1443 CORONADO HILLS DRIVE,AUSTIN,TX,742245337,GSA_MIGRATION,No,787522974,8178651360,TX,SOUTHLAKE,PARTNER,"515 W SOUTHLAKE BLVD, STE 150",760926176,USA,MICHAEL MARTIN,MMARTIN@MGROUPTX.COM,"M GROUP, LLP",,263548566,,14,2024-01-29,2018-04-11,2018-04-12,2018-04-12,2018-04-11,2017-09-30,2016-09-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,No,14,non-profit,,annual,1611512,UG,true,CENSUS,2018-04-11,MICHAEL MARTIN,PARTNER +255000,2017-06-CENSUS-0000133121,OMNITRANS,CEO/GENERAL MANAGER,P SCOTT GRAHAM,SCOTT.GRAHAM@OMNITRANS.ORG,OMNITRANS,9093797112,CEO/GENERAL MANAGER,1700 WEST FIFTH STREET,SAN BERNARDINO,CA,953035112,GSA_MIGRATION,No,92411,9094664410,CA,RANCHO CUCAMONGA,PARTNER,"10681 FOOTHILL BLVD., SUITE 300",91730,USA,ROGER ALFARO,RALFARO@VTDCPA.COM,"VAVRINEK, TRINE, DAY & CO., LLP",,952648289,,20,2024-01-29,2018-01-24,2018-01-25,2018-01-25,2018-01-24,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,1167662,Yes,20,local,,annual,38922080,UG,true,CENSUS,2018-01-24,ROGER ALFARO,PARTNER +256000,2016-06-CENSUS-0000119886,"SANTA CRUZ COUNTY PARENTS ASSOCIATION, INC.",EXECUTIVE DIRECTOR,ELAINE HENNING,STAFF@VOUCHERPROJECT.ORG,"SANTA CRUZ COUNTY PARENTS ASSOCIATION, INC.",8316882152,EXECUTIVE DIRECTOR,"9047 SOQUEL DRIVE, SUITE D",APTOS,CA,942410291,GSA_MIGRATION,No,95003,5104525051,CA,OAKLAND,MANAGING PARTNER,"266 17TH STREET, SUITE 200",946124124,USA,SANWAR HARSHWAL,PATEL@PATELCPA.COM,PATEL & ASSOCIATES LLP,,364781081,,93,2024-01-29,2016-12-12,2016-12-13,2016-12-13,2016-12-12,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,93,non-profit,,annual,1920267,UG,true,CENSUS,2016-12-12,SANWAR HARSHWAL,MANAGING PARTNER +257000,2016-06-CENSUS-0000131710,LITTLE ROCK SCHOOL DISTRICT,"DIRECTOR, FINANCE AND ACCOUNTING",JEAN RING,JEAN.RING@LRSD.ORG,LITTLE ROCK SCHOOL DISTRICT,5014471089,"DIRECTOR, FINANCE AND ACCOUNTING",810 WEST MARKHAM STREET,LITTLE ROCK,AR,716014717,GSA_MIGRATION,No,722011306,5016838600,AR,LITTLE ROCK,DEPUTY LEGISLATIVE AUDITOR,"500 WOODLANE STREET, STE 172",722011099,USA,"LARRY W. HUNTER, CPA, CFE",DCF@ARKLEGAUDIT.GOV,ARKANSAS LEGISLATIVE AUDIT,,716042275,,12,2024-01-29,2017-03-26,2017-03-24,2017-03-27,2017-03-26,2016-06-30,2015-07-01,2016,single-audit,not_gaap,regulatory_basis,Yes,unmodified_opinion,No,No,No,No,No,974680,Yes,"12, 84, 15, 10, 93",local,,annual,32489344,UG,true,CENSUS,2017-03-26,"LARRY W. HUNTER, CPA, CFE",DEPUTY LEGISLATIVE AUDITOR +258000,2016-06-CENSUS-0000133979,MODOC JOINT UNIFIED SCHOOL DISTRICT,BUSINESS MANAGER,NIKKI GUZMAN,EGUZMAN@MODOC.K12.CA.US,MODOC JOINT UNIFIED SCHOOL DISTRICT,5302337201,BUSINESS MANAGER,906 WEST FOURTH STREET,ALTURAS,CA,010560280,GSA_MIGRATION,No,96101,5302571040,CA,SUSANVILLE,AUDIT PARTNER,"1740 MAIN STREET, SUITE A",96130,USA,CLAY SINGLETON,CSINGLETON@SA-CPAS.COM,"SINGLETONAUMAN, PC",,753179488,,15,2024-01-29,2017-01-16,2017-01-15,2017-01-17,2017-01-16,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,No,"84, 93, 15, 10",local,,annual,1063186,UG,true,CENSUS,2017-01-16,CLAY SINGLETON,AUDIT PARTNER +259000,2017-06-CENSUS-0000165945,SPERRY PUBLIC SCHOOL DISTRICT NO. I-8,SUPERINTENDENT,BRIAN BEAGLES,BBEAGLES@SPERRY.K12.OK.US,SPERRY PUBLIC SCHOOL DISTRICT NO. I-8,9182886258,SUPERINTENDENT,400 WEST MAIN STREET,SPERRY,OK,736081383,GSA_MIGRATION,No,740730610,9183664440,OK,TULSA,PARTNER,10026-A SOUTH MINGO ROAD #309,741335700,USA,JACK JENKINS,JHJCPA@JACKHJENKINSCPA.COM,"JENKINS & KEMPER, CPAS, P.C.",,731514558,,84,2024-01-29,2018-03-28,2018-03-06,2018-03-29,2018-03-28,2017-06-30,2016-06-30,2017,single-audit,not_gaap,regulatory_basis,Yes,qualified_opinion,No,No,No,No,No,750000,No,"84, 10",state,,annual,1041462,UG,true,CENSUS,2018-03-28,JACK JENKINS,PARTNER +260000,2017-06-CENSUS-0000160592,KIRYAS JOEL UNION FREE SCHOOL DISTRICT,SUPERINTENDENT OF SCHOOLS,MR. JOEL PETLIN,JPETLIN@KJSD.K12.NY.US,KIRYAS JOEL UNION FREE SCHOOL DISTRICT,8457822300,SUPERINTENDENT OF SCHOOLS,48 BAKERTOWN ROAD,MONROE,NY,061301151,GSA_MIGRATION,No,10950,6314733400,NY,PORT JEFFERSON STATION,PARTNER,1650 ROUTE 112,117763060,USA,"ALAN YU, CPA",AYU@CDLLP.NET,"CULLEN & DANOWSKI, LLP",,112459188,,84,2024-01-29,2018-03-25,2018-03-26,2018-03-26,2018-03-25,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,84,local,,annual,9319153,UG,true,CENSUS,2018-03-25,"ALAN YU, CPA",PARTNER +261000,2017-12-CENSUS-0000170987,"CITY OF PIERRE, SOUTH DAKOTA",FINANCE OFFICER,TWILA HIGHT,TWILA.HIGHT@CI.PIERRE.SD.US,"CITY OF PIERRE, SOUTH DAKOTA",6057737407,FINANCE OFFICER,222 E DAKOTA AVE,PIERRE,SD,466000356,GSA_MIGRATION,No,575011253,6043481930,SD,RAPID CITY,AUDITOR DIRECTOR,"909 ST JOSEPH ST, STE 101",57701,USA,DEIDRE BUDAHL,DEIDREB2@CASEYPETERSON.COM,"CASEY PETERSON, LTD.",,460403496,,15,2024-01-29,2018-09-30,2018-09-28,2018-10-01,2018-09-30,2017-12-31,2016-12-31,2017,single-audit,unmodified_opinion,,,,No,Yes,Yes,Yes,No,750000,No,"20, 39, 15, 97",local,,annual,3155357,UG,true,CENSUS,2018-09-30,DEIDRE BUDAHL,AUDITOR DIRECTOR +262000,2016-06-CENSUS-0000164179,BAY VILLAGE CITY SCHOOL DISTRICT,TREASURER,KEVIN ROBERTSON,KEVIN.ROBERTSON@BAYSCHOOLSOHIO.ORG,BAY VILLAGE CITY SCHOOL DISTRICT,4406177312,TREASURER,377 DOVER CENTER ROAD,BAY VILLAGE,OH,346000208,GSA_MIGRATION,No,44140,2167873665,OH,COLUMBUS,SENIOR AUDIT MANAGER,88 EAST BROAD STREET,43215,USA,EVAN WITTE,ETWITTE@OHIOAUDITOR.GOV,"DAVE YOST, AUDITOR OF STATE OF OHIO",,311334821,,84,2024-01-29,2017-03-09,2017-03-10,2017-03-10,2017-03-09,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 84",local,,annual,822325,UG,true,CENSUS,2017-03-09,EVAN WITTE,SENIOR AUDIT MANAGER +263000,2016-06-CENSUS-0000175497,"TOWN OF WAITSFIELD, VERMONT",TOWN ADMINISTRATOR,VALERIE CAPELS,TOWNADMIN@GMAVT.NET,"TOWN OF WAITSFIELD, VERMONT",8024962218,TOWN ADMINISTRATOR,4144 MAIN STREET,WAITSFIELD,VT,036000726,GSA_MIGRATION,No,05673,8022232352,VT,MONTPELIER,PARTNER,PO BOX 947,05601,USA,FRED DUPLESSIS,FDUPLESSIS@SULLIVANPOWERS.COM,"SULLIVAN, POWERS & CO.",,030276150,,66,2024-01-29,2017-03-30,2017-03-30,2017-03-31,2017-03-30,2016-06-30,2015-07-01,2016,single-audit,not_gaap,cash_basis,No,qualified_opinion,No,Yes,Yes,No,Yes,750000,No,"20, 97, 14, 66",local,,annual,1553682,UG,true,CENSUS,2017-03-30,FRED DUPLESSIS,PARTNER +264000,2016-06-CENSUS-0000181767,"PLANNING AND SERVICE AREA 2, AREA AGENCY ON AGING",EXECUTIVE DIRECTOR,TERI GABRIEL,TERI@PSA2.ORG,"PLANNING AND SERVICE AREA 2, AREA AGENCY ON AGING",5308421687,EXECUTIVE DIRECTOR,PO BOX 1400,YREKA,CA,942679692,GSA_MIGRATION,No,960971400,5309263881,CA,MOUNT SHASTA,CPA,PO BOX 158,960670158,USA,STEVE DRAGESET,STEVE@AGTCPA.COM,AIELLO GOODRICH & TEUSCHER,,680146027,,10,2024-01-29,2017-01-16,2017-01-16,2017-01-10,2017-01-16,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,Yes,No,750000,Yes,"10, 93",local,,annual,1752446,UG,true,CENSUS,2017-01-16,STEVE DRAGESET,CPA +265000,2016-06-CENSUS-0000186403,"ACADEMIC DEVELOPMENT INSTITUTE, INC.",EXECUTIVE DIRECTOR,SAM REDDING,SREDDING@ADI.ORG,"ACADEMIC DEVELOPMENT INSTITUTE, INC.",2177326462,EXECUTIVE DIRECTOR,121 N. KICKAPOO,LINCOLN,IL,371153267,GSA_MIGRATION,No,62656,2175251111,IL,SPRINGFIELD,PARTNER,227 S SEVENTH STREET,62701,USA,BRADLEY J PUNKE,BPUNKE@ESPCPA.COM,"ECK, SCHAFER & PUNKE, LLP",,371335003,,84,2024-01-30,2017-02-20,2017-02-21,2017-02-21,2017-02-20,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,84,non-profit,,annual,1263571,UG,true,CENSUS,2017-02-20,BRADLEY J PUNKE,PARTNER +266000,2016-06-CENSUS-0000171430,"ROANE COUNTY, TENNESSEE",COUNTY EXECUTIVE,RON WOODY,RON.WOODY@ROANECOUNTYTN.GOV,"ROANE COUNTY, TENNESSEE",8653765671,COUNTY EXECUTIVE,"200 RACE STREET, SUITE 1",KINGSTON,TN,626000806,GSA_MIGRATION,No,377630643,6154017846,TN,NASHVILLE,AUDIT REVIEW MANAGER,SUITE 1500 JAMES K. POLK BLDG.,372431402,USA,GREG WORLEY,GREG.WORLEY@COT.TN.GOV,"COMPTROLLER OF THE TREASURY, DIVISION OF LOCAL GOVERNMENT AUDIT",,626001445,,16,2024-01-30,2016-12-13,2016-12-06,2016-12-14,2016-12-13,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"16, 97, 81, 84, 14, 20, 93, 12, 66, 11, 10",local,,annual,8898269,UG,true,CENSUS,2016-12-13,GREG WORLEY,AUDIT REVIEW MANAGER +267000,2016-06-CENSUS-0000187609,"WARNER UNIVERSITY, INC.",VP FOR FINANCE AND BUSINESS,GREGORY A. RODDEN,GREG.RODDEN@WARNER.EDU,"WARNER UNIVERSITY, INC.",8636387215,VP FOR FINANCE AND BUSINESS,13895 US HWY 27,LAKE WALES,FL,591275800,GSA_MIGRATION,No,338598797,8636767981,FL,LAKE WALES,PARTNER,230 E. TILLMAN AVE.,338533714,USA,"WILLIAM M. JACOBS, CPA",BILL.JACOBS@BTICPA.COM,"BUNTING, TRIPP & INGLEY, LLP",,590673514,,84,2024-01-30,2017-03-29,2017-03-30,2017-03-29,2017-03-29,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,84,higher-ed,,annual,12049053,UG,true,CENSUS,2017-03-29,"WILLIAM M. JACOBS, CPA",PARTNER +268000,2016-09-CENSUS-0000235718,TMC DEVELOPMENT WORKING SOLUTIONS,CHIEF EXECUTIVE OFFICER,EMILY GASNER,EMILY@WORKINGSOLUTIONS.ORG,TMC DEVELOPMENT WORKING SOLUTIONS,4156555433,CHIEF EXECUTIVE OFFICER,"930 MONTGOMERY STREET, STE 400",SAN FRANCISCO,CA,911951777,GSA_MIGRATION,No,94133,4159251120,CA,LARKSPUR,PARTNER,101 LARKSPUR LANDING CIRCLE # 200,949391750,USA,MICHAEL SMITH,MSMITH@WMSHB.COM,"WILSON MARKLE STUCKEY HARDESTY & BOTT, LLP",,263789391,,21,2024-01-30,2017-04-30,2017-05-01,2017-04-28,2017-04-30,2016-09-30,2015-10-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,Yes,"11, 14, 21, 59",non-profit,,annual,2206380,UG,true,CENSUS,2017-04-30,MICHAEL SMITH,PARTNER +269000,2016-06-CENSUS-0000193820,CITY OF TRENTON,COMPTROLLER,JANET SCHOENHAAR,JSCHOENHAAR@TRENTONNJ.ORG,CITY OF TRENTON,6099894248,COMPTROLLER,319 EAST STATE STREET,TRENTON,NJ,216001242,GSA_MIGRATION,No,086082099,6096899700,NJ,HAMILTON,PRINCIPAL,3625 QUAKERBRIDGE ROAD,086197648,USA,"WARREN A. BROUDY, CPA, RMA, CGFM",WBROUDY@MERCADIEN.COM,"MERCADIEN, P.C. CPAS",,223271712,,97,2024-01-30,2017-04-23,2017-04-24,2017-04-20,2017-04-23,2016-06-30,2015-07-01,2016,single-audit,not_gaap,regulatory_basis,Yes,unmodified_opinion,No,Yes,Yes,No,Yes,750000,No,"20, 14, 10, 97, 66, 16, 93",local,,annual,13250034,UG,true,CENSUS,2017-04-23,"WARREN A. BROUDY, CPA, RMA, CGFM",PRINCIPAL +270000,2016-06-CENSUS-0000201467,MOUNT VERNON CITY SCHOOL DISTRICT,TREASURER,JUDY FORNEY,JFORNEY@MVCSD.US,MOUNT VERNON CITY SCHOOL DISTRICT,7403977422,TREASURER,300 NEWARK ROAD,MOUNT VERNON,OH,310718901,GSA_MIGRATION,No,43050,8004439275,OH,COLUMBUS,AUDIT MANAGER,88 EAST BROAD STREET,43215,USA,MELISSA KANDEL,MKKANDEL@OHIOAUDITOR.GOV,DAVE YOST,,311334820,,84,2024-01-30,2017-03-20,2017-03-21,2017-03-20,2017-03-20,2016-06-30,2015-07-01,2016,single-audit,not_gaap,cash_basis,No,unmodified_opinion,No,Yes,Yes,Yes,No,750000,No,"84, 10",local,,annual,4041049,UG,true,CENSUS,2017-03-20,MELISSA KANDEL,AUDIT MANAGER +271000,2016-06-CENSUS-0000204030,"STEPPING OUT HOUSING, INC. 121-EH308-NP-CMI-L8",ACCOUNTING MANAGER,SANDRA MARONEY,SANDY.MARONEY@ENCOMPASSCS.ORG,"STEPPING OUT HOUSING, INC. 121-EH308-NP-CMI-L8",8314691700,ACCOUNTING MANAGER,"380 ENCINAL STREET, SUITE 200",SANTA CRUZ,CA,770157232,GSA_MIGRATION,No,95060,8317630997,CA,WATSONVILLE,OWNER,36 B ASPEN WAY,95076,USA,JOHN TEUTEBERG,JJTCPA@CRUZIO.COM,"JOHN J. TEUTEBERG, CPA",,770142495,,14,2024-01-30,2016-12-01,2016-12-02,2016-12-02,2016-12-01,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,865372,UG,true,CENSUS,2016-12-01,JOHN TEUTEBERG,OWNER +272000,2016-06-CENSUS-0000206116,MCINTOSH TRAIL COMMUNITY SERVICE BOARD,CFO,STEFANIE JACKSON,SJACKSON@MCTRAIL.ORG,MCINTOSH TRAIL COMMUNITY SERVICE BOARD,7703588254,CFO,1501-A KALAMAZOO DRIVE,GRIFFIN,GA,582098758,GSA_MIGRATION,No,30224,2294463600,GA,ALBANY,MEMBER,2303 DAWSON ROAD,31708,USA,RYAN INLOW,RINLOW@MJCPA.COM,RYAN INLOW,,580692043,,93,2024-01-30,2017-03-23,2017-03-24,2017-01-31,2017-03-23,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"14, 93, 84",local,,annual,2435662,UG,true,CENSUS,2017-03-23,RYAN INLOW,MEMBER +273000,2016-12-CENSUS-0000219732,GOODWILL VILLAGE WEST II 117-HD018,"PRESIDENT, GORMAN MANAGEMENT COMPANY",THOMAS A. GORMAN,TAGORMAN@AOL.COM,GOODWILL VILLAGE WEST II 117-HD018,9183332244,"PRESIDENT, GORMAN MANAGEMENT COMPANY",398060 WEST 2200 ROAD,BARTLESVILLE,OK,731529929,GSA_MIGRATION,No,74006,9183360008,OK,BARTLESVILLE,PARTNER,302 SE OSAGE AVENUE,74003,USA,DEBORAH E. MUEGGENBORG,DEBBIE@ARCHAMBOFINANCIAL.COM,"STOTTS, ARCHAMBO, MUEGGENBORG & BARCLAY, PC",,471209122,,14,2024-01-30,2017-03-30,2017-03-31,2017-03-30,2017-03-30,2016-12-31,2016-01-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,939880,UG,true,CENSUS,2017-03-30,DEBORAH E. MUEGGENBORG,PARTNER +274000,2016-06-CENSUS-0000246480,"PROTOTYPES MERGED WITH HEALTHRIGHT 360 EFF MARCH 30,2016",CFO,TONY DUONG,TDUONG@HEALTHRIGHT360.ORG,"PROTOTYPES MERGED WITH HEALTHRIGHT 360 EFF MARCH 30,2016",4157623717,CFO,1735 MISSION STREET,SAN FRANCISCO,CA,946129071,GSA_MIGRATION,No,94103,4155683291,CA,SAN FRANCISCO,PARTNER,44 MONTGOMERY STREET,94104,USA,GRANT LAM,GRANT.LAM@ARMANINOLLP.COM,ARMANINO LLP,,946214841,,93,2024-01-30,2017-10-10,2017-09-29,2017-10-11,2017-10-10,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,Yes,"93, 16, 14",non-profit,,annual,9649915,UG,true,CENSUS,2017-10-10,GRANT LAM,PARTNER +275000,2016-06-CENSUS-0000220944,"RICHLAND RESIDENCES, INC. NFP",CFO,MARY BUCK,MBUCK56@HOTMAIL.COM,"RICHLAND RESIDENCES, INC. NFP",6183954309,CFO,RR1 4 MICAH DRIVE,OLNEY,IL,371221757,GSA_MIGRATION,No,62450,6183824151,IL,CARMI,CPA,113 E MAIN STREET,62821,USA,H. KEITH BOTSCH,CPA@BOTSCH.COM,"BOTSCH & ASSOCIATES, CPA'S, LLC",,371412029,,14,2024-01-30,2016-11-02,2016-11-03,2016-11-03,2016-11-02,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,Yes,No,750000,Yes,14,non-profit,,annual,1910588,UG,true,CENSUS,2016-11-02,H. KEITH BOTSCH,CPA +276000,2016-06-CENSUS-0000231359,SPANISH EDUCATION DEVELOPMENT (SED) CENTER,EXECUTIVE DIRECTOR,MARTHA C EGAS,MARTHAEGAS@SEDCENTER.ORG,SPANISH EDUCATION DEVELOPMENT (SED) CENTER,2027224404,EXECUTIVE DIRECTOR,4110 KANSAS AVENUE NW,WASHINGTON,DC,237147887,GSA_MIGRATION,No,200115704,7037266475,VA,ASHBURN,PRESIDENT,43482 CROSS BREEZE PLACE,20147,USA,JOHN BEHRENS,JOHN@BEHRENSCPA.COM,JOHN BEHRENS,,541556200,,93,2024-01-30,2016-11-22,2016-11-22,2016-11-21,2016-11-22,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"98, 93, 99",non-profit,,annual,2757173,UG,true,CENSUS,2016-11-22,JOHN BEHRENS,PRESIDENT +277000,2016-09-CENSUS-0000214499,GUAM DEPARTMENT OF EDUCATION,"DEPUTY SUPERINTENDENT, FINANCE & ADMIN",TALING M. TAITANO,TMTAITANO@GDOE.NET,GUAM DEPARTMENT OF EDUCATION,6713001556,"DEPUTY SUPERINTENDENT, FINANCE & ADMIN",500 MARINER AVENUE,BARRIGADA,GU,660491518,GSA_MIGRATION,No,969131608,6716463884,GU,TAMUNING,PARTNER,361 SOUTH MARINE CORPS DRIVE,96913,USA,DANIEL S. FITZGERALD,DAFITZGERALD@DELOITTE.COM,DELOITTE & TOUCHE LLP,,980047535,84,,2024-01-30,2017-05-16,2017-05-07,2017-05-16,2017-05-16,2016-09-30,2015-10-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,1930497,No,"84, 10, 93, 15, 12",state,,annual,64349925,UG,true,CENSUS,2017-05-16,DANIEL S. FITZGERALD,PARTNER +278000,2016-06-CENSUS-0000238046,PIEDMONT INTERNATIONAL UNIVERSITY,CFO,CHRIS RONK,RONKC@PIEDMONTU.EDU,PIEDMONT INTERNATIONAL UNIVERSITY,3367147987,CFO,420 S. BROAD ST,WINSTON SALEM,NC,560594591,GSA_MIGRATION,No,271015025,3367250635,NC,WINSTON SALEM,AUDIT PARTNER,2160 COUNTRY CLUB RD,27104,USA,RICHARD J. TAMER,RTAMER@CANNON.PRO,"CANNON & COMPANY, L.L.P.",,560727655,,84,2024-01-30,2017-01-19,2017-01-19,2017-01-19,2017-01-19,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,84,non-profit,,annual,4156569,UG,true,CENSUS,2017-01-19,RICHARD J. TAMER,AUDIT PARTNER +279000,2017-06-CENSUS-0000078360,RANDOLPH-MACON COLLEGE,DIRECTOR OF BUDGET AND FINANCIAL ANALYSI,CAROLINE BUSCH,CBUSCH@RMC.EDU,RANDOLPH-MACON COLLEGE,8047523267,DIRECTOR OF BUDGET AND FINANCIAL ANALYSI,PO BOX 5005,ASHLAND,VA,540505940,GSA_MIGRATION,No,23005,5403450936,VA,ROANOKE,PARTNER,319 MCCLANAHAN STREET,24014,USA,JOHN HASH,JHASH@BECPAS.COM,BROWN EDWARDS,,540504608,,84,2024-01-30,2018-03-25,2018-03-26,2018-03-26,2018-03-25,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"07, 93, 84, 47, 45, 15",higher-ed,,annual,13274702,UG,true,CENSUS,2018-03-25,JOHN HASH,PARTNER +280000,2016-06-CENSUS-0000244215,HEARTLAND CENTER FOR BEHAVIORAL CHANGE,CFO,ANNE JETTER,AJETTER@HEARTLANDCBC.ORG,HEARTLAND CENTER FOR BEHAVIORAL CHANGE,8162149505,CFO,1730 PROSPECT AVENUE,KANSAS CITY,MO,431262765,GSA_MIGRATION,No,64127,8167562620,MO,KANSAS CITY,MANAGING MEMBER,"920 MAIN STREET, SUITE 640",64105,USA,BRIAN WELCH,BWELCH@WELCHCPAFIRM.COM,"WELCH & ASSOCIATES, L.L.C.",,431794646,,93,2024-01-30,2017-03-29,2017-03-24,2017-03-30,2017-03-29,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,93,non-profit,,annual,1448045,UG,true,CENSUS,2017-03-29,BRIAN WELCH,MANAGING MEMBER +281000,2016-12-CENSUS-0000025062,MIDATLANTIC EMPLOYERS' ASSOCIATION,CHIEF OPERATING OFFICER,SHAWN MCGUIRE,SMCGUIRE@MEAINFO.ORG,MIDATLANTIC EMPLOYERS' ASSOCIATION,6109947636,CHIEF OPERATING OFFICER,"234 MALL BLVD., SUITE 200",KING OF PRUSSIA,PA,230837700,GSA_MIGRATION,No,19406,6105653930,PA,MEDIA,PARTNER,"2 WEST BALTIMORE AVE, SUITE 210",19063,USA,COLLEEN VARALLO,CVARALLO@WIPFLI.COM,WIPFLI LLP,,390758449,,11,2024-01-30,2017-05-09,2017-05-10,2017-05-10,2017-05-09,2016-12-31,2016-01-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"11, 17",non-profit,,annual,1450137,UG,true,CENSUS,2017-05-09,COLLEEN VARALLO,PARTNER +282000,2016-06-CENSUS-0000245690,ROMAN CATHOLIC BISHOP OF SPRINGFIELD,FINANCE OFFICER,WILLIAM LABROAD,WILLIAM.LABROAD@DIOSPRINGFIELD.ORG,ROMAN CATHOLIC BISHOP OF SPRINGFIELD,4134520687,FINANCE OFFICER,P. O. BOX 1730,SPRINGFIELD,MA,042106751,GSA_MIGRATION,No,011021730,4137266852,MA,BOSTON,MEMBER OF THE FIRM,99 HIGH STREET,021102320,USA,MARTIN CAINE,MCAINE@WOLFANDCO.COM,"WOLF & COMPANY, P.C.",,042689883,,97,2024-01-30,2017-02-06,2017-02-07,2017-02-07,2017-02-06,2016-06-30,2015-07-01,2016,program-specific,unmodified_opinion,,,,No,No,No,No,No,750000,No,97,non-profit,,annual,805767,UG,true,CENSUS,2017-02-06,MARTIN CAINE,MEMBER OF THE FIRM +283000,2016-06-CENSUS-0000006625,FAMILY CENTERS INC.,VICE PRESIDENT,PIERA IORI,PIORI@FAMILYCENTERS.ORG,FAMILY CENTERS INC.,2038694848,VICE PRESIDENT,40 ARCH STREET,GREENWICH,CT,060646656,GSA_MIGRATION,No,06836,2033245117,CT,STAMFORD,PARTNER,666 SUMMER STREET,06901,USA,MICHAEL V BUZZEO,MIKEBUZZEOCPA@OPTIMUM.NET,HAIMS BUZZEO & COMPANY P.C.,,061135365,,93,2024-01-30,2017-01-17,2016-12-28,2017-01-18,2017-01-17,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"14, 10, 97, 84, 93",non-profit,,annual,2166580,UG,true,CENSUS,2017-01-17,MICHAEL V BUZZEO,PARTNER +284000,2016-06-CENSUS-0000045573,"PLANNED PARENTHOOD OF INDIANA AND KENTUCKY, INC.",CHIEF FINANCIAL OFFICER,CHERYL GATZMER,CHERYL.GATZMER@PPINK.ORG,"PLANNED PARENTHOOD OF INDIANA AND KENTUCKY, INC.",3176374378,CHIEF FINANCIAL OFFICER,"200 SOUTH MERIDIAN STREET, SUITE 400",INDIANAPOLIS,IN,350874276,GSA_MIGRATION,No,46225,5029923822,KY,LOUISVILLE,DIRECTOR,2650 EASTPOINT PKWY STE. 300,40223,USA,STEVE JONES,SJONES@BLUEANDCO.COM,"BLUE & CO., LLC",,351178661,,93,2024-01-30,2017-03-26,2017-03-24,2017-03-27,2017-03-26,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,No,93,non-profit,,annual,1675441,UG,true,CENSUS,2017-03-26,STEVE JONES,DIRECTOR +285000,2016-12-CENSUS-0000036766,"PLUM PRESBYTERIAN SENIOR HOUSING, INC. DBA PLUM CREEK ACRES 033-11073",DIRECTOR OF BUDGET & FCIAL COMPLIANCE,MIKE STOKES,MSTOKES@SRCARE.ORG,"PLUM PRESBYTERIAN SENIOR HOUSING, INC. DBA PLUM CREEK ACRES 033-11073",4128266529,DIRECTOR OF BUDGET & FCIAL COMPLIANCE,1215 HULTON ROAD,OAKMONT,PA,251471814,GSA_MIGRATION,No,15139,4126976405,PA,PITTSBURGH,PARTNER,20 STANWIX STREET,15222,USA,TODD BOSLAU,TODD.BOSLAU@BAKERTILLY.COM,BAKER TILLY VIRCHOW KRAUSE,,390859910,,14,2024-01-30,2017-04-19,2017-04-18,2017-04-20,2017-04-19,2016-12-31,2016-01-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,2218929,UG,true,CENSUS,2017-04-19,TODD BOSLAU,PARTNER +286000,2018-06-CENSUS-0000135674,"CITY OF NEW BRITAIN, CONNECTICUT",ASSISTANT FINANCE DIRECTOR,REBECCA SALERNI,BSALERNI@NEWBRITAINCT.GOV,"CITY OF NEW BRITAIN, CONNECTICUT",8608263434,ASSISTANT FINANCE DIRECTOR,27 WEST MAIN STREET,NEW BRITAIN,CT,066001874,GSA_MIGRATION,No,06051,8605614000,CT,WEST HARTFORD,PARTNER,29 SOUTH MAIN STREET,06107,USA,LESLIE ZOLL,LZOLL@BLUMSHAPIRO.COM,"BLUM, SHAPIRO & COMPANY, P.C.",,061009205,,14,2024-01-30,2019-01-23,2019-01-18,2019-01-16,2019-01-23,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,809334,No,"16, 97, 10, 20, 14, 84, 93",local,,annual,26977789,UG,true,CENSUS,2019-01-23,LESLIE ZOLL,PARTNER +287000,2018-06-CENSUS-0000156753,EAST ORANGE BOARD OF EDUCATION,BOARD SECRETARY/SCHOOL BUS. ADMIN.,CRAIG SMITH,CRAIG.SMITH@EASTORANGE.K12.NJ.US,EAST ORANGE BOARD OF EDUCATION,9732665700,BOARD SECRETARY/SCHOOL BUS. ADMIN.,199 FOURTH AVENUE,EAST ORANGE,NJ,226001770,GSA_MIGRATION,No,07017,2017917100,NJ,FAIR LAWN,PARTNER,17-17 ROUTE 208 NORTH,07410,USA,DIETER P. LERCH,DLERCH@LVHCPA.COM,"LERCH, VINCI & HIGGINS, LLP",,223015339,,84,2024-01-30,2019-03-06,2019-03-07,2019-03-06,2019-03-06,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,Yes,Yes,750000,No,"10, 84, 93",local,,annual,15147143,UG,true,CENSUS,2019-03-06,DIETER P. LERCH,PARTNER +288000,2018-06-CENSUS-0000182887,NORTHWEST OREGON HOUSING AUTHORITY,FINANCE DIRECTOR,JUDY PAGAN,JUDY@NWOHA.ORG,NORTHWEST OREGON HOUSING AUTHORITY,5039945023,FINANCE DIRECTOR,147 SOUTH MAIN AVE,WARRENTON,OR,450487504,GSA_MIGRATION,No,971460000,8018729470,UT,BOUNTIFUL,CPA,PO BOX 1516,840110000,USA,BRENT STRATTON,BRENT@B2ACPA.COM,STRATTON & MILLER LLC,,811010413,,14,2024-01-30,2020-02-04,2020-02-05,2019-03-21,2020-02-04,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,Yes,No,No,750000,No,"10, 14",state,,annual,8300469,UG,true,CENSUS,2020-02-04,BRENT STRATTON,CPA +289000,2018-12-CENSUS-0000194351,CITY OF MARION,CONTROLLER,JULIE FLORES,JFLORES@CITYOFMARION.IN.GOV,CITY OF MARION,7656684460,CONTROLLER,301 SOUTH BRANSON STREET,MARION,IN,356001102,GSA_MIGRATION,No,46952,3172322514,IN,INDIANAPOLIS,STATE EXAMINER,"302 WEST WASHINGTON STREET, ROOM E418",46204,USA,"PAUL D JOYCE, CPA",FEDDATA@SBOA.IN.GOV,INDIANA STATE BOARD OF ACCOUNTS,,356000158,,20,2024-01-30,2019-09-19,2019-09-20,2019-09-20,2019-09-19,2018-12-31,2017-12-31,2018,single-audit,not_gaap,regulatory_basis,No,unmodified_opinion,No,No,Yes,No,No,750000,No,"16, 14, 20",local,,annual,1181051,UG,true,CENSUS,2019-09-19,"PAUL D JOYCE, CPA",STATE EXAMINER +290000,2018-09-CENSUS-0000221874,"CATHERINE BOOTH FRIENDSHIP HOUSE RESIDENCE, INC. 113-EE021","HUD SPECIALIST, THQ (LEGAL)",SRIPARNA MITRA,SRIPARNA.MITRA@USS.SALVATIONARMY.ORG,"CATHERINE BOOTH FRIENDSHIP HOUSE RESIDENCE, INC. 113-EE021",4047286700,"HUD SPECIALIST, THQ (LEGAL)",1901 EAST SEMINARY DRIVE,FORT WORTH,TX,030494708,GSA_MIGRATION,No,76119,4048479447,GA,ATLANTA,PARTNER,"3560 LENOX RD NE, SUITE 2800",30326,USA,DAN WORRALL,DAN.WORRALL@COHNREZNICK.COM,"COHNREZNICK, LLP",,221478099,,14,2024-01-30,2019-06-23,2019-06-20,2019-06-24,2019-06-23,2018-09-30,2017-09-30,2018,single-audit,unmodified_opinion,,,,No,No,Yes,No,No,750000,No,14,non-profit,,annual,6544865,UG,true,CENSUS,2019-06-23,DAN WORRALL,PARTNER +291000,2018-06-CENSUS-0000247643,WALHALLA COMMUNITY HOSPITAL ASSOCIATION,ADMINISTRATOR,DEBRA FRASER,DEBRA.FRASER@PEMBILIERNC.COM,WALHALLA COMMUNITY HOSPITAL ASSOCIATION,7015493310,ADMINISTRATOR,500 DELANO AVE,WALHALLA,ND,450278093,GSA_MIGRATION,No,582829704,7012398543,ND,FARGO,PARTNER,4310 17TH AVE S,58103,USA,ASHLEY BRANDT-DUDA,ABRANDTDUDA@EIDEBAILLY.COM,EIDE BAILLY LLP,,450250958,,10,2024-01-30,2018-10-23,2018-10-24,2018-10-24,2018-10-23,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,Yes,No,No,Yes,750000,No,10,non-profit,,annual,4322234,UG,true,CENSUS,2018-10-23,ASHLEY BRANDT-DUDA,PARTNER +292000,2018-06-CENSUS-0000247915,"CAREER ACADEMY OF SOUTH BEND, INC.",BUSINESS MANAGER,KIM RICHARDSON,KLNER@COMCAST.NET,"CAREER ACADEMY OF SOUTH BEND, INC.",5742999800,BUSINESS MANAGER,3801 CRESCENT CIRCLE,SOUTH BEND,IN,273113436,GSA_MIGRATION,No,46628,5742894011,IN,SOUTH BEND,PARTNER,"210 SOUTH MICHIGAN ST, STE 200",46601,USA,MARGENE ZINK,MZINK@KLCPAS.COM,"KRUGGEL, LAWTON & COMPANY, LLC",,351307701,,84,2024-01-30,2020-03-08,2020-03-06,2020-03-06,2020-03-08,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,Yes,Yes,Yes,Yes,750000,No,"84, 10",non-profit,,annual,1505032,UG,true,CENSUS,2020-03-08,MARGENE ZINK,PARTNER +293000,2023-06-GSAFAC-0000021497,Sean Witte,Associate VP Financial Operations and Controller,SEAN WITTE,WITTES@DICKINSON.EDU,DICKINSON COLLEGE,7172548149,ASSOC VP FOR FINANCIAL OPERATIONS/CONTROLLER,28 N. COLLEGE STREET,CARLISLE,PA,231365954,G4KQDKZBL848,No,17013,6177610600,MA,BOSTON,SHAREHOLDER,500 BOYLSTON STREET,02116,USA,PATRICK QUINN,PQUINN@CBIZ.COM,MAYER HOFFMAN MCCANN PC,,431947695,,84,2024-01-25,2024-01-25,2024-02-08,2024-02-08,2024-02-08,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,higher-ed,,annual,12213900,UG,true,GSAFAC,2024-02-08,PATRICK QUINN,SHAREHOLDER +294000,2023-08-GSAFAC-0000024423,Alfredo Vela,Chief Financial Officer,Alfredo Vela,alfredo.vela@cotullaisd.net,Cotulla Independent School District,8308793073,Finance Director,310 N. Main Street,Cotulla,TX,746003466,UE6CQEMSBJY3,No,78014,2102271389,TX,San Antonio,Manager,207 Arden Grove St,78215,USA,Ruben Martinez,rmartinez@gga-cpa.biz,Garza/Gonzalez & Associates,,741867466,,84,2024-02-09,2024-02-12,2024-02-12,2024-02-14,2024-02-14,2023-08-31,2022-09-01,2023,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,Yes,00,local,,annual,3521699,UG,true,GSAFAC,2024-02-14,RUBEN MARTINEZ,AUDIT MANAGER +295000,2023-08-GSAFAC-0000023208,Vicki Taylor,Director of Business Services,Vicki Taylor,vicki.taylor@esu6.org,Educational Service Unit No. 6,4027617011,Director of Business Services,210 5th Street,Milford,NE,470495481,DGKDW9G4J6W5,No,68405,4024799300,NE,Lincoln,Auditor,"1248 O Street, Suite 500",68508,USA,Kerry Gustafsson,gustafsson@danacole.com,"Dana F. Cole & Company, LLP",,470526649,,84,2024-02-02,2024-02-06,2024-02-21,2024-02-21,2024-02-21,2023-08-31,2022-09-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,00,local,,annual,1500123,UG,true,GSAFAC,2024-02-21,"KERRY GUSTAFSSON, CPA",PARTNER +296000,2023-06-GSAFAC-0000001380,Sherri Rowland,Controller,Sarah Balas,sbalas@clemson.edu,Clemson University,8646566598,Sr. Grant Supervisor,G06 Sikes Hall,Clemson,SC,576000254,H2BMNX7DSKU8,No,29634,4103088064,MD,Timonium,Principal,"1966 Greenspring Drive, Suite 300",21093,USA,Christina Bowman,christina.bowman@claconnect.com,Clifton Larson Allen LLP (CLA),,410746749,84,,2023-10-09,2023-11-08,2024-02-16,2024-02-27,2024-02-27,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,3000000,Yes,00,higher-ed,,annual,302699610,UG,true,GSAFAC,2024-02-27,Christina Bowman,Principal +297000,2023-06-GSAFAC-0000028111,Charles Peckio,Superintendent,Richard Peckio,rpeckio@frink.k12.ok.us,Frink-Chambers School District,9184232434,Superintendent,PO Box 699,McAlester,OK,770648127,YR6NN8E1FP55,No,74502,9182508838,OK,Broken Arrow,Partner,2101 N Willow Ave,74012,USA,Kerry Patten,kerry.patten@pattencpa.net,"Patten & Odom, CPAs, PLLC",,931904036,,84,2024-03-04,2024-03-04,2024-03-04,2024-03-04,2024-03-04,2023-06-30,2022-07-01,2023,single-audit,not_gaap,other_basis,No,qualified_opinion,No,No,No,No,No,750000,No,00,local,,annual,908838,UG,true,GSAFAC,2024-03-04,Kerry Patten,Partner +298000,2023-05-GSAFAC-0000027922,Timothy Walker,COO,Timothy Walker,tw@eechealth.com,Elaine Ellis Center of Health,2028032340,COO,1627 Kenilworth AVE NE,Washington,DC,273048576,RFJKYDVW7NJ3,No,20019,2023935600,DC,Washington,Partner,1090 Vermont AVE Suite 250 NW,20005,USA,Stephanie Lane,slane@bertsmithco.com,Bert Smith & CO,,521094722,,93,2024-03-01,2024-03-11,2024-03-11,2024-03-11,2024-03-11,2023-05-31,2022-06-01,2023,single-audit,unmodified_opinion,,,,No,Yes,No,No,Yes,750000,No,00,non-profit,,annual,4182911,UG,true,GSAFAC,2024-03-11,Stephanie Lane,Partner +299000,2023-06-GSAFAC-0000030499,Melinda Brouillard,Interim Finance Director,Melinda Brouillard,mbrouillard@cumberlandri.org,"Town of Cumberland, Rhode Island",4017282400,Deputy Finance Director,45 Broad Street,Cumberland,RI,056000115,QRL4RNF9E5L8,No,02864,4016004722,RI,Providence,Partner,"100 Westminster Street, Suite 500",02903,USA,Kyle Connors,kyle.connors@marcumllp.com,Marcum LLP,,111986323,,84,2024-03-15,2024-03-15,2024-03-15,2024-03-15,2024-03-15,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,Yes,No,No,750000,No,00,local,,annual,9686909,UG,true,GSAFAC,2024-03-15,Kyle Connors,Partner +300000,2022-12-GSAFAC-0000008542,Sabine Cox,Comptroller,Sabine Cox,scox@ehmchm.org,Chestnut Hill Housing Inc,2032304809,Comptroller EHM,40 Edgemont Lane,Wolcott,CT,061451581,JH36NMXLMH88,No,06716,2032977700,CT,Woodbridge,Partner,16 Lunar Drive,06525,USA,Brian S Borgerson CPA,bborg@baileymoore.com,Bailey Moore Glazer Schaeffer & Proto LLP,,060674931,,14,2023-11-22,2024-03-21,2024-03-21,2024-03-21,2024-03-21,2022-12-31,2022-01-01,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,3395066,UG,true,GSAFAC,2024-03-21,"Brian S Borgerson, CPA",Partner +301000,2023-06-GSAFAC-0000005142,C. Shawn Boehringer,Executive Director,SHAWN BOEHRINGER,SBOEHRINGER@LASP.ORG,"LEGAL AID OF SOUTHEASTERN PENNSYLVANIA, INC.",6102755400,EXECUTIVE DIRECTOR,625 SWEDE STREET,NORRISTOWN,PA,231901014,MNVEJFEMKAJ4,No,19401,7172321230,PA,HARRISBURG,ENGAGEMENT PARTNER,"1800 LINGLESTOWN ROAD, SUITE 306",17110,USA,LISA RITTER,LRITTER@MD-CPAS.COM,"MAHER DUESSEL, CPAS",,251622758,,09,2023-11-02,2024-03-14,2024-03-14,2024-03-25,2024-03-25,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,00,non-profit,,annual,3578623,UG,true,GSAFAC,2024-03-25,Lisa Ritter,Engagement Partner +302000,2023-06-GSAFAC-0000023768,Tina Leingang,Business Manager,Tina Leingang,TINA.LEINGANG@K12.ND.US,STANDING ROCK COMMUNITY GRANT SCHOOL,7018542142,Business Manager,9889 HWY 24,Fort Yates,ND,450407044,EYC8CFBB3N89,No,58538,9897321156,MI,GAYLORD,SENIOR AUDIT PARTNER,215 S COURT AVE,49735,USA,J. Kurt Tucker,KURT@MWP.US.COM,MIDWEST PROFESSIONALS,,383571340,,15,2024-02-06,2024-03-26,2024-03-26,2024-03-27,2024-03-27,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,tribal,,annual,14627976,UG,true,GSAFAC,2024-03-27,Christopher Preville,Auditor +303000,2023-06-GSAFAC-0000029224,Dennis Meyer,Budget & Fiscal Director,Dennis Meyer,DMMEYER@LANCASTER.NE.GOV,"Lancaster County, Nebraska",4024416869,Budget and Fiscal Officer,"555 South 10th Street, Room 110",Lincoln,NE,476006482,PGJKWY8LBAR1,No,68508,8167514027,NE,Omaha,Partner,1299 Farnam St.,68102,USA,Kevin Smith,kevin.smith@rsmus.com,RSM US LLP,,420714325,,21,2024-03-08,2024-03-28,2024-03-28,2024-03-28,2024-03-28,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,local,,annual,20064033,UG,true,GSAFAC,2024-03-28,Kevin Smith,Partner +304000,2023-06-GSAFAC-0000033720,Diana Snyder,Finance Director,Diana Snyder,dsnyder@hrdewv.org,"Marion Unity Apartments, Inc.",3042968223,Finance Director,1369 Stewartstown Road,Morgantown,WV,550678214,RNCDTKSYJ5K2,No,26505,3043434603,WV,Charleston,Partner,P.O. Box 1991,25327,USA,R. Gabe Rushden,gabe.rushden@blaircpas.com,Blair & Company,,844083294,,14,2024-03-26,2024-03-28,2024-03-28,2024-03-29,2024-03-29,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,5072057,UG,true,GSAFAC,2024-03-29,R. Gabe Rushden,Partner +305000,2022-06-GSAFAC-0000035727,Christy Ramirez,Finance Director,CHRISTY RAMIREZ,CRAMIREZ@SPCITY.ORG,CITY OF SANTA PAULA,8059334211,FINANCE DIRECTOR,970 E. VENTURA STREET,SANTA PAULA,CA,956000791,TDLLJXMQYPJ9,No,93060,9098890871,CA,SAN BERNARDINO,PARTNER,735 E. CARNEGIE DRIVE SUITE 100,92408,USA,BRIANNA SCHULTZ,BSCHULTZ@RAMSCPA.NET,"ROGERS, ANDERSON, MALODY & SCOTT, LLP.",,952662063,,97,2024-03-29,2024-03-29,2024-03-29,2024-03-29,2024-03-29,2022-06-30,2021-07-01,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,00,local,,annual,3002983,UG,true,GSAFAC,2024-03-29,Brianna Schultz,Partner +306000,2023-06-GSAFAC-0000036275,Yolanda Brumfield,Director of Finance,YOLANDA BRUMFIELD,YBRUMFIELD@SOWIB.ORG,SOUTHWESTERN OREGON WORKFORCE INVESTMENT BOARD,8445326893,FINANCE DIRECTOR,PO BOX 415,COOS BAY,OR,473873798,G3LVWKKZN9L9,No,97420,3605333370,WA,Montesano,PRINCIPAL,"324 S Main Street, Unit A",98563,USA,Lonnie Rich,lonnie@aiken-sanders.com,AIKEN & SANDERS INC. PS,,910870697,,11,2024-04-01,2024-04-01,2024-04-01,2024-04-01,2024-04-01,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,non-profit,,annual,4946972,UG,true,GSAFAC,2024-04-01,Lonnie Rich,Principal +307000,2023-09-GSAFAC-0000037293,Timothy S Brown,CFO,Tim Brown,tbrown@pinesbhs.org,Branch County Community Mental Health Authority,5172782129,Chief Financial Officer,200 Orleans Blvd.,Coldwater,MI,383622335,P27VJPZMNB76,No,49036,9894636123,MI,Alma,Partner,"525 West Warwick Drive, Suite A",48801,USA,Christina Schaub,christina.schaub@rpccpas.com,"Roslund, Prestage & Company, P.C.",,382462587,,93,2024-04-10,2024-04-10,2024-04-10,2024-04-10,2024-04-10,2023-09-30,2022-10-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,00,local,,annual,2470496,UG,true,GSAFAC,2024-04-10,Christina Schaub,Partner +308000,2023-12-GSAFAC-0000036438,Takisha Artis,Controller,TAKISHA ARTIS,takisha@hsrpro.com,"SILENT COOPERATIVE APARTMENTS, INC.",8158069990,OUTSOURCED CFO,2500 WEST BELMONT AVENUE,CHICAGO,IL,363390850,NELBSURV5F29,No,60618,8475804138,IL,LINCOLNSHIRE,PARTNER,100 TRI STATE INTERNATIONAL STE 300,60069,USA,ALEX PEKLER,ALEX.PEKLER@WIPFLI.COM,WIPFLI LLP,,390758449,,14,2024-04-02,2024-04-22,2024-04-22,2024-04-22,2024-04-22,2023-12-31,2023-01-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,non-profit,,annual,6389958,UG,true,GSAFAC,2024-04-22,Alex Pekler,Partner +309000,2023-12-GSAFAC-0000023806,Julie Reed,Housing Accounting Manager,JULIE REED,JREED@UCHINC.ORG,"UCC II, INC. D/B/A STERLING PLACE 067-EE047-WAH",7403824885,MANAGER OF HOUSING ACCOUNTING,170 East Center Street P.O. Box 1806,MARION,OH,341789176,P9YSGYCPJNV1,No,43302,6148493000,OH,COLUMBUS,ENGAGEMENT PARTNER,250 SOUTH HIGH STREET SUITE 100,43215,USA,JASON REES,Jason.rees@plantemoran.com,"PLANTE & MORAN, PLLC",,381357951,,14,2024-02-07,2024-04-28,2024-04-28,2024-04-30,2024-04-30,2023-12-31,2023-01-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,non-profit,,annual,4141930,UG,true,GSAFAC,2024-04-30,Jason T Rees,Partner +310000,2023-06-GSAFAC-0000036113,Eric C Hern,CFO,Erin Hern,ehern@guidancecenter.net,The Guidance Center,8143626536,Senior Director of Operations & Chief Financial Officer,110 Campus Drive,Bradford,PA,251469392,PB38PFWQMND7,No,16701,4123677102,PA,Pittsburgh,Partner,"3800 McKnight East Drive, Suite 3805",15237,USA,Brian Chruscial,bchruscial@zallc.org,Zelenkofske Axelrod LLC,,233022325,,93,2024-04-01,2024-04-01,2024-04-01,2024-04-09,2024-05-13,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,00,non-profit,,annual,3035973,UG,true,GSAFAC,2024-05-13,Brian Chruscial,Partner +311000,2023-12-GSAFAC-0000038239,Beth Long,CFO,Beth Long,beth.long@tphtrust.org,"South Main Street Housing Trust, Inc.",8022917000,CFO,"226 Holiday Drive, Suite 20",White River Junction,VT,030364340,FWBVY17ZZU35,No,05001,2077801100,ME,South Portland,Partner,324 Gannett Drive,04106,USA,Bridget Sylvester,bsylvester@otisatwell.com,Otis Atwell,,203690847,,10,2024-04-18,2024-05-23,2024-05-23,2024-05-23,2024-05-23,2023-12-31,2023-01-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,non-profit,,annual,13424945,UG,true,GSAFAC,2024-05-23,Bridget Sylvester,Partner +312000,2023-12-GSAFAC-0000042594,Peter Halle,President,Peter Halle,peterhalle@yahoo.com,"Salisbury Housing Committee, Inc.",8604350049,President,"34 Cobble Street, P.O. Box 10",Salisbury,CT,115309016,LCCQX834NCQ7,No,06068,2032489341,CT,North Haven,Partner,296 State St,06473,USA,Michele Loso Boisvert,mloso@sewardmonde.com,Seward and Monde,,060530830,,10,2024-06-03,2024-06-04,2024-06-04,2024-06-04,2024-06-04,2023-12-31,2023-01-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,10,non-profit,,annual,1124543,UG,true,GSAFAC,2024-06-04,Michele Loso Boisvert,Partner +313000,2023-06-GSAFAC-0000042985,Lori Willis,Business Manager,LORI WILLIS,LORI.WILLIS@SMACKOVER.NET,SMACKOVER-NORPHLET SCHOOL DISTRICT NO. 39,8707253132,BUSINESS MANAGER,112 E 8TH ST,SMACKOVER,AR,716021546,HTUUC34B8SX5,No,71762,5012252133,AR,LITTLE ROCK,PARTNER,P.O. BOX 21675,72221,USA,MICHAEL L. COBB,MCOBB@COBBANDSUSKIE.COM,"COBB AND SUSKIE, LTD.",,710671623,,84,2024-06-05,2024-06-10,2024-06-10,2024-06-17,2024-06-17,2023-06-30,2022-07-01,2023,single-audit,not_gaap,other_basis,Yes,unmodified_opinion,No,No,No,No,No,750000,No,00,local,,annual,2424712,UG,true,GSAFAC,2024-06-17,Melissa Hodgson,Auditor Certifying Official +314000,2023-09-GSAFAC-0000044342,Todd Blount,Fiscal Officer,Todd Blount,tblount@ecpdd.org,"EAST CENTRAL PLANNING & DEVELOPMENT DISTRICT, INC.",6016831220,Fiscal Officer,PO BOX 499,NEWTON,MS,640468881,HCG8JZ9N18K7,No,39345,6624485885,MS,Houston,Partner,109 N Jackson St - PO box 391,38851,USA,Jason Brooks,Jason.Brooks@wws.cpa,"Watkins, Ward and Stafford, PLLC",,640394922,,11,2024-06-18,2024-06-18,2024-06-24,2024-06-25,2024-06-25,2023-09-30,2022-10-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,local,,annual,5683747,UG,true,GSAFAC,2024-06-25,Jason Brooks,Partner +315000,2023-09-GSAFAC-0000046014,Jawan Warren Simpson,Chief Financial Officer,JAWAN SIMPSON,jsimpson@waynecountymi.gov,WAYNE COUNTY HEALTHY COMMUNITIES,3138263320,INTERIM CHEIF FINANCIAL OFFICER,9011 JOSEPH CAMPAU,HAMTRAMCK,MI,300672911,KZMDRDLPTD44,No,48212,3139645270,MI,DETROIT,PRESIDENT,PO BOX 32605,48232,USA,GLEN OLIVACHE,olivache@sbcglobal.net,GLEN OLIVACHE CPA PC,,382770838,,93,2024-06-27,2024-06-28,2024-06-28,2024-06-28,2024-06-28,2023-09-30,2022-10-01,2023,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,No,93,non-profit,,annual,2321019,UG,true,GSAFAC,2024-06-28,GLEN OLIVACHE,PRESIDENT +3000,2023-06-GSAFAC-0000002638,Mike Andreshak,Director of Business Services/CSBO,Mike L Andreshak,mandreshak@ksd140.org,Kirby School District 140,7085326462,Director of Business Services/CSBO,16931 S Grissom Drive,Tinley Park,IL,366004355,XT85V8G9QLM8,No,60477,6305668574,IL,Rockford,"Partner-in-Charge, Government Services","401 W State Street,Suite 509",61101,USA,Anthony Cervini,anthony.cervini@sikich.com,Sikich,,363168081,,84,2023-10-18,2023-10-18,2023-10-18,2023-10-18,2023-10-18,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,00,local,,annual,2880895,UG,true,GSAFAC,2023-10-18,DEFAULT_AUDITOR_CERTIFY_NAME,DEFAULT_AUDITOR_CERTIFY_TITLE +4000,2023-01-GSAFAC-0000003440,ORLANDO TORRES,FINANCE DEPARTMENT DIRECTOR,ORLANDO TORRES,orlando.torres@chs015.com,"CAMUY HEALTH SERVICES, INC.",7872626603,FINANCE DIRECTOR,PO BOX 660,CAMUY,PR,660428652,NJSNXFBJ2LP5,No,00627,7877935353,PR,San Juan,PRESIDENT,PO BOX 194806,00919,USA,JOSE LUIS CARDONA GONZALEZ,jlc@jlcardona.com,José L. Cardona & Co. PSC,,660567451,,93,2023-10-23,2023-10-24,2023-10-24,2023-10-24,2023-10-24,2023-01-31,2022-02-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,non-profit,,annual,4241758,UG,true,GSAFAC,2023-10-24,DEFAULT_AUDITOR_CERTIFY_NAME,DEFAULT_AUDITOR_CERTIFY_TITLE +5000,2023-03-GSAFAC-0000004663,Tadd S. Greenfield,Chief Executive Officer,Gayl Smith,gsmi4@cabinetpeaks.org,St. John's Lutheran Hospital DBA Cabinet Peaks Medical Center,4062837139,Controller,209 Health Park Dr.,Libby,MT,810241755,YMEAZVHDGJ56,No,59923,4068962449,MT,BILLINGS,Partner,401 N 31ST ST STE 1120,59103,USA,Scott Nelson,snelson@eidebailly.com,Eide Bailly LLP,,450250958,,14,2023-10-31,2023-11-01,2023-11-01,2023-11-01,2023-11-01,2023-03-31,2022-04-01,2023,single-audit,unmodified_opinion,,,,No,Yes,Yes,No,No,750000,No,00,non-profit,,annual,22622437,UG,true,GSAFAC,2023-11-01,DEFAULT_AUDITOR_CERTIFY_NAME,DEFAULT_AUDITOR_CERTIFY_TITLE +6000,2023-06-GSAFAC-0000006066,Laura Parnell,Director of Finance,LAURA PARNELL,LBPARNELL@ASD2.ORG,ANDERSON COUNTY SCHOOL DISTRICT TWO,8643697364,DIRECTOR OF FINANCE,10990 BELTON HONEA PATH HIGHWAY,HONEA PATH,SC,576003892,XNC8JL9DLSX5,No,29654,8642325204,SC,Mauldin,PARTNER,"211 E Butler Road, Suite C-6",29662,USA,KEN MEADOWS,KEN.MEADOWS@GFC.CPA,"Greene Finney Cauley, LLP",,522212837,,84,2023-11-08,2023-11-09,2023-11-10,2023-11-14,2023-11-14,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,00,local,,annual,7617713,UG,true,GSAFAC,2023-11-14,DEFAULT_AUDITOR_CERTIFY_NAME,DEFAULT_AUDITOR_CERTIFY_TITLE +7000,2023-06-GSAFAC-0000007575,Jeffrey Carraway,CFO,Jeffrey Carraway,Jeffrey.Carraway@lutheranseniorlife.org,Titusville Senior Housing Corporation,7247422246,CFO,127 Beechwood Drive,Titusville,PA,270115829,N457WM5LA4V1,No,16354,3302664124,OH,Canton,Principal,4334 Munson St.,44718,USA,Kyle Bowers,kyle.bowers@claconnect.com,CliftonLarsonAllen,,410746749,,14,2023-11-16,2023-11-22,2023-11-22,2023-11-27,2023-11-27,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,No,14,non-profit,,annual,1821192,UG,true,GSAFAC,2023-11-27,DEFAULT_AUDITOR_CERTIFY_NAME,DEFAULT_AUDITOR_CERTIFY_TITLE +8000,2023-06-GSAFAC-0000005626,Jessica S. Rogers,Chief Financial Officer,Jessica Rogers,jessica.roger1@nelson.kyschools.us,Nelson County Board of Education,5023314310,Chief Financial Officer,1244 Tom Greer Road,Bardstown,KY,616001240,CBD2FKG94EZ3,No,40004,8598081329,KY,Winchester,Partner,124 Candlewood Drive,40391,USA,Tammy Patrick,tammy@patrickassoc.com,"Patrick & Associates, LLC",,831439494,,84,2023-11-06,2023-12-01,2023-12-01,2023-12-04,2023-12-04,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,unknown,,annual,9280538,UG,true,GSAFAC,2023-12-04,DEFAULT_AUDITOR_CERTIFY_NAME,DEFAULT_AUDITOR_CERTIFY_TITLE +9000,2022-06-GSAFAC-0000010150,Daniel Amato,Executive Director,Daniel Amato,damato@mvap.org,Merrimack Valley Assistance Program,6032260607,Executive Director,170 Lowell Street,Manchester,NH,223396725,MP9VTSBDMVG3,No,03104,6036227070,NH,Manchester,Audit Manager,608 Chestnut Street,03104,USA,Cory Philbrick,cphilbrick@vachonclukay.com,Vachon Clukay & Company PC,,020400031,,14,2023-12-04,2023-12-04,2023-12-04,2023-12-11,2023-12-11,2022-06-30,2021-07-01,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,00,non-profit,,annual,819015,UG,true,GSAFAC,2023-12-11,DEFAULT_AUDITOR_CERTIFY_NAME,DEFAULT_AUDITOR_CERTIFY_TITLE +10000,2023-06-GSAFAC-0000011690,Pamela Rocha,Chief Financial Officer,Pamela Rocha,pamelarocha@nafi.com,"NFI Vermont, Inc.",9788824868,CFO,30 Airport Road,Burlington,VT,030304434,SQT2TJZXMW81,No,05403,5084413300,MA,Quincy,Principal,"4 Batterymarch Park, Suite 100",02169,USA,Jennifer Olivier,jen.olivier@claconnect.com,CliftonLarsonAllen LLP,,410746749,,93,2023-12-08,2023-12-15,2023-12-15,2023-12-18,2023-12-18,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,non-profit,,annual,1227738,UG,true,GSAFAC,2023-12-18,DEFAULT_AUDITOR_CERTIFY_NAME,DEFAULT_AUDITOR_CERTIFY_TITLE +11000,2023-06-GSAFAC-0000014202,Chris Baird,Controller,Chris Baird,chris.baird@g-b.com,Edison Terrace South,5093444904,CFO,"818 W Riverside, Suite 300",Spokane,WA,264362933,EJ98S93Z13T6,No,99201,5097472158,WA,Spokane,CPA,"422 W Riverside Ave, Suite 1420",99201,USA,Darcy Skjothaug,dskjothaug@schoedel.com,Schoedel & Schoedel CPAs PLLC,,910614823,,14,2023-12-19,2023-12-19,2023-12-19,2023-12-21,2023-12-21,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,non-profit,,annual,1726491,UG,true,GSAFAC,2023-12-21,DEFAULT_AUDITOR_CERTIFY_NAME,DEFAULT_AUDITOR_CERTIFY_TITLE +12000,2023-06-GSAFAC-0000015058,Anna Kasprzyk,Chief School Business Official,ANNA KASPRZYK,anna.kasprzyk@district41.org,LAKE VILLA COMMUNITY CONSOLIDATED SCHOOL DISTRICT NO. 41,8473562385,CHIEF SCHOOL BUSINESS OFFICIAL,131 MCKINLEY AVENUE,LAKE VILLA,IL,366004850,HSDYSNMJBCX7,No,60046,8153441300,IL,MCHENRY,PARTNER,"5400 WEST ELM STREET, SUITE 203",60050,USA,CHERYDEN JUERGENSEN,cjuergensen@eccezion.com,ECCEZION,,363614997,,84,2023-12-22,2023-12-22,2023-12-22,2024-01-02,2024-01-02,2023-06-30,2022-07-01,2023,single-audit,"adverse_opinion, not_gaap",cash_basis,No,unmodified_opinion,No,No,No,No,No,750000,No,00,local,,annual,2703240,UG,true,GSAFAC,2024-01-02,DEFAULT_AUDITOR_CERTIFY_NAME,DEFAULT_AUDITOR_CERTIFY_TITLE +13000,2023-06-GSAFAC-0000013380,Suwen Su,"Director, Fiscal Services",SUWEN SU,SSU@ROSEMEAD.K12.CA.US,ROSEMEAD SCHOOL DISTRICT,6263122900,DIRECTOR OF FISCAL SERVICES,3907 ROSEMEAD BLVD,ROSEMEAD,CA,956002586,CQ4LFQM3M1J5,No,91770,3106702745,CA,CULVER CITY,PARTNER,"5800 HANNUM AVE, SUITE E",90230,USA,HADLEY HUI,HHUI@MLHCPAS.COM,"MOSS, LEVY & HARTZHEIM, LLP",,753194011,,84,2023-12-15,2024-01-11,2024-01-11,2024-01-12,2024-01-12,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,local,,annual,7492901,UG,true,GSAFAC,2024-01-12,HADLEY HUI,partner +14000,2023-06-GSAFAC-0000016731,Megan Leonard,Director of Management & Finance,MEGAN LEONARD,MLEONARD@NORPC.ORG,"REGIONAL PLANNING COMMISSION FOR JEFFERSON, ORLEANS, PLAQUEMINES, ST.",5044838525,DIRECTOR OF MANAGEMENT AND FINANCE,10 VETERANS BLVD,NEW ORLEANS,LA,720595531,YJJLYGNK6FE5,No,70124,5048332436,LA,Metairie,Partner,"111 VETERANS BOULEVARD, SUITE 350",70005,USA,Becky Hammond,BHAMMOND@CRICPA.COM,"CARR, RIGGS & INGRAM, LLC",,721396621,,20,2024-01-03,2024-01-11,2024-01-11,2024-01-11,2024-01-11,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,local,,annual,2859431,UG,true,GSAFAC,2024-01-11,Becky Hammond,Partner +15000,2023-06-GSAFAC-0000014261,Denise M Mooney,Treasurer,DENISE MOONEY,MOONEY.D@VANTAGECAREERCENTER.COM,VANTAGE CAREER CENTER,4192385411,TREASURER,818 NORTH FRANKLIN STREET,VAN WERT,OH,341146218,D47MDVJZKMF5,No,45891,9372856677,OH,Columbus,SENIOR AUDIT MANAGER,88 East Broad Street,43215,USA,KATIE EDDY,KAEDDY@OHIOAUDITOR.GOV,Ohio Auditor of State,,311334820,,84,2023-12-20,2023-12-28,2023-12-28,2024-01-17,2024-01-17,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,Yes,Yes,No,750000,No,21,local,,annual,855829,UG,true,GSAFAC,2024-01-17,Katie Eddy,Senior Audit Manager +16000,2023-06-GSAFAC-0000016547,Jordan Tuten,Finance/Human Resources Director,Jordan Tuten,jtuten@ridgelandsc.gov,Town of Ridgeland,8437267503,Finance/HR Director,One Town Square,Ridgeland,SC,566001100,V26EFCE28S31,No,29936,8037393090,SC,West Columbia,Partner,501 State Street,29169,USA,William Hancock,whancock@tbgcpas.com,"The Brittingham Group, LLP",,464116137,,21,2024-01-03,2024-01-03,2024-01-09,2024-01-10,2024-01-23,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,00,local,,annual,863743,UG,true,GSAFAC,2024-01-23,William H Hancock,Engagement Partner +17000,2023-09-GSAFAC-0000021853,Angela Daigle,"Senior Director, Finance and Affordable Housing",Angie Daigle,adaigle@goodwillacadiana.org,"Village du Vie, Inc.",3372615811,Senior Director Finance and Affordable Housing,1104 Tarleton Street,Jeanerette,LA,582089102,E8SEUJ2L75Y9,No,70544,3372323312,LA,Lafayette,Partner,2000 Kaliste Saloom Rd Suite 300,70508,USA,Christine Dunn,christied@dsfcpas.com,"Darnall, Sikes and Frederick CPAs",,720738838,,14,2024-01-26,2024-01-26,2024-01-26,2024-01-27,2024-01-27,2023-09-30,2022-10-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,non-profit,,annual,1031087,UG,true,GSAFAC,2024-01-27,Christine R. Dunn,Partner +18000,2023-06-GSAFAC-0000021348,Joseph Sterbank,Business Administrator,JOSEPH STERBANK,JSTERBANK@ONONDAGACSD.ORG,ONONDAGA CENTRAL SCHOOL DISTRICT,3155525001,SCHOOL BUSINESS ADMINISTRATOR,4466 SOUTH ONONDAGA RD,NEDROW,NY,156002303,VL79J2GZ6JK3,No,13120,5854231860,NY,rochester,partner,"100 chestnut street, suite 1200",14604,USA,thomas zuber,jnobles@mmb-co.com,mmb+co.,,161092347,,84,2024-01-24,2024-01-25,2024-01-29,2024-01-29,2024-02-01,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,00,state,,annual,1617460,UG,true,GSAFAC,2024-02-01,thomas zuber,partner +19000,2023-06-GSAFAC-0000006751,Kristin Rossow,VP of Accounting,Kristin Rossow,kristin.rossow@mosaicinfo.org,Mosaic Housing Corporation VII,4028963884,VP of Accounting,4980 South 118th St,Omaha,NE,470828015,WU6JS45L3Z36,No,68137,7012398627,ND,Fargo,Partner,4310 17th Ave S PO Box 2545,58108,USA,Reggie MacMaster,rmacmaster@eidebailly.com,Eide Bailly LLP,,450250958,,14,2023-11-13,2023-11-13,2024-02-05,2024-02-07,2024-02-07,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,non-profit,,annual,903868,UG,true,GSAFAC,2024-02-07,Reggie MacMaster,Partner +20000,2022-12-CENSUS-0000258232,VILLAGE OF HAWTHORN WOODS,CHIEF FINANCIAL OFFICER,KATREINA YORK,KYORK@VHW.ORG,VILLAGE OF HAWTHORN WOODS,8474385500,CHIEF FINANCIAL OFFICER,2 LAGOON DRIVE,HAWTHORN WOODS,IL,362726116,WT7DKVMQLN86,No,60047,6305668400,IL,NAPERVILLE,PARTNER,"1415 W. DIEHL ROAD, SUITE 400",60563,USA,BRIAN LEFEVRE,BRIAN.LEFEVRE@SIKICH.COM,SIKICH LLP,,366168081,,21,2024-01-19,2023-05-30,2023-05-30,2023-05-30,2023-05-30,2022-12-31,2021-12-31,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,21,local,,annual,1179310,UG,true,CENSUS,2023-05-30,BRIAN LEFEVRE,PARTNER +21000,2022-09-CENSUS-0000042728,GREAT LAKES COMMUNITY ACTION PARTNERSHIP,CHIEF FINANCIAL OFFICER,DAVID KIPPLEN,DRKIPPLEN@GLCAP.ORG,GREAT LAKES COMMUNITY ACTION PARTNERSHIP,4193348919,CHIEF FINANCIAL OFFICER,127 FRONT STREET,FREMONT,OH,340975934,NJFNHGSBUJU1,No,43420,6082741980,WI,MADISON,PARTNER,PO BOX 8700,537168700,USA,"JOHN HEMMING, CPA",JHEMMING@WIPFLI.COM,WIPFLI LLP,,390758449,,93,2024-01-19,2023-06-07,2023-06-08,2023-06-07,2023-06-07,2022-09-30,2021-09-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,1339061,Yes,"66, 10, 14, 21, 17, 20, 84, 19, 93, 64, 23, 81",non-profit,,annual,48219750,UG,true,CENSUS,2023-06-07,"JOHN HEMMING, CPA",PARTNER +22000,2022-06-CENSUS-0000231406,"EDKEY, INC.",CFO,PATRIC GREER,PGREER@EDKEY.ORG,"EDKEY, INC.",4804613200,CFO,2345 N HORNE,MESA,AZ,743033931,VN5GRJGAAXJ4,No,852031823,4807527728,AZ,TEMPE,PARTNER,2055 E WARNER ROAD STE 101,85284,USA,COLETTE KAMPS,COLETTE.KAMPS@BAKERTILLY.COM,"BAKER TILLY U.S., LLP",,390859910,,84,2024-01-19,2023-01-15,2023-01-13,2023-01-16,2023-01-15,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"84, 10, 93, 99",non-profit,,annual,20290249,UG,true,CENSUS,2023-01-15,COLETTE KAMPS,PARTNER +23000,2022-06-CENSUS-0000218633,"ASI - FLORENCE, INC.",ACCOUNTING MANAGER,CHUCK REUTER,CREUTER@ACCESSIBLESPACE.ORG,"ASI - FLORENCE, INC.",6516457271,ACCOUNTING MANAGER,2550 UNIVERSITY AVE. W. SUITE 330 N,ST. PAUL,MN,631251743,Q12WMUTM7CK6,No,55114,9525428010,MN,MINNEAPOLIS,SHAREHOLDER,"1000 SHELARD PARKWAY, STE. 110",55426,USA,BOB BAKER,BOB@HINRICHSASSOCIATES.COM,"HINRICHS & ASSOCIATES, LTD",,411629155,,14,2024-01-19,2023-02-28,2023-03-01,2023-02-25,2023-02-28,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,14,non-profit,,annual,1663370,UG,true,CENSUS,2023-02-28,BOB BAKER,SHAREHOLDER +24000,2022-06-CENSUS-0000251368,"CATHOLIC CHARITIES OF CENTRAL COLORADO, INC.",CFO,LORI KAPU,LKAPU@CCHARITIESCC.ORG,"CATHOLIC CHARITIES OF CENTRAL COLORADO, INC.",7198666423,CFO,228 NORTH CASCADE AVE,COLORADO SPRINGS,CO,840586169,J86JMKV4H2B6,No,80903,7196362321,CO,COLORADO SPRINGS,PARTNER,601 NORTH NEVADA AVENUE,80903,USA,MITCHELL DOWNS,MITCH@EBKCPA.COM,"ERICKSON, BROWN & KLOSTER, LLC",,840957308,,21,2024-01-19,2023-01-03,2023-01-04,2022-12-16,2023-01-03,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"14, 10, 16, 93, 97, 21",non-profit,,annual,4453358,UG,true,CENSUS,2023-01-03,MITCHELL DOWNS,PARTNER +25000,2022-06-CENSUS-0000232685,PLUMAS LAKE ELEMENTARY SCHOOL DISTRICT,DIRECTOR OF BUSINESS SERVICES,AJIT KANG,AKANG@PLUSD.ORG,PLUMAS LAKE ELEMENTARY SCHOOL DISTRICT,5307431408,DIRECTOR OF BUSINESS SERVICES,2743 PLUMAS SCHOOL ROAD,PLUMAS LAKE,CA,510636774,TRNJJ7CF5JH5,No,95961,9096898219,CA,RANCHO CUCAMONGA,PROPRIETOR,12223 HIGHLAND AVE. #106-625,91739,USA,ROBERT T DENNIS,RTDENNIS@DENNIS-CPA.COM,RT DENNIS ACCOUNTANCY,,464664250,,10,2024-01-19,2023-02-13,2023-02-14,2023-02-14,2023-02-13,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"84, 10, 93",local,,annual,1734870,UG,true,CENSUS,2023-02-13,ROBERT T DENNIS,PROPRIETOR +26000,2022-06-CENSUS-0000213785,RANDALLSTOWN NON-PROFIT HOUSING CORPORATION,CFO,CINDY LAMB,CINDY.LAMB@CSI.COOP,RANDALLSTOWN NON-PROFIT HOUSING CORPORATION,5867539002,CFO,8425 E. TWELVE MILE ROAD SUITE 100,WARREN,MI,522148647,H6N9FGNML4K5,No,48093,3178485700,IN,CARMEL,MEMBER,501 CONGRESSIONAL BLVD.,46032,USA,JACOB BUEHLER,JBUEHLER@DOZLLC.COM,"DAUBY O'CONNOR & ZALESKI, LLC",,351750664,,14,2024-01-19,2023-01-16,2023-01-17,2023-01-12,2023-01-16,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,6280641,UG,true,CENSUS,2023-01-16,JACOB BUEHLER,MEMBER +27000,2022-06-CENSUS-0000222455,FREEDOM HOUSING CORPORATION,EXECUTIVE DIRECTOR,CHRISTIAN BELDEN,CBELDEN@CCHCNEWPORT.ORG,FREEDOM HOUSING CORPORATION,4018465114,EXECUTIVE DIRECTOR,50 WASHINGTON SQUARE,NEWPORT,RI,134236650,HBTMMW93AE44,No,02840,4014742300,RI,WOONSOCKET,OWNER,531 HARRIS AVENUE,02895,USA,CRAIG D'AMBRA,CRAIGSDAMBRA@GMAIL.COM,D'AMBRA CPA,,050503395,,14,2024-01-19,2022-10-18,2022-10-19,2022-10-19,2022-10-18,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,1502431,UG,true,CENSUS,2022-10-18,CRAIG D'AMBRA,OWNER +28000,2022-06-CENSUS-0000232107,EASTERN AROOSTOOK REGIONAL SCHOOL UNIT #39,BUSINESS MANAGER,MARK BOUCHARD,MBOUCHARD@RSU39.ORG,EASTERN AROOSTOOK REGIONAL SCHOOL UNIT #39,2074966311,BUSINESS MANAGER,75 GLENN STREET,CARIBOU,ME,264505834,MAJDRQ5KQNE5,No,04736,2074983176,ME,CARIBOU,PARTNER,P.O. BOX 906,04736,USA,"GISELE MACDONALD, CPA",GMACDONALD@FELCHCPA.COM,FELCH & COMPANY LLC,,010540055,,84,2024-01-19,2023-01-05,2023-01-05,2022-12-30,2023-01-05,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 84",local,,annual,3377002,UG,true,CENSUS,2023-01-05,"GISELE MACDONALD, CPA",PARTNER +29000,2022-04-CENSUS-0000038992,"NARTHEX, INC.",HOUSING ACCOUNTING SUPPORT SUPERVISOR,ELIZABETH KING,NCRAUDITTEAM@NATIONALRESIDENCES.ORG,"NARTHEX, INC.",6142733577,HOUSING ACCOUNTING SUPPORT SUPERVISOR,2335 NORTH BANK DRIVE,COLUMBUS,OH,311013799,NYH3S7GZFEQ3,No,43220,6145281440,OH,COLUMBUS,PARTNER,"4249 EASTON WAY, SUITE 210",43219,USA,ANDREW GANTZER,ANDY.GANTZER@TIDWELLGROUP.COM,"TIDWELL GROUP, LLC",,271490692,,14,2024-01-19,2023-05-08,2023-04-24,2023-05-09,2023-05-08,2022-04-11,2021-04-11,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,other,2122349,UG,true,CENSUS,2023-05-08,ANDREW GANTZER,PARTNER +30000,2022-06-CENSUS-0000257720,STIEFEL THEATRE FOR THE PERFORMING ARTS FOUNDATION,EXECUTIVE DIRECTOR,JANE GATES,JGATES@STIEFELTHEATRE.ORG,STIEFEL THEATRE FOR THE PERFORMING ARTS FOUNDATION,7858271998,EXECUTIVE DIRECTOR,151 S. SANTA FE AVE,SALINA,KS,311537194,FTWNJFQLF463,No,67401,7858251561,KS,SALINA,CPA,3030 CORTLAND CIRCLE,67401,USA,CORLENE R. LANGE,LANGE@KCOE.COM,"KCOE ISOM, LLP",,480567703,,59,2024-01-19,2023-03-30,2023-03-31,2023-03-31,2023-03-30,2022-06-30,2021-06-30,2022,program-specific,unmodified_opinion,,,,No,No,No,No,Yes,750000,No,59,non-profit,,annual,1085901,UG,true,CENSUS,2023-03-30,CORLENE R. LANGE,CPA +31000,2022-06-CENSUS-0000248016,ACLED ANALYSIS INCORPORATED,SENIOR FINANCE MANAGER,AMANDA BERTRAN LLOVERA,A.BERTRAN.LLOVERA@ACLEDDATA.COM,ACLED ANALYSIS INCORPORATED,2623020379,SENIOR FINANCE MANAGER,361 FALLS RD #501,GRAFTON,WI,470972683,QDN9FURLZ8J3,No,53024,5104679506,CA,OAKLAND,PRESIDENT,1000 BROADWAY 200G,94607,USA,IRYNA ORESHKOVA,IO@IRYNACPA.COM,IRYNA ACCOUNTANCY CORPORATION,,204994635,,19,2024-01-19,2023-01-25,2023-01-26,2023-01-26,2023-01-25,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"98, 19",non-profit,,annual,777580,UG,true,CENSUS,2023-01-25,IRYNA ORESHKOVA,PRESIDENT +32000,2022-12-CENSUS-0000192624,STEEL VALLEY REGIONAL TRANSIT AUTHORITY,FINANCE MANAGER,TIM TURNER,TTURNER@SVRTA.COM,STEEL VALLEY REGIONAL TRANSIT AUTHORITY,7402826145,FINANCE MANAGER,555 ADAMS STREET,STEUBENVILLE,OH,341815642,L8SLLRZ77EP8,No,43952,7403730056,OH,MARIETTA,AUDIT MANAGER,313 SECOND STREET,45750,USA,MARIAH RICHARDS,MRICHARDS@PERRYCPAS.NET,PERRY AND ASSOCIATES CPAS AC,,550771624,,20,2024-01-19,2023-09-26,2023-09-27,2023-09-27,2023-09-26,2022-12-31,2021-12-31,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,20,local,,annual,824231,UG,true,CENSUS,2023-09-26,MARIAH RICHARDS,AUDIT MANAGER +33000,2022-12-CENSUS-0000250381,"BURROUGHS MANOR, INC.","PRESIDENT, GORMAN MANAGEMENT COMPANY",THOMAS A. GORMAN,TAGORMAN@AOL.COM,"BURROUGHS MANOR, INC.",9183332244,"PRESIDENT, GORMAN MANAGEMENT COMPANY",398060 WEST 2200 ROAD,BARTLESVILLE,OK,731124865,EULPKNCVLXM3,No,74006,9183360008,OK,BARTLESVILLE,ENGAGEMENT PARTNER,302 SE OSAGE AVENUE,74003,USA,JIM G. NICHOLS,JIM.NICHOLS@SAMBCPAS.COM,"STOTTS, ARCHAMBO, MUEGGENBORG & BARCLAY, PC",,471209122,,14,2024-01-19,2023-03-18,2023-03-17,2023-03-17,2023-03-18,2022-12-31,2021-12-31,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,1499783,UG,true,CENSUS,2023-03-18,JIM G. NICHOLS,ENGAGEMENT PARTNER +34000,2022-09-CENSUS-0000250713,THE SALVATION ARMY LAS VEGAS SOCIAL SERVICES,CONTROLLER,JEFFREY WHITE,JEFF.WHITE@USW.SALVATIONARMY.ORG,THE SALVATION ARMY LAS VEGAS SOCIAL SERVICES,6022674198,CONTROLLER,2900 PALOMINO LANE,LAS VEGAS,NV,941156347,WKZ9BWP1U5L3,No,89107,7149748143,CA,ORANGE,CPA,"2808 E. KATELLA AVENUE, SUITE 200",92867,USA,MARK FREEMAN,MJFREEMAN27@GMAIL.COM,FREEMAN & FREEMAN,,330112006,,64,2024-01-20,2023-06-22,2023-06-23,2023-06-23,2023-06-22,2022-09-30,2021-09-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"97, 14, 64, 16, 21",non-profit,,annual,8508867,UG,true,CENSUS,2023-06-22,MARK FREEMAN,CPA +35000,2022-06-CENSUS-0000232734,HUDSON COMMUNITY SCHOOL DISTRICT,SCHOOL BUSINESS OFFICIAL,CHAD WAGNER,CHADWAGNER@HUDSCHOOLS.ORG,HUDSON COMMUNITY SCHOOL DISTRICT,3199883233,SCHOOL BUSINESS OFFICIAL,136 S WASHINGTON ST,HUDSON,IA,426025403,F7EHCEHN3H64,No,50638,6417921910,IA,NEWTON,PRESIDENT,115 N 3RD AVE W,50208,USA,NANCY JANSSEN,INFO@NCJCPAIA.COM,"NOLTE, CORNMAN & JOHNSON",,421255013,,10,2024-01-20,2023-04-25,2023-04-25,2023-04-26,2023-04-25,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,Yes,Yes,No,No,750000,No,"84, 10",local,,annual,843708,UG,true,CENSUS,2023-04-25,NANCY JANSSEN,PRESIDENT +36000,2022-12-CENSUS-0000114049,WASHINGTON GORGE ACTION PROGRAMS,EXECUTIVE DIRECTOR,LESLIE NARAMORE,LESLIE@WAGAP.ORG,WASHINGTON GORGE ACTION PROGRAMS,5094932662,EXECUTIVE DIRECTOR,PO BOX 805,BINGEN,WA,910793062,G142DZKFX3B8,No,98605,5095751040,WA,YAKIMA,MEMBER OF THE FIRM,3702 KERN ROAD,98902,USA,ANGELA PRATT,ANGELA@PETERSENADVISORS.COM,"PETERSEN CPAS AND ADVISORS, PLLC",,261262413,,21,2024-01-19,2023-09-28,2023-09-29,2023-09-29,2023-09-28,2022-12-31,2021-12-31,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 16, 14, 93, 21",non-profit,,annual,4820177,UG,true,CENSUS,2023-09-28,ANGELA PRATT,MEMBER OF THE FIRM +37000,2022-06-CENSUS-0000089494,"REFUGE HOUSE, INC.",EXECUTIVE DIRECTOR,EMILY MITCHEM,EMITCHEM@REFUGEHOUSE.COM,"REFUGE HOUSE, INC.",8509226062,EXECUTIVE DIRECTOR,P.O. BOX 20910,TALLAHASSEE,FL,591869324,NKEWXG2SEUN8,No,32316,8503857444,FL,TALLAHASSEE,AUDIT PARTNER,3375 G CAPITAL CIRCLE NE,32308,USA,JOHN KIRK,JKIRK@TBL-CPA.COM,THOMSON BROCK LUGER & COMPANY,,202259573,,16,2024-01-20,2023-03-30,2023-03-31,2023-03-31,2023-03-30,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,No,"97, 16, 14, 93",non-profit,,annual,1330756,UG,true,CENSUS,2023-03-30,JOHN KIRK,AUDIT PARTNER +38000,2022-06-CENSUS-0000211066,CLEVELAND HEIGHTS-UNIVERSITY HEIGHTS CITY SCHOOL DISTRICT,TREASURER/CFO,SCOTT GAINER,S_GAINER@CHUH.ORG,CLEVELAND HEIGHTS-UNIVERSITY HEIGHTS CITY SCHOOL DISTRICT,2163717171,TREASURER/CFO,2155 MIRAMAR BOULEVARD,UNIVERSITY HEIGHTS,OH,346000687,NHLKJ8WKAZ36,No,44118,2167873665,OH,COLUMBUS,SENIOR AUDIT MANAGER,88 EAST BROAD STREET,43215,USA,MATTHEW GOLDMAN,MAGOLDMAN@OHIOAUDITOR.GOV,"KEITH FABER, AUDITOR OF STATE",,311334820,,84,2024-01-20,2023-01-16,2023-01-17,2023-01-17,2023-01-16,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 10, 32, 21",local,,annual,21449156,UG,true,CENSUS,2023-01-16,MATTHEW GOLDMAN,SENIOR AUDIT MANAGER +39000,2022-06-CENSUS-0000183906,VERMILLION HOUSING AND REDEVELOPMENT COMMISSION,EXECUTIVE DIRECTOR,DAVE THIESSE,VERMILLIONHOUSING@YAHOO.COM,VERMILLION HOUSING AND REDEVELOPMENT COMMISSION,6056777191,EXECUTIVE DIRECTOR,25 CENTER ST,VERMILLION,SD,460413120,MV5VTL3QLJY7,No,57069,6059967717,SD,MITCHELL,PARTNER,PO BOX 249,57301,USA,TERRI POST,TERRI.POST@ELOCPA.COM,ELO CPAS & ADVISORS,,460434947,,14,2024-01-20,2023-02-26,2023-02-27,2023-02-27,2023-02-26,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,No,14,local,,annual,951537,UG,true,CENSUS,2023-02-26,TERRI POST,PARTNER +40000,2022-12-CENSUS-0000231482,"FIRST RICHLAND MORROW HOUSING, INC. X",CONTROLLER,SHELLEY DARFUS,SDARFUS@BARCUSCOMPANY.COM,"FIRST RICHLAND MORROW HOUSING, INC. X",6144519000,CONTROLLER,1601 BETHEL ROAD,COLUMBUS,OH,341930439,LNJSEWR9BYF7,No,43220,6142240955,OH,COLUMBUS,CPA,"370 S FIFTH STREET, SUITE 1",43215,USA,TRACIE BAUGUS,TBAUGUS@SMS-CPAS.COM,"STEMEN, MERTENS, STICKLER CPAS & ASSOCIATES",,205511881,,14,2024-01-20,2023-04-18,2023-04-19,2023-04-19,2023-04-18,2022-12-31,2021-12-31,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,1435297,UG,true,CENSUS,2023-04-18,TRACIE BAUGUS,CPA +41000,2022-06-CENSUS-0000160804,WAVERLY CENTRAL SCHOOL DISTRICT,BUSINESS ADMINISTRATOR,KATHY ROTE,KROTE@GSTBOCES.ORG,WAVERLY CENTRAL SCHOOL DISTRICT,6075652841,BUSINESS ADMINISTRATOR,15 FREDRICK STREET,WAVERLY,NY,156002429,PD2KK5PY4JL3,No,14892,6072724444,NY,ITHACA,PARTNER,20 THORNWOOD DRIVE SUITE 200,14850,USA,LESLIE SPURGIN,DCF@INSEROCPA.COM,"INSERO & CO. CPA'S, LLP",,475324570,,84,2024-01-20,2022-11-07,2022-11-08,2022-11-08,2022-11-07,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,Yes,No,750000,Yes,"10, 84",local,,annual,3948327,UG,true,CENSUS,2022-11-07,LESLIE SPURGIN,PARTNER +42000,2022-06-CENSUS-0000250238,"DRUG RECOVERY, INC. (DBA CATALYST BEHAVIORAL SERVICES, INC)",EXECUTIVE DIRECTOR,EDIE NAYFA,ENAYFA@CATALYSTOK.ORG,"DRUG RECOVERY, INC. (DBA CATALYST BEHAVIORAL SERVICES, INC)",4052329804,EXECUTIVE DIRECTOR,"3033 N. WALNUT, WEST BUILDING",OKLAHOMA CITY,OK,730968383,CDCJZ69PCEL8,No,73105,4055942041,OK,OKLAHOMA CITY,VDUTTON@EIDEBAILLY.COM,"621 N. ROBINSON, STE 200",731026232,USA,VANESSA DUTTON,VDUTTON@EIDEBAILLY.COM,EIDE BAILLY LLP,,450250958,,93,2024-01-20,2023-03-30,2023-03-31,2023-03-31,2023-03-30,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,No,"16, 93",non-profit,,annual,2844302,UG,true,CENSUS,2023-03-30,VANESSA DUTTON,VDUTTON@EIDEBAILLY.COM +43000,2022-06-CENSUS-0000160312,DUTCHESS COUNTY BOCES,SCHOOL BUSINESS ADMINISTRATOR,MATTHEW METZGER,MATTHEW.METZGER@DCBOCES.ORG,DUTCHESS COUNTY BOCES,8454864800,SCHOOL BUSINESS ADMINISTRATOR,5 BOCES ROAD,POUGHKEEPSIE,NY,146012196,GNQGKNTDVDS6,No,12601,5188284616,NY,HUDSON,DIRECTOR,"4071 ROUTE 9, STOP 1",12534,USA,"VICTOR V. CHURCHILL, CPA",VCHURCHILL@RBTCPAS.COM,"RBT CPA'S, LLP",,141604297,,84,2024-01-20,2023-03-30,2023-03-31,2023-03-31,2023-03-30,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"10, 84",local,,annual,2951989,UG,true,CENSUS,2023-03-30,"VICTOR V. CHURCHILL, CPA",DIRECTOR +44000,2022-06-CENSUS-0000235281,CATALYST SCHOOLS,DIRECTOR OF FINANCE AND COMPLIANCE,MICHELE NUNEZ,MNUNEZ@CATALYSTSCHOOLS.ORG,CATALYST SCHOOLS,7738975825,DIRECTOR OF FINANCE AND COMPLIANCE,6727 S CALIFORNIA AVE,CHICAGO,IL,204069346,XB1KKBRLHQJ9,No,60629,3126344414,IL,CHICAGO,SENIOR DIRECTOR,30 SOUTH WACKER DR,60606,USA,MANDY PITTMAN,MANDY.PITTMAN@RSMUS.COM,RSM US LLP,,420714325,,84,2024-01-20,2022-11-03,2022-11-04,2022-11-04,2022-11-03,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 10",non-profit,,annual,4121154,UG,true,CENSUS,2022-11-03,MANDY PITTMAN,SENIOR DIRECTOR +45000,2022-06-CENSUS-0000164186,CLEVELAND MUNICIPAL SCHOOL DISTRICT,INTERIM CHIEF FINANCIAL OFFICER,NATHAN MORTIMER,NATHAN.MORTIMER@CLEVELANDMETROSCHOOLS.ORG,CLEVELAND MUNICIPAL SCHOOL DISTRICT,2168380391,INTERIM CHIEF FINANCIAL OFFICER,"1111 SUPERIOR AVENUE, SUITE 1800",CLEVELAND,OH,346000662,ESYAYXNJYTD9,No,44114,2167873665,OH,COLUMBUS,SENIOR AUDIT MANAGER,88 EAST BROAD STREET,43216,USA,DOUGLAS CHAMOT,DWCHAMOT@OHIOAUDITOR.GOV,"KEITH FABER, AUDITOR OF STATE",,311334820,84,,2024-01-20,2023-01-24,2023-01-24,2023-01-25,2023-01-24,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,Yes,No,No,3000000,No,"84, 12, 10, 93",local,,annual,200303287,UG,true,CENSUS,2023-01-24,DOUGLAS CHAMOT,SENIOR AUDIT MANAGER +46000,2022-06-CENSUS-0000229656,"MOHN STREET ACCESSIBLE HOUSING, INC.",PRESIDENT,LEAH EPPINGER,LEPPINGER@DAUPHINHOUSING.ORG,"MOHN STREET ACCESSIBLE HOUSING, INC.",7179399301,PRESIDENT,"501 MOHN STREET ACCESSIBLE HOUSING, INC.",STEELTON,PA,870762383,XP1NMSKDMM81,No,17113,7175619200,PA,HARRISBURG,PARTNER,"830 SIR THOMAS COURT, SUITE 100",17109,USA,KIMBERLY STANK,KSTANK@ZALLC.ORG,ZELENKOFSKE AXELROD LLC,,233022325,,14,2024-01-20,2023-02-21,2023-02-13,2023-02-17,2023-02-21,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,1598234,UG,true,CENSUS,2023-02-21,KIMBERLY STANK,PARTNER +47000,2022-06-CENSUS-0000206378,"TURNING POINT, INC.",EXECUTIVE DIRECTOR,SARAH PONITZ,SPONITZ@TURNPT.ORG,"TURNING POINT, INC.",8153388081,EXECUTIVE DIRECTOR,PO BOX 273,WOODSTOCK,IL,363163296,JL4FNMZDKSK8,No,60098,8153441300,IL,MCHENRY,PARTNER,"5400 W ELM ST, SUITE 203",60050,USA,CHERYDEN JUERGENSEN,CPAS@EDERCASELLA.COM,"EDER, CASELLA & CO",,363614997,,93,2024-01-20,2022-11-09,2022-11-10,2022-11-09,2022-11-09,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"14, 16, 93, 97",non-profit,,annual,946382,UG,true,CENSUS,2022-11-09,CHERYDEN JUERGENSEN,PARTNER +48000,2022-12-CENSUS-0000247951,EXTENSION FOUNDATION,CONTROLLER,ARIELLE SMITH,ARIELLESMITH@EXTENSION.ORG,EXTENSION FOUNDATION,8327365157,CONTROLLER,ONE KANSAS CITY PL 1200 MAIN S,KANSAS CITY,MO,204781422,DNZBHAA5N4G3,No,641052122,4024799321,NE,LINCOLN,CPA/PARTNER,1248 O STREET SUITE 500,68508,USA,KERRY GUSTAFSSON,GUSTAFSSON@DANACOLE.COM,DANA F. COLE & COMPANY LLP,,470526649,,10,2024-01-20,2023-09-28,2023-09-29,2023-09-29,2023-09-28,2022-12-31,2021-12-31,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 66",non-profit,,annual,5096909,UG,true,CENSUS,2023-09-28,KERRY GUSTAFSSON,CPA/PARTNER +49000,2022-06-CENSUS-0000150455,SHELBY PUBLIC SCHOOLS,BUSINESS MANAGER,AMY BUDDE,ABUDDE@SHELBY.K12.MI.US,SHELBY PUBLIC SCHOOLS,2318615211,BUSINESS MANAGER,525 N. STATE STREET,SHELBY,MI,386003167,T3DJNDK73JY8,No,49455,6169493200,MI,GRAND RAPIDS,SENIOR MANAGER,2910 LUCERNE DRIVE SE,49546,USA,MARC SAWYERS,MSAWYERS@HUNGERFORDNICHOLS.COM,HUNGERFORD NICHOLS,,382184825,,10,2024-01-20,2023-01-13,2023-01-14,2022-11-30,2023-01-13,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 84, 93",local,,annual,4702717,UG,true,CENSUS,2023-01-13,MARC SAWYERS,SENIOR MANAGER +50000,2022-09-CENSUS-0000102763,"EDINBURG CHILD CARE, INC",EXECUTIVE DIRECTOR,GYLMA GARZA,GMG.ECCI@GMAIL.COM,"EDINBURG CHILD CARE, INC",9563836789,EXECUTIVE DIRECTOR,1338 W UNIVERSITY,EDINBURG,TX,742358732,M3PGGA5FRAF6,No,785392930,9567879909,TX,PHARR,AUDITOR,208 W FERGUSON UNIT 1,78577,USA,JANET ROBLES,JANETROBLES@ORGCPA.COM,OSCAR R GONZALEZ CPA AND ASSOCIATES,,273582518,,10,2024-01-20,2023-06-28,2023-06-29,2023-06-29,2023-06-28,2022-09-30,2021-09-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,10,non-profit,,annual,4343763,UG,true,CENSUS,2023-06-28,JANET ROBLES,AUDITOR +51000,2022-06-CENSUS-0000166110,CITY OF ROSEBURG,FINANCE DIRECTOR,RON HARKER,RHARKER@CITYOFROSEBURG.ORG,CITY OF ROSEBURG,5414926710,FINANCE DIRECTOR,900 SE DOUGLAS AVE,ROSEBURG,OR,936002247,ZFPTXNQ1CVH8,No,97470,5416724886,OR,ROSEBURG,CERTIFIED PUBLIC ACCOUNTANT,2500 W HARVARD AVE,97471,USA,JEFFREY R. COOLEY,JEFF@NEUNERDAVIDSON.COM,NEUNER DAVIDSON & CO,,930764156,,14,2024-01-20,2023-01-16,2023-01-17,2023-01-17,2023-01-16,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"14, 21, 16, 45, 20",state,,annual,1236483,UG,true,CENSUS,2023-01-16,JEFFREY R. COOLEY,CERTIFIED PUBLIC ACCOUNTANT +52000,2022-06-CENSUS-0000205148,SCHOOL DISTRICT OF OSCEOLA,ADMINISTRATOR,MARK LUEBKER,LUEBKERM@OSCEOLAK12.ORG,SCHOOL DISTRICT OF OSCEOLA,7152944140,ADMINISTRATOR,PO BOX 128,OSCEOLA,WI,396003796,HB1AL86ZYTT3,No,54020,7158521100,WI,EAU CLAIRE,PRINCIPAL,"3402 OAKWOOD MALL DRIVE, SUITE 100",54701,USA,APRIL ANDERSON,APRIL.ANDERSON@CLACONNECT.COM,CLIFTONLARSONALLEN LLP,,410746749,,10,2024-01-20,2023-03-29,2023-03-30,2023-03-30,2023-03-29,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,No,"84, 93, 32, 10",unknown,,annual,2877490,UG,true,CENSUS,2023-03-29,APRIL ANDERSON,PRINCIPAL +53000,2022-03-CENSUS-0000230532,"PAGEDALE RHF HOUSING, INC. DBA MOUNT BEULAH TERRACE 085-EE-090",CONTROLLER,CHRISTOPHER PURCELL,CHRISTOPHER.PURCELL@RHF.ORG,"PAGEDALE RHF HOUSING, INC. DBA MOUNT BEULAH TERRACE 085-EE-090",5622575100,CONTROLLER,911 NORTH STUDEBAKER ROAD,LONG BEACH,CA,205267298,E67LXYHYDCG3,No,90815,3178485700,IN,CARMEL,PRINCIPAL,501 CONGRESSIONAL BLVD.,46032,USA,MATT CATLIN,MCATLIN@DOZLLC.COM,"DAUBY O'CONNOR & ZALESKI, LLC",,351750664,,14,2024-01-20,2022-12-19,2022-12-20,2022-12-20,2022-12-19,2022-03-31,2021-03-31,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 14",non-profit,,annual,5248382,UG,true,CENSUS,2022-12-19,MATT CATLIN,PRINCIPAL +54000,2022-12-CENSUS-0000159275,TOWN OF TONAWANDA,COMPTROLLER,DEBORAH SCHWARZ,DSCHWARZ@TONAWANDA.NY.US,TOWN OF TONAWANDA,7168778810,COMPTROLLER,2919 DELAWARE AVE,KENMORE,NY,166002385,W9YVK5RR6SN7,No,14217,7165652299,NY,ELMA,PARTNER,2721 TRANSIT RD SUITE 111,14059,USA,MATTHEW MONTALBO,MMONTALBO@DRESCHERMALECKI.COM,DRESCHER & MALECKI LLP,,743118183,,14,2024-01-20,2023-07-24,2023-07-25,2023-07-24,2023-07-24,2022-12-31,2021-12-31,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"21, 14, 20, 16",local,,annual,6646790,UG,true,CENSUS,2023-07-24,MATTHEW MONTALBO,PARTNER +55000,2022-06-CENSUS-0000181993,"CHILD AND FAMILY RESOURCES, INC. AND SUBSIDIARY",CHIEF FINACIAL OFFICER,CONNIE CURNETT,CCURNETT@CFRAZ.ORG,"CHILD AND FAMILY RESOURCES, INC. AND SUBSIDIARY",5208818940,CHIEF FINACIAL OFFICER,2800 E. BROADWAY,TUCSON,AZ,860251984,YBNKJ1BNH4Z9,No,85716,5206248229,AZ,TUCSON,PARTNER,"4801 E. BROADWAY BLVD, STE 501",857113648,USA,SUSAN VOS,SVOS@RCM.CPA,"REGIER, CARR, & MONROE, LLP CPAS",,480573184,,93,2024-01-20,2022-11-22,2022-11-14,2022-11-23,2022-11-22,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 10",non-profit,,annual,4649326,UG,true,CENSUS,2022-11-22,SUSAN VOS,PARTNER +187000,2018-06-CENSUS-0000171150,BON HOMME SCHOOL DISTRICT NO. 4-2,BUSINESS MANAGER,GARY KORTAN,GARY.KORTAN@K12.SD.US,BON HOMME SCHOOL DISTRICT NO. 4-2,6055893388,BUSINESS MANAGER,PO BOX 28,TYNDALL,SD,460311996,GSA_MIGRATION,No,570660028,6059287241,SD,PARKSTON,CPA,"PO BOX 247, 105 E MAIN ST",573660247,USA,RANDY SCHOENFISH,WJSCPA@SANTEL.NET,"SCHOENFISH & CO, INC",,460353724,,84,2024-01-26,2019-03-26,2019-03-26,2019-03-27,2019-03-26,2018-06-30,2017-06-30,2018,single-audit,qualified_opinion,,,,No,Yes,No,Yes,No,750000,No,"10, 84",local,,annual,807077,UG,true,CENSUS,2019-03-26,RANDY SCHOENFISH,CPA diff --git a/backend/dissemination/sql/api/api_v1_1_0/sling.md b/backend/dissemination/sql/api/api_v1_1_0/sling.md new file mode 100644 index 0000000000..9974a5d676 --- /dev/null +++ b/backend/dissemination/sql/api/api_v1_1_0/sling.md @@ -0,0 +1,41 @@ +# sling + +Point to the DB + +``` +export PG='postgresql://postgres@localhost:5432/postgres?sslmode=disable' +``` + +This gives me the general table as a CSV to STDOUT: + +``` +sling run --src-conn $PG --src-stream 'public.dissemination_general' --stdout +``` + +With a configuration file (YAML): + +`config1.yaml`: + +``` +source: postgresql://postgres@localhost:5432/postgres?sslmode=disable +target: LOCAL + +defaults: + target_options: + format: csv + +streams: + public.dissemination_general: + object: file:///tmp/dg.csv + + public.dissemination_federalaward: + object: file:///tmp/dfa/*.csv + target_options: + file_max_rows: 100000 +``` + +and then + +``` +sling run -r config1.yaml +``` diff --git a/backend/dissemination/sql/api/api_v1_1_0/standup.sql b/backend/dissemination/sql/api/api_v1_1_0/standup.sql new file mode 100644 index 0000000000..62ba940193 --- /dev/null +++ b/backend/dissemination/sql/api/api_v1_1_0/standup.sql @@ -0,0 +1,523 @@ +DO +$do$ +BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = 'authenticator') THEN + RAISE NOTICE 'Role "authenticator" already exists. Skipping.'; + ELSE + CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; + END IF; +END +$do$; + +DO +$do$ +BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = 'api_fac_gov') THEN + RAISE NOTICE 'Role "api_fac_gov" already exists. Skipping.'; + ELSE + CREATE ROLE api_fac_gov NOLOGIN; + END IF; +END +$do$; + +GRANT api_fac_gov TO authenticator; + +NOTIFY pgrst, 'reload schema'; +begin; + +do +$$ +begin + DROP SCHEMA IF EXISTS api_v1_1_0 CASCADE; + DROP SCHEMA IF EXISTS api_v1_1_0_functions CASCADE; + + if not exists (select schema_name from information_schema.schemata where schema_name = 'api_v1_1_0') then + create schema api_v1_1_0; + create schema api_v1_1_0_functions; + + grant usage on schema api_v1_1_0_functions to api_fac_gov; + grant usage on schema api_v1_1_0 to api_fac_gov; + grant select, usage on all sequences in schema api_v1_1_0 to api_fac_gov; + + -- Grant access to tables and views + -- NOTE 20240920: This grants SELECT to all tables + -- created in this schema *in the future*. Nothing + -- that already exists is affected. + -- https://www.postgresql.org/docs/current/sql-alterdefaultprivileges.html + alter default privileges + in schema api_v1_1_0 + grant select + -- this includes views + on tables + to api_fac_gov; + + alter default privileges + in schema api_v1_1_0_functions + grant select + -- this includes views + on tables + to api_fac_gov; + + -- Grant access to future sequences, if we have them + alter default privileges + in schema api_v1_1_0 + grant select, usage + on sequences + to api_fac_gov; + + end if; +end +$$ +; + +-- https://postgrest.org/en/stable/references/api/openapi.html +-- This is the title (version number) and description (text). +COMMENT ON SCHEMA api_v1_1_0 IS +$$v1.1.0 + +A RESTful API that serves data from the SF-SAC.$$; + + +commit; + +notify pgrst, + 'reload schema'; + +-- WARNING +-- Under PostgreSQL 12, the functions below work. +-- Under PostgreSQL 14, these will break. +-- +-- Note the differences: +-- +-- raise info 'Works under PostgreSQL 12'; +-- raise info 'request.header.x-magic %', (SELECT current_setting('request.header.x-magic', true)); +-- raise info 'request.jwt.claim.expires %', (SELECT current_setting('request.jwt.claim.expires', true)); +-- raise info 'Works under PostgreSQL 14'; +-- raise info 'request.headers::json->>x-magic %', (SELECT current_setting('request.headers', true)::json->>'x-magic'); +-- raise info 'request.jwt.claims::json->expires %', (SELECT current_setting('request.jwt.claims', true)::json->>'expires'); +-- +-- To quote the work of Dav Pilkey, "remember this now." + + +CREATE OR REPLACE FUNCTION api_v1_1_0_functions.get_header(item text) RETURNS text + AS $get_header$ + declare res text; + begin + SELECT (current_setting('request.headers', true)::json)->>item into res; + return res; + end; +$get_header$ LANGUAGE plpgsql; + +create or replace function api_v1_1_0_functions.get_api_key_uuid() returns TEXT +as $gaku$ +declare uuid text; +begin + select api_v1_1_0_functions.get_header('x-api-user-id') into uuid; + return uuid; +end; +$gaku$ LANGUAGE plpgsql; + +create or replace function api_v1_1_0_functions.has_tribal_data_access() +returns boolean +as $has_tribal_data_access$ +DECLARE + uuid_header UUID; + key_exists boolean; +BEGIN + + SELECT api_v1_1_0_functions.get_api_key_uuid() INTO uuid_header; + SELECT + CASE WHEN EXISTS ( + SELECT key_id + FROM public.dissemination_TribalApiAccessKeyIds taaki + WHERE taaki.key_id = uuid_header::TEXT) + THEN 1::BOOLEAN + ELSE 0::BOOLEAN + END + INTO key_exists; + RAISE INFO 'api_v1_1_0 has_tribal % %', uuid_header, key_exists; + RETURN key_exists; +END; +$has_tribal_data_access$ LANGUAGE plpgsql; + +-- If you change the constant defined by this function, +-- you must regenerate the index. +CREATE OR REPLACE FUNCTION api_v1_1_0_functions.batch_size() + RETURNS int + LANGUAGE sql IMMUTABLE PARALLEL SAFE AS +'SELECT 20000'; +GRANT EXECUTE ON FUNCTION api_v1_1_0_functions.batch_size() TO api_fac_gov; + +CREATE OR REPLACE FUNCTION api_v1_1_0_functions.batch (id bigint) +returns bigint +as $batch$ +declare num bigint; +begin + select div(id, api_v1_1_0_functions.batch_size()) into num; + return num; +end; +$batch$ +language plpgsql immutable; +GRANT EXECUTE ON FUNCTION api_v1_1_0_functions.batch(bigint) TO api_fac_gov; + +CREATE OR REPLACE FUNCTION api_v1_1_0.batches (in_table text) +returns integer +as $batches$ +declare count integer; +declare batch_size bigint; +begin + select api_v1_1_0_functions.batch_size() into batch_size; + case + when in_table = 'general' then + select div(count(*), batch_size) into count + from public.dissemination_general; + when in_table = 'federal_awards' then + select div(count(*), batch_size) into count + from public.dissemination_federalaward; + else + count := 0; + end case; + RETURN count; +end; +$batches$ +language plpgsql; +GRANT EXECUTE ON FUNCTION api_v1_1_0.batches(text) TO api_fac_gov; + +CREATE OR REPLACE FUNCTION api_v1_1_0.get_general_batch (batch_no bigint) +returns setof dissemination_general +as $batches$ + select * from public.dissemination_general where api_v1_1_0_functions.batch(id) = batch_no; +$batches$ +language sql; +GRANT EXECUTE ON FUNCTION api_v1_1_0.get_general_batch(bigint) TO api_fac_gov; + +CREATE OR REPLACE FUNCTION api_v1_1_0.get_federal_award_batch (batch_no bigint) +returns setof dissemination_federalaward +as $batches$ + select * from public.dissemination_federalaward where api_v1_1_0_functions.batch(public.dissemination_federalaward.id) = batch_no; +$batches$ +language sql; +GRANT EXECUTE ON FUNCTION api_v1_1_0.get_federal_award_batch(bigint) TO api_fac_gov; + + +-- We should consider dropping and regenerating this index +-- every night after MV generation. +-- drop index batch_by_id_dfa; +create index IF NOT EXISTS batch_by_id_dfa + on public.dissemination_federalaward + using btree(api_v1_1_0_functions.batch(public.dissemination_federalaward.id)); + + +NOTIFY pgrst, 'reload schema'; +begin; + +--------------------------------------- +-- finding_text +--------------------------------------- +create view api_v1_1_0.findings_text as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + ft.finding_ref_number, + ft.contains_chart_or_table, + ft.finding_text + from + dissemination_findingtext ft, + dissemination_general gen + where + ft.report_id = gen.report_id + and + (gen.is_public = true + or (gen.is_public = false and api_v1_1_0_functions.has_tribal_data_access())) + order by ft.id +; + +--------------------------------------- +-- additional_ueis +--------------------------------------- +create view api_v1_1_0.additional_ueis as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + uei.additional_uei + from + dissemination_general gen, + dissemination_additionaluei uei + where + gen.report_id = uei.report_id + order by uei.id +; + +--------------------------------------- +-- finding +--------------------------------------- +create view api_v1_1_0.findings as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + finding.award_reference, + finding.reference_number, + finding.is_material_weakness, + finding.is_modified_opinion, + finding.is_other_findings, + finding.is_other_matters, + finding.prior_finding_ref_numbers, + finding.is_questioned_costs, + finding.is_repeat_finding, + finding.is_significant_deficiency, + finding.type_requirement + from + dissemination_finding finding, + dissemination_general gen + where + finding.report_id = gen.report_id + order by finding.id +; + +--------------------------------------- +-- federal award +--------------------------------------- +create view api_v1_1_0.federal_awards as + select + award.report_id, + gen.auditee_uei, + gen.audit_year, + --- + award.award_reference, + award.federal_agency_prefix, + award.federal_award_extension, + award.additional_award_identification, + award.federal_program_name, + award.amount_expended, + award.cluster_name, + award.other_cluster_name, + award.state_cluster_name, + award.cluster_total, + award.federal_program_total, + award.is_major, + award.is_loan, + award.loan_balance, + award.is_direct, + award.audit_report_type, + award.findings_count, + award.is_passthrough_award, + award.passthrough_amount + from + dissemination_federalaward award, + dissemination_general gen + where + award.report_id = gen.report_id + order by award.id +; + + +--------------------------------------- +-- corrective_action_plan +--------------------------------------- +create view api_v1_1_0.corrective_action_plans as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + ct.finding_ref_number, + ct.contains_chart_or_table, + ct.planned_action + from + dissemination_CAPText ct, + dissemination_General gen + where + ct.report_id = gen.report_id + and + (gen.is_public = true + or (gen.is_public = false and api_v1_1_0_functions.has_tribal_data_access())) + order by ct.id +; + +--------------------------------------- +-- notes_to_sefa +--------------------------------------- +create view api_v1_1_0.notes_to_sefa as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + note.note_title as title, + note.accounting_policies, + note.is_minimis_rate_used, + note.rate_explained, + note.content, + note.contains_chart_or_table + from + dissemination_general gen, + dissemination_note note + where + note.report_id = gen.report_id + and + (gen.is_public = true + or (gen.is_public = false and api_v1_1_0_functions.has_tribal_data_access())) + order by note.id +; + +--------------------------------------- +-- passthrough +--------------------------------------- +create view api_v1_1_0.passthrough as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + pass.award_reference, + pass.passthrough_id, + pass.passthrough_name + from + dissemination_general as gen, + dissemination_passthrough as pass + where + gen.report_id = pass.report_id + order by pass.id +; + + +--------------------------------------- +-- general +--------------------------------------- +create view api_v1_1_0.general as + select + -- every table starts with report_id, UEI, and year + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + gen.auditee_certify_name, + gen.auditee_certify_title, + gen.auditee_contact_name, + gen.auditee_email, + gen.auditee_name, + gen.auditee_phone, + gen.auditee_contact_title, + gen.auditee_address_line_1, + gen.auditee_city, + gen.auditee_state, + gen.auditee_ein, + gen.auditee_zip, + -- auditor + gen.auditor_certify_name, + gen.auditor_certify_title, + gen.auditor_phone, + gen.auditor_state, + gen.auditor_city, + gen.auditor_contact_title, + gen.auditor_address_line_1, + gen.auditor_zip, + gen.auditor_country, + gen.auditor_contact_name, + gen.auditor_email, + gen.auditor_firm_name, + gen.auditor_foreign_address, + gen.auditor_ein, + -- agency + gen.cognizant_agency, + gen.oversight_agency, + -- dates + gen.date_created, + gen.ready_for_certification_date, + gen.auditor_certified_date, + gen.auditee_certified_date, + gen.submitted_date, + gen.fac_accepted_date, + gen.fy_end_date, + gen.fy_start_date, + gen.audit_type, + gen.gaap_results, + gen.sp_framework_basis, + gen.is_sp_framework_required, + gen.sp_framework_opinions, + gen.is_going_concern_included, + gen.is_internal_control_deficiency_disclosed, + gen.is_internal_control_material_weakness_disclosed, + gen.is_material_noncompliance_disclosed, + gen.dollar_threshold, + gen.is_low_risk_auditee, + gen.agencies_with_prior_findings, + gen.entity_type, + gen.number_months, + gen.audit_period_covered, + gen.total_amount_expended, + gen.type_audit_code, + gen.is_public, + gen.data_source, + gen.is_aicpa_audit_guide_included, + gen.is_additional_ueis, + CASE EXISTS(SELECT ein.report_id FROM dissemination_additionalein ein WHERE ein.report_id = gen.report_id) + WHEN FALSE THEN 'No' + ELSE 'Yes' + END AS is_multiple_eins, + CASE EXISTS(SELECT aud.report_id FROM dissemination_secondaryauditor aud WHERE aud.report_id = gen.report_id) + WHEN FALSE THEN 'No' + ELSE 'Yes' + END AS is_secondary_auditors + from + dissemination_general gen + order by gen.id +; + +--------------------------------------- +-- auditor (secondary auditor) +--------------------------------------- +create view api_v1_1_0.secondary_auditors as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + sa.auditor_ein, + sa.auditor_name, + sa.contact_name, + sa.contact_title, + sa.contact_email, + sa.contact_phone, + sa.address_street, + sa.address_city, + sa.address_state, + sa.address_zipcode + from + dissemination_General gen, + dissemination_SecondaryAuditor sa + where + sa.report_id = gen.report_id + order by sa.id +; + +create view api_v1_1_0.additional_eins as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + ein.additional_ein + from + dissemination_general gen, + dissemination_additionalein ein + where + gen.report_id = ein.report_id + order by ein.id +; + +commit; + + + +notify pgrst, + 'reload schema'; + diff --git a/backend/dissemination/sql/api/api_v1_1_0/teardown.sql b/backend/dissemination/sql/api/api_v1_1_0/teardown.sql new file mode 100644 index 0000000000..d0e2e484d3 --- /dev/null +++ b/backend/dissemination/sql/api/api_v1_1_0/teardown.sql @@ -0,0 +1,27 @@ +begin; + +DROP SCHEMA IF EXISTS api_v1_1_0 CASCADE; +-- DROP ROLE IF EXISTS authenticator; +-- DROP ROLE IF EXISTS api_fac_gov; + +commit; + +notify pgrst, + 'reload schema'; +begin; + drop table if exists api_v1_1_0.metadata; + drop view if exists api_v1_1_0.general; + drop view if exists api_v1_1_0.auditor; + drop view if exists api_v1_1_0.federal_awards; + drop view if exists api_v1_1_0.findings; + drop view if exists api_v1_1_0.findings_text; + drop view if exists api_v1_1_0.corrective_action_plans; + drop view if exists api_v1_1_0.additional_ueis; + drop view if exists api_v1_1_0.notes_to_sefa; + drop view if exists api_v1_1_0.passthrough; + drop view if exists api_v1_1_0.secondary_auditors; + drop view if exists api_v1_1_0.additional_eins; +commit; + +notify pgrst, + 'reload schema'; diff --git a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/base.sql b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/base.sql new file mode 100644 index 0000000000..c6f8853bfa --- /dev/null +++ b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/base.sql @@ -0,0 +1,36 @@ +DO +$do$ +BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = 'authenticator') THEN + RAISE NOTICE 'Role "authenticator" already exists. Skipping.'; + ELSE + CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; + END IF; +END +$do$; + +DO +$do$ +BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = 'api_fac_gov') THEN + RAISE NOTICE 'Role "api_fac_gov" already exists. Skipping.'; + ELSE + CREATE ROLE api_fac_gov NOLOGIN; + END IF; +END +$do$; + +GRANT api_fac_gov TO authenticator; + +CREATE SEQUENCE IF NOT EXISTS public_data_general + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE; + + +NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/create_functions.sql b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/create_functions.sql new file mode 100644 index 0000000000..5c4db5d713 --- /dev/null +++ b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/create_functions.sql @@ -0,0 +1,4 @@ +-- Under the new approach, we don't need +-- any functions here. + +NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/create_schema.sql b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/create_schema.sql new file mode 100644 index 0000000000..0c0d6d58bf --- /dev/null +++ b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/create_schema.sql @@ -0,0 +1,42 @@ +begin; + +do +$$ +BEGIN + DROP SCHEMA IF EXISTS public_api_v2_0_0_alpha CASCADE; + DROP SCHEMA IF EXISTS public_api_v2_0_0_alpha_functions CASCADE; + + CREATE SCHEMA IF NOT EXISTS public_api_v2_0_0_alpha; + CREATE SCHEMA IF NOT EXISTS public_api_v2_0_0_alpha_functions; + + GRANT USAGE ON SCHEMA public_api_v2_0_0_alpha_functions TO api_fac_gov; + + -- Grant access to tables and views + ALTER DEFAULT PRIVILEGES + IN SCHEMA public_api_v2_0_0_alpha + GRANT SELECT + -- this includes views + ON tables + TO api_fac_gov; + + -- Grant access to sequences, if we have them + GRANT USAGE ON SCHEMA public_api_v2_0_0_alpha to api_fac_gov; + GRANT SELECT, USAGE + ON ALL SEQUENCES + IN SCHEMA public_api_v2_0_0_alpha + TO api_fac_gov; + + -- ALTER DEFAULT PRIVILEGES + -- IN SCHEMA public_api_v2_0_0_alpha + -- GRANT SELECT, USAGE + -- ON sequences + -- TO api_fac_gov; +END +$$ +; + +COMMIT; + +notify pgrst, + 'reload schema'; + diff --git a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/create_views.sql b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/create_views.sql new file mode 100644 index 0000000000..c98acde746 --- /dev/null +++ b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/create_views.sql @@ -0,0 +1,182 @@ +BEGIN; + +CREATE VIEW public_api_v2_0_0_alpha.additional_eins AS + SELECT * FROM public_data.additional_eins +; + +--------------------------------------- +-- additional_ueis +--------------------------------------- +create view public_api_v2_0_0_alpha.additional_ueis AS + SELECT * FROM additional_ueis +; + +--------------------------------------- +-- corrective_action_plan +--------------------------------------- +CREATE VIEW public_api_v2_0_0_alpha.corrective_action_plans AS + SELECT * FROM public_data.corrective_action_plans + ORDER BY public_data.corrective_action_plans.id +; + +--------------------------------------- +-- finding +--------------------------------------- +CREATE VIEW public_api_v2_0_0_alpha.findings AS + SELECT * FROM public_data.findings +; + +--------------------------------------- +-- finding_text +--------------------------------------- +CREATE VIEW public_api_v2_0_0_alpha.findings_text AS + SELECT * FROM public_data.findings_text ft + ORDER BY ft.id +; + +--------------------------------------- +-- federal award +--------------------------------------- +CREATE VIEW public_api_v2_0_0_alpha.federal_awards AS + SELECT * FROM public_data.federal_awards +; + +--------------------------------------- +-- general +--------------------------------------- +CREATE VIEW public_api_v2_0_0_alpha.general AS + SELECT * FROM public_data.general +; + +--------------------------------------- +-- notes_to_sefa +--------------------------------------- +create view public_api_v2_0_0_alpha.notes_to_sefa AS + SELECT * FROM public_data.notes_to_sefa nts + ORDER BY nts.id +; + +--------------------------------------- +-- passthrough +--------------------------------------- +CREATE VIEW public_api_v2_0_0_alpha.passthrough AS + SELECT * FROM public_data.passthrough +; + + +--------------------------------------- +-- auditor (secondary auditor) +--------------------------------------- +CREATE VIEW public_api_v2_0_0_alpha.secondary_auditors AS + SELECT * FROM public_data.secondary_auditors + ; + +-- Specify every field in dissemination_combined, omitting the id. +-- Generated fields like ALN are done in the creation of the table, not here. +create view public_api_v2_0_0_alpha.combined as + select + combined.report_id, + combined.award_reference, + combined.reference_number, + combined.aln, + combined.agencies_with_prior_findings, + combined.audit_period_covered, + combined.audit_type, + combined.audit_year, + combined.auditee_address_line_1, + combined.auditee_certified_date, + combined.auditee_certify_name, + combined.auditee_certify_title, + combined.auditee_city, + combined.auditee_contact_name, + combined.auditee_contact_title, + combined.auditee_ein, + combined.auditee_email, + combined.auditee_name, + combined.auditee_phone, + combined.auditee_state, + combined.auditee_uei, + combined.auditee_zip, + combined.auditor_address_line_1, + combined.auditor_certified_date, + combined.auditor_certify_name, + combined.auditor_certify_title, + combined.auditor_city, + combined.auditor_contact_name, + combined.auditor_contact_title, + combined.auditor_country, + combined.auditor_ein, + combined.auditor_email, + combined.auditor_firm_name, + combined.auditor_foreign_address, + combined.auditor_phone, + combined.auditor_state, + combined.auditor_zip, + combined.cognizant_agency, + combined.data_source, + combined.date_created, + combined.dollar_threshold, + combined.entity_type, + combined.fac_accepted_date, + combined.fy_end_date, + combined.fy_start_date, + combined.gaap_results, + combined.is_additional_ueis, + combined.is_aicpa_audit_guide_included, + combined.is_going_concern_included, + combined.is_internal_control_deficiency_disclosed, + combined.is_internal_control_material_weakness_disclosed, + combined.is_low_risk_auditee, + combined.is_material_noncompliance_disclosed, + combined.is_public, + combined.is_sp_framework_required, + combined.number_months, + combined.oversight_agency, + combined.ready_for_certification_date, + combined.sp_framework_basis, + combined.sp_framework_opinions, + combined.submitted_date, + combined.total_amount_expended, + combined.type_audit_code, + combined.additional_award_identification, + combined.amount_expended, + combined.cluster_name, + combined.cluster_total, + combined.federal_agency_prefix, + combined.federal_award_extension, + combined.federal_program_name, + combined.federal_program_total, + combined.findings_count, + combined.is_direct, + combined.is_loan, + combined.is_major, + combined.is_passthrough_award, + combined.loan_balance, + combined.audit_report_type, + combined.other_cluster_name, + combined.passthrough_amount, + combined.state_cluster_name, + combined.is_material_weakness, + combined.is_modified_opinion, + combined.is_other_findings, + combined.is_other_matters, + combined.is_questioned_costs, + combined.is_repeat_finding, + combined.is_significant_deficiency, + combined.prior_finding_ref_numbers, + combined.type_requirement, + combined.passthrough_name, + combined.passthrough_id + from + dissemination_combined combined + where + (combined.is_public = true + or (combined.is_public = false and public_api_v2_0_0_alpha_functions.has_tribal_data_access())) + order by combined.id +; + +COMMIT; + +notify pgrst, + 'reload schema'; + diff --git a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/drop_schema.sql b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/drop_schema.sql new file mode 100644 index 0000000000..cb24063256 --- /dev/null +++ b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/drop_schema.sql @@ -0,0 +1,12 @@ + +begin; + +DROP SCHEMA IF EXISTS public_api_v2_0_0_alpha CASCADE; + +-- DROP ROLE IF EXISTS authenticator; +-- DROP ROLE IF EXISTS api_fac_gov; + +commit; + +notify pgrst, + 'reload schema'; diff --git a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/drop_views.sql b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/drop_views.sql new file mode 100644 index 0000000000..c95c0fc139 --- /dev/null +++ b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/drop_views.sql @@ -0,0 +1,18 @@ +begin; + drop table if exists api_v1_1_1.metadata; + drop view if exists api_v1_1_1.general; + drop view if exists api_v1_1_1.auditor; + drop view if exists api_v1_1_1.federal_awards; + drop view if exists api_v1_1_1.findings; + drop view if exists api_v1_1_1.findings_text; + drop view if exists api_v1_1_1.corrective_action_plans; + drop view if exists api_v1_1_1.additional_ueis; + drop view if exists api_v1_1_1.notes_to_sefa; + drop view if exists api_v1_1_1.passthrough; + drop view if exists api_v1_1_1.secondary_auditors; + drop view if exists api_v1_1_1.additional_eins; + drop view if exists api_v1_1_1.combined; +commit; + +notify pgrst, + 'reload schema'; diff --git a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/setup.sql b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/setup.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/standup.sql b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/standup.sql new file mode 100644 index 0000000000..16e76dab85 --- /dev/null +++ b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/standup.sql @@ -0,0 +1,269 @@ +DO +$do$ +BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = 'authenticator') THEN + RAISE NOTICE 'Role "authenticator" already exists. Skipping.'; + ELSE + CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; + END IF; +END +$do$; + +DO +$do$ +BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = 'api_fac_gov') THEN + RAISE NOTICE 'Role "api_fac_gov" already exists. Skipping.'; + ELSE + CREATE ROLE api_fac_gov NOLOGIN; + END IF; +END +$do$; + +GRANT api_fac_gov TO authenticator; + +CREATE SEQUENCE IF NOT EXISTS public_data_general + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE; + + +NOTIFY pgrst, 'reload schema'; +begin; + +do +$$ +BEGIN + DROP SCHEMA IF EXISTS public_api_v2_0_0_alpha CASCADE; + DROP SCHEMA IF EXISTS public_api_v2_0_0_alpha_functions CASCADE; + + CREATE SCHEMA IF NOT EXISTS public_api_v2_0_0_alpha; + CREATE SCHEMA IF NOT EXISTS public_api_v2_0_0_alpha_functions; + + GRANT USAGE ON SCHEMA public_api_v2_0_0_alpha_functions TO api_fac_gov; + + -- Grant access to tables and views + ALTER DEFAULT PRIVILEGES + IN SCHEMA public_api_v2_0_0_alpha + GRANT SELECT + -- this includes views + ON tables + TO api_fac_gov; + + -- Grant access to sequences, if we have them + GRANT USAGE ON SCHEMA public_api_v2_0_0_alpha to api_fac_gov; + GRANT SELECT, USAGE + ON ALL SEQUENCES + IN SCHEMA public_api_v2_0_0_alpha + TO api_fac_gov; + + -- ALTER DEFAULT PRIVILEGES + -- IN SCHEMA public_api_v2_0_0_alpha + -- GRANT SELECT, USAGE + -- ON sequences + -- TO api_fac_gov; +END +$$ +; + +COMMIT; + +notify pgrst, + 'reload schema'; + +-- Under the new approach, we don't need +-- any functions here. + +NOTIFY pgrst, 'reload schema'; +BEGIN; + +CREATE VIEW public_api_v2_0_0_alpha.additional_eins AS + SELECT * FROM public_data_v1_0_0.additional_eins ae + ORDER BY ae.id +; + +--------------------------------------- +-- additional_ueis +--------------------------------------- +create view public_api_v2_0_0_alpha.additional_ueis AS + SELECT * FROM public_data_v1_0_0.additional_ueis au + ORDER BY au.id +; + +--------------------------------------- +-- corrective_action_plan +--------------------------------------- +CREATE VIEW public_api_v2_0_0_alpha.corrective_action_plans AS + SELECT * FROM public_data_v1_0_0.corrective_action_plans cap + ORDER BY cap.id +; + +--------------------------------------- +-- finding +--------------------------------------- +CREATE VIEW public_api_v2_0_0_alpha.findings as + SELECT * FROM public_data_v1_0_0.findings f + ORDER BY f.id +; + +--------------------------------------- +-- finding_text +--------------------------------------- +CREATE VIEW public_api_v2_0_0_alpha.findings_text AS + SELECT * FROM public_data_v1_0_0.findings_text ft + ORDER BY ft.id +; + +--------------------------------------- +-- federal award +--------------------------------------- +CREATE VIEW public_api_v2_0_0_alpha.federal_awards AS + SELECT * FROM public_data_v1_0_0.federal_awards fa + ORDER BY fa.id +; + +--------------------------------------- +-- general +--------------------------------------- +CREATE VIEW public_api_v2_0_0_alpha.general AS + SELECT * FROM public_data_v1_0_0.general +; + +--------------------------------------- +-- notes_to_sefa +--------------------------------------- +create view public_api_v2_0_0_alpha.notes_to_sefa AS + SELECT * FROM public_data_v1_0_0.notes_to_sefa nts + ORDER BY nts.id +; + +--------------------------------------- +-- passthrough +--------------------------------------- +CREATE VIEW public_api_v2_0_0_alpha.passthrough AS + SELECT * FROM public_data_v1_0_0.passthrough p + ORDER BY p.id +; + + +--------------------------------------- +-- auditor (secondary auditor) +--------------------------------------- +CREATE VIEW public_api_v2_0_0_alpha.secondary_auditors AS + SELECT * FROM public_data_v1_0_0.secondary_auditors sa + ORDER BY sa.id + ; + +-- Specify every field in dissemination_combined, omitting the id. +-- Generated fields like ALN are done in the creation of the table, not here. +-- create view public_api_v2_0_0_alpha.combined as +-- select +-- combined.report_id, +-- combined.award_reference, +-- combined.reference_number, +-- combined.aln, +-- combined.agencies_with_prior_findings, +-- combined.audit_period_covered, +-- combined.audit_type, +-- combined.audit_year, +-- combined.auditee_address_line_1, +-- combined.auditee_certified_date, +-- combined.auditee_certify_name, +-- combined.auditee_certify_title, +-- combined.auditee_city, +-- combined.auditee_contact_name, +-- combined.auditee_contact_title, +-- combined.auditee_ein, +-- combined.auditee_email, +-- combined.auditee_name, +-- combined.auditee_phone, +-- combined.auditee_state, +-- combined.auditee_uei, +-- combined.auditee_zip, +-- combined.auditor_address_line_1, +-- combined.auditor_certified_date, +-- combined.auditor_certify_name, +-- combined.auditor_certify_title, +-- combined.auditor_city, +-- combined.auditor_contact_name, +-- combined.auditor_contact_title, +-- combined.auditor_country, +-- combined.auditor_ein, +-- combined.auditor_email, +-- combined.auditor_firm_name, +-- combined.auditor_foreign_address, +-- combined.auditor_phone, +-- combined.auditor_state, +-- combined.auditor_zip, +-- combined.cognizant_agency, +-- combined.data_source, +-- combined.date_created, +-- combined.dollar_threshold, +-- combined.entity_type, +-- combined.fac_accepted_date, +-- combined.fy_end_date, +-- combined.fy_start_date, +-- combined.gaap_results, +-- combined.is_additional_ueis, +-- combined.is_aicpa_audit_guide_included, +-- combined.is_going_concern_included, +-- combined.is_internal_control_deficiency_disclosed, +-- combined.is_internal_control_material_weakness_disclosed, +-- combined.is_low_risk_auditee, +-- combined.is_material_noncompliance_disclosed, +-- combined.is_public, +-- combined.is_sp_framework_required, +-- combined.number_months, +-- combined.oversight_agency, +-- combined.ready_for_certification_date, +-- combined.sp_framework_basis, +-- combined.sp_framework_opinions, +-- combined.submitted_date, +-- combined.total_amount_expended, +-- combined.type_audit_code, +-- combined.additional_award_identification, +-- combined.amount_expended, +-- combined.cluster_name, +-- combined.cluster_total, +-- combined.federal_agency_prefix, +-- combined.federal_award_extension, +-- combined.federal_program_name, +-- combined.federal_program_total, +-- combined.findings_count, +-- combined.is_direct, +-- combined.is_loan, +-- combined.is_major, +-- combined.is_passthrough_award, +-- combined.loan_balance, +-- combined.audit_report_type, +-- combined.other_cluster_name, +-- combined.passthrough_amount, +-- combined.state_cluster_name, +-- combined.is_material_weakness, +-- combined.is_modified_opinion, +-- combined.is_other_findings, +-- combined.is_other_matters, +-- combined.is_questioned_costs, +-- combined.is_repeat_finding, +-- combined.is_significant_deficiency, +-- combined.prior_finding_ref_numbers, +-- combined.type_requirement, +-- combined.passthrough_name, +-- combined.passthrough_id +-- from +-- dissemination_combined combined +-- where +-- (combined.is_public = true +-- or (combined.is_public = false and public_api_v2_0_0_alpha_functions.has_tribal_data_access())) +-- order by combined.id +-- ; + +COMMIT; + +notify pgrst, + 'reload schema'; diff --git a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/teardown.sql b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/teardown.sql new file mode 100644 index 0000000000..718504fa32 --- /dev/null +++ b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/teardown.sql @@ -0,0 +1,9 @@ +BEGIN; + -- + -- This drops all the views, too. + --- + DROP SCHEMA IF EXISTS public_api_v2_0_0_alpha CASCADE; +COMMIT; + +notify pgrst, + 'reload schema'; diff --git a/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml b/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml new file mode 100644 index 0000000000..d5db7a613d --- /dev/null +++ b/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml @@ -0,0 +1,469 @@ +# This uses an env var called PG to +# set the connection string. +source: FAC_DB_URI +target: FAC_DB_URI +env: + SLING_ALLOW_EMPTY: true + +# Handy +# SELECT column_name +# FROM information_schema.columns +# WHERE table_schema = 'public' +# AND table_name = 'dissemination_general' +# AND column_name NOT IN ('id'); + +streams: + # The first table we create is the general table. All of the + # general table is public data, so we copy it wholesale. + public.dissemination_general: + mode: full-refresh + object: public_data_v1_0_0.general + sql: | + -- All of the general table is public data. + SELECT + nextval('public_data_v1_0_0.seq_general') AS id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.agencies_with_prior_findings, + gen.audit_period_covered, + gen.audit_type, + gen.auditee_address_line_1, + gen.auditee_certified_date, + gen.auditee_certify_name, + gen.auditee_certify_title, + gen.auditee_city, + gen.auditee_contact_name, + gen.auditee_contact_title, + gen.auditee_ein, + gen.auditee_email, + gen.auditee_name, + gen.auditee_phone, + gen.auditee_state, + gen.auditee_zip, + gen.auditor_address_line_1, + gen.auditor_certified_date, + gen.auditor_certify_name, + gen.auditor_certify_title, + gen.auditor_city, + gen.auditor_contact_name, + gen.auditor_contact_title, + gen.auditor_country, + gen.auditor_ein, + gen.auditor_email, + gen.auditor_firm_name, + gen.auditor_foreign_address, + gen.auditor_phone, + gen.auditor_state, + gen.auditor_zip, + gen.cognizant_agency, + gen.data_source, + gen.date_created, + gen.dollar_threshold, + gen.entity_type, + gen.fac_accepted_date, + gen.fy_end_date, + gen.fy_start_date, + gen.gaap_results, + gen.is_additional_ueis, + gen.is_aicpa_audit_guide_included, + gen.is_going_concern_included, + gen.is_internal_control_deficiency_disclosed, + gen.is_internal_control_material_weakness_disclosed, + gen.is_low_risk_auditee, + gen.is_material_noncompliance_disclosed, + CASE EXISTS + ( + SELECT + ein.report_id + FROM + dissemination_additionalein ein + WHERE + ein.report_id = gen.report_id + ) + WHEN FALSE THEN 'No' + ELSE 'Yes' + END AS is_multiple_eins, + gen.is_public, + CASE EXISTS + ( + SELECT + aud.report_id + FROM + dissemination_secondaryauditor aud + WHERE + aud.report_id = gen.report_id + ) + WHEN FALSE THEN 'No' + ELSE 'Yes' + END AS is_secondary_auditors, + gen.is_sp_framework_required, + gen.number_months, + gen.oversight_agency, + gen.ready_for_certification_date, + gen.sp_framework_basis, + gen.sp_framework_opinions, + gen.submitted_date, + gen.total_amount_expended, + gen.type_audit_code + FROM + public.dissemination_general gen + ORDER BY gen.id + + # All additional EINs are public. + public.additional_eins: + mode: full-refresh + object: public_data_v1_0_0.additional_eins + sql: | + SELECT + nextval('public_data_v1_0_0.seq_additional_eins') AS id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + ein.additional_ein + FROM + public_data_v1_0_0.general gen, + public.dissemination_additionalein ein + WHERE + gen.report_id = ein.report_id + ORDER BY ein.id + + # All of the additional UEI info is public info. + public.dissemination_additional_ueis: + mode: full-refresh + object: public_data_v1_0_0.additional_ueis + sql: | + SELECT + nextval('public_data_v1_0_0.seq_additional_ueis') AS id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + uei.additional_uei + FROM + public_data_v1_0_0.general gen, + public.dissemination_additionaluei uei + WHERE + gen.report_id = uei.report_id + ORDER BY uei.id + + # Corrective action plans are NOT always public. + public.dissemination_corrective_action_plan: + mode: full-refresh + object: public_data_v1_0_0.corrective_action_plans + sql: | + SELECT + nextval('public_data_v1_0_0.seq_corrective_action_plans') AS id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + ct.contains_chart_or_table, + ct.finding_ref_number, + ct.planned_action + FROM + public.dissemination_captext ct, + public_data_v1_0_0.general gen + WHERE + ct.report_id = gen.report_id + AND + -- Only include the public corrective action plans. + gen.is_public = true + ORDER BY ct.id + + # All Federal award data is public. + public.dissemination_federalaward: + mode: full-refresh + object: public_data_v1_0_0.federal_awards + sql: | + SELECT + nextval('public_data_v1_0_0.seq_federal_awards') AS id, + dg.report_id, + dg.auditee_uei, + dg.audit_year, + dg.fac_accepted_date, + --- + award.additional_award_identification, + award.amount_expended, + award.audit_report_type, + award.award_reference, + award.cluster_name, + award.cluster_total, + award.federal_agency_prefix, + award.federal_award_extension, + award.federal_program_name, + award.federal_program_total, + award.findings_count, + award.is_direct, + award.is_loan, + award.is_major, + award.is_passthrough_award, + award.loan_balance, + award.other_cluster_name, + award.passthrough_amount, + award.state_cluster_name + FROM + public.dissemination_federalaward award, + public_data_v1_0_0.general dg + WHERE + award.report_id = dg.report_id + ORDER BY award.id + + # The findings table is public. + public.dissemination_findings: + mode: full-refresh + object: public_data_v1_0_0.findings + sql: | + SELECT + nextval('public_data_v1_0_0.seq_findings') AS id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + finding.award_reference, + finding.is_material_weakness, + finding.is_modified_opinion, + finding.is_other_findings, + finding.is_other_matters, + finding.is_questioned_costs, + finding.is_repeat_finding, + finding.is_significant_deficiency, + finding.prior_finding_ref_numbers, + finding.reference_number, + finding.type_requirement + FROM + public.dissemination_finding finding, + public_data_v1_0_0.general gen + WHERE + finding.report_id = gen.report_id + ORDER BY finding.id + + # Findings text is NOT always public. + public.dissemination_findingstext: + mode: full-refresh + object: public_data_v1_0_0.findings_text + sql: | + SELECT + nextval('public_data_v1_0_0.seq_findings_text') AS id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + ft.finding_ref_number, + ft.contains_chart_or_table, + ft.finding_text + FROM + public.dissemination_findingtext ft, + public_data_v1_0_0.general gen + WHERE + ft.report_id = gen.report_id + AND + gen.is_public = true + ORDER BY ft.id + + # The notes to SEFA are NOT all public. + public.dissemination_notes: + mode: full-refresh + object: public_data_v1_0_0.notes_to_sefa + sql: | + SELECT + nextval('public_data_v1_0_0.seq_notes_to_sefa') AS id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + note.accounting_policies, + note.contains_chart_or_table, + note.content, + note.is_minimis_rate_used, + note.note_title as title, + note.rate_explained + FROM + public_data_v1_0_0.general gen, + public.dissemination_note note + WHERE + note.report_id = gen.report_id + AND + -- Some notes are not public. + gen.is_public = true + ORDER BY note.id + + # All passthrough information is public. + public.dissemination_passthrough: + mode: full-refresh + object: public_data_v1_0_0.passthrough + sql: | + SELECT + nextval('public_data_v1_0_0.seq_passthrough') AS id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + pass.award_reference, + pass.passthrough_id, + pass.passthrough_name + FROM + public_data_v1_0_0.general gen, + public.dissemination_passthrough pass + WHERE + gen.report_id = pass.report_id + ORDER BY pass.id + + # All secondary auditor info is public. + public.secondary_auditors: + mode: full-refresh + object: public_data_v1_0_0.secondary_auditors + sql: | + SELECT + nextval('public_data_v1_0_0.seq_secondary_auditors') AS id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + sa.address_city, + sa.address_state, + sa.address_street, + sa.address_zipcode, + sa.auditor_ein, + sa.auditor_name, + sa.contact_email, + sa.contact_name, + sa.contact_phone, + sa.contact_title + FROM + public_data_v1_0_0.general gen, + public.dissemination_secondaryauditor sa + WHERE + sa.report_id = gen.report_id + ORDER BY sa.id + + public.combined: + mode: full-refresh + object: public_data_v1_0_0.combined + sql: | + SELECT + nextval('public_data_v1_0_0.seq_combined') AS id, + dg.report_id, + dfa.award_reference, + df.reference_number, + concat(dfa.federal_agency_prefix,'.',dfa.federal_award_extension) as aln, + -- + -- general + -- + dg.auditee_uei, + dg.audit_year, + dg.agencies_with_prior_findings, + dg.audit_period_covered, + dg.audit_type, + dg.auditee_address_line_1, + dg.auditee_certified_date, + dg.auditee_certify_name, + dg.auditee_certify_title, + dg.auditee_city, + dg.auditee_contact_name, + dg.auditee_contact_title, + dg.auditee_ein, + dg.auditee_email, + dg.auditee_name, + dg.auditee_phone, + dg.auditee_state, + dg.auditee_zip, + dg.auditor_address_line_1, + dg.auditor_certified_date, + dg.auditor_certify_name, + dg.auditor_certify_title, + dg.auditor_city, + dg.auditor_contact_name, + dg.auditor_contact_title, + dg.auditor_country, + dg.auditor_ein, + dg.auditor_email, + dg.auditor_firm_name, + dg.auditor_foreign_address, + dg.auditor_phone, + dg.auditor_state, + dg.auditor_zip, + dg.cognizant_agency, + dg.data_source, + dg.date_created, + dg.dollar_threshold, + dg.entity_type, + dg.fac_accepted_date, + dg.fy_end_date, + dg.fy_start_date, + dg.gaap_results, + dg.is_additional_ueis, + dg.is_aicpa_audit_guide_included, + dg.is_going_concern_included, + dg.is_internal_control_deficiency_disclosed, + dg.is_internal_control_material_weakness_disclosed, + dg.is_low_risk_auditee, + dg.is_material_noncompliance_disclosed, + dg.is_multiple_eins, + dg.is_public, + dg.is_secondary_auditors, + dg.is_sp_framework_required, + dg.number_months, + dg.oversight_agency, + dg.ready_for_certification_date, + dg.sp_framework_basis, + dg.sp_framework_opinions, + dg.submitted_date, + dg.total_amount_expended, + dg.type_audit_code, + -- + -- federal_award + -- + dfa.additional_award_identification, + dfa.amount_expended, + dfa.audit_report_type, + dfa.cluster_name, + dfa.cluster_total, + dfa.federal_agency_prefix, + dfa.federal_award_extension, + dfa.federal_program_name, + dfa.federal_program_total, + dfa.findings_count, + dfa.is_direct, + dfa.is_loan, + dfa.is_major, + dfa.is_passthrough_award, + dfa.loan_balance, + dfa.other_cluster_name, + dfa.passthrough_amount, + dfa.state_cluster_name, + -- + -- finding + -- + df.is_material_weakness, + df.is_modified_opinion, + df.is_other_findings, + df.is_other_matters, + df.is_questioned_costs, + df.is_repeat_finding, + df.is_significant_deficiency, + df.prior_finding_ref_numbers, + df.type_requirement, + -- + -- passthrough + -- + dp.passthrough_id, + dp.passthrough_name + FROM + public_data_v1_0_0.federal_awards dfa + LEFT JOIN public_data_v1_0_0.general dg + ON dfa.report_id = dg.report_id + LEFT JOIN public_data_v1_0_0.findings df + ON dfa.report_id = df.report_id + AND dfa.award_reference = df.award_reference + LEFT JOIN public_data_v1_0_0.passthrough dp + ON dfa.report_id = dp.report_id + AND dfa.award_reference = dp.award_reference diff --git a/backend/dissemination/sql/sling/public_data_v1_0_0/standup.sql b/backend/dissemination/sql/sling/public_data_v1_0_0/standup.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/dissemination/sql/sling/public_data_v1_0_0/teardown.sql b/backend/dissemination/sql/sling/public_data_v1_0_0/teardown.sql new file mode 100644 index 0000000000..022a8b409a --- /dev/null +++ b/backend/dissemination/sql/sling/public_data_v1_0_0/teardown.sql @@ -0,0 +1,78 @@ + +BEGIN; + + -- + -- We always tear everything down. Every time we create the public data, + -- it is a complete refresh. So, rip out the schemas. + -- + DROP SCHEMA IF EXISTS public_data_v1_0_0 CASCADE; + + -- + -- The teardown has to do some standup. Why? In this case, we're about to run + -- `sling`. It will need to access some sequences to create the new tables with unique + -- row IDs. Those sequences need to exist before sling runs. + -- + CREATE SCHEMA IF NOT EXISTS public_data_v1_0_0; + + -- + -- Sequences for the public tables. + CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_general + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE; + CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_additional_eins + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE; + CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_additional_ueis + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE; + CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_corrective_action_plans + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE; + CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_federal_awards + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE; + CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_findings + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE; + CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_findings_text + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE; + CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_notes_to_sefa + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE; + CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_passthrough + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE; + CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_secondary_auditors + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE; + CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_combined + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE; + +COMMIT; + +notify pgrst, + 'reload schema'; diff --git a/backend/docker-compose.yml b/backend/docker-compose.yml index f8ee53cada..afa155a863 100644 --- a/backend/docker-compose.yml +++ b/backend/docker-compose.yml @@ -63,6 +63,7 @@ services: condition: service_started environment: DATABASE_URL: postgres://postgres@db/postgres + SNAPSHOT_URL: postgres://postgres@db2/postgres POSTGREST_URL: http://api:3000 DJANGO_DEBUG: true SAM_API_KEY: ${SAM_API_KEY} @@ -129,7 +130,7 @@ services: PGRST_DB_SCHEMAS: "api_v1_0_3, api_v1_1_0, admin_api_v1_1_0" PGRST_JWT_SECRET: ${PGRST_JWT_SECRET:-32_chars_fallback_secret_testing} # Fallback value for testing environments # Enable this to inspect the DB plans for queries via EXPLAIN - PGRST_DB_PLAN_ENABLED: ${PGRST_DB_PLAN_ENABLED:-false} + PGRST_DB_PLAN_ENABLED: 1 depends_on: db: condition: service_healthy diff --git a/backend/run.sh b/backend/run.sh index 297c7b5738..30e71f1093 100755 --- a/backend/run.sh +++ b/backend/run.sh @@ -10,6 +10,7 @@ source tools/util_startup.sh # for local envs (LOCAL or TESTING) and cloud.gov source tools/setup_env.sh source tools/api_teardown.sh +source tools/sling_public_tables.sh source tools/migrate_app_tables.sh source tools/api_standup.sh source tools/seed_cog_baseline.sh @@ -26,6 +27,24 @@ gonogo "setup_env" api_teardown gonogo "api_teardown" +##### +# PUBLIC TABLES +# We only want to do this if the sling tables don't exist. +# Because they should run nightly, we'll generally skip this +# during startup. +check_table_exists $FAC_DB_URI 'public_data_v1_0_0.general' +if [ $? -ne 0 ]; +then + # This takes time. We'll need to watch our deploy timeout. + # And, it will interrupt search/API while the tables are being + # recreated. So, we generally don't want to do this on deploy, only + # as part of nightly batch jobs. + sling_public_tables + gonogo "sling_public_tables" +else + startup_log "RUN" "Skipping sling" +fi + ##### # MIGRATE APP TABLES migrate_app_tables diff --git a/backend/tools/api_standup.sh b/backend/tools/api_standup.sh index 23567bed9a..1c75a0ea06 100644 --- a/backend/tools/api_standup.sh +++ b/backend/tools/api_standup.sh @@ -1,29 +1,31 @@ source tools/util_startup.sh + function api_standup { startup_log "API_STANDUP" "BEGIN" + run_sql_for_public_apis 'api' 'standup.sql' + # First create non-managed tables - startup_log "CREATE_API_ACCESS_TABLES" "BEGIN" - python manage.py create_api_access_tables - local d1=$? - startup_log "CREATE_API_ACCESS_TABLES" "END" + #startup_log "CREATE_API_ACCESS_TABLES" "BEGIN" + # python manage.py create_api_access_tables + #run_sql_for_public_apis 'api' 'create_access_tables.sql' + #startup_log "CREATE_API_ACCESS_TABLES" "END" # Bring the API back, possibly installing a new API - startup_log "CREATE_API_SCHEMA" "BEGIN" - python manage.py create_api_schema - local d2=$? - startup_log "CREATE_API_SCHEMA" "END" + # startup_log "CREATE_API_SCHEMA" "BEGIN" + # # python manage.py create_api_schema + # run_sql_for_public_apis 'api' 'base.sql' + # run_sql_for_public_apis 'api' 'create_schema.sql' + # startup_log "CREATE_API_SCHEMA" "END" - startup_log "CREATE_API_VIEWS" "BEGIN" - python manage.py create_api_views && - local d3=$? - startup_log "CREATE_API_VIEWS" "END" - + # startup_log "CREATE_API_VIEWS" "BEGIN" + # # python manage.py create_api_views && + # run_sql_for_public_apis 'api' 'create_functions.sql' + # run_sql_for_public_apis 'api' 'create_views.sql' + # startup_log "CREATE_API_VIEWS" "END" startup_log "API_STANDUP" "END" - result=$((($d1 + $d2) + $d3)) - # If these are all zero, we're all good. - return $result + return 0 } diff --git a/backend/tools/api_teardown.sh b/backend/tools/api_teardown.sh index 80292d3181..a8892e9c15 100644 --- a/backend/tools/api_teardown.sh +++ b/backend/tools/api_teardown.sh @@ -3,18 +3,24 @@ source tools/util_startup.sh function api_teardown { startup_log "API_TEARDOWN" "BEGIN" - startup_log "DROP_DEPRECATED_API_SCHEMA_AND_VIEWS" "BEGIN" - python manage.py drop_deprecated_api_schema_and_views - local d1=$? - startup_log "DROP_DEPRECATED_API_SCHEMA_AND_VIEWS" "END" - startup_log "DROP_API_SCHEMA" "BEGIN" - python manage.py drop_api_schema - local d2=$? - startup_log "DROP_API_SCHEMA" "END" + # startup_log "DROP_DEPRECATED_API_SCHEMA_AND_VIEWS" "BEGIN" + # # python manage.py drop_deprecated_api_schema_and_views + # local d1=$? + # startup_log "DROP_DEPRECATED_API_SCHEMA_AND_VIEWS" "END" + # startup_log "DROP_API_SCHEMA" "BEGIN" + # python manage.py drop_api_schema + + # for api_version in "${public_api_versions[@]}" + # do + # echo "VERSION $api_version" + # $PSQL_EXE $FAC_SNAPSHOT_URI < dissemination/sql/api/${api_version}/drop_schema.sql + # gonogo "$api_version teardown.sql" + # done + + run_sql_for_public_apis 'api' 'teardown.sql' startup_log "API_TEARDOWN" "END" - result=$(($d1 + $d2)) # If these are both zero, we're all good. - return $result + return 0 } diff --git a/backend/tools/historic_migrator/all_audits_that_failed_migrations.csv b/backend/tools/historic_migrator/all_audits_that_failed_migrations.csv new file mode 100644 index 0000000000..b09f16662a --- /dev/null +++ b/backend/tools/historic_migrator/all_audits_that_failed_migrations.csv @@ -0,0 +1,3329 @@ +dbkey,audit_year +245186,2016 +181884,2016 +238896,2016 +187529,2016 +237862,2016 +184042,2016 +74828,2016 +245454,2016 +188100,2016 +184711,2016 +187796,2016 +229655,2016 +141780,2016 +110841,2016 +112120,2016 +243942,2016 +247316,2016 +9156,2016 +188358,2016 +246222,2016 +82235,2016 +73891,2016 +202062,2016 +64586,2016 +241362,2016 +239399,2016 +204681,2016 +183865,2016 +202639,2016 +166571,2016 +138640,2016 +43776,2016 +165496,2016 +49070,2016 +170759,2016 +219517,2016 +112265,2016 +104928,2016 +183596,2016 +233733,2016 +245635,2016 +245991,2016 +183478,2016 +243590,2016 +211245,2016 +196078,2016 +241815,2016 +227102,2016 +154684,2016 +181446,2016 +141521,2016 +170497,2016 +169599,2016 +221915,2016 +132198,2016 +82235,2017 +180129,2017 +183392,2017 +247316,2017 +221360,2017 +95902,2017 +131921,2017 +147959,2017 +242069,2017 +247312,2017 +248094,2017 +89892,2017 +99963,2017 +80841,2017 +188343,2017 +82235,2018 +131409,2017 +125677,2017 +219241,2017 +243141,2017 +81941,2017 +136308,2017 +234451,2017 +247217,2017 +247311,2017 +161183,2017 +196218,2017 +150045,2017 +208410,2017 +148858,2017 +177871,2017 +196850,2017 +226934,2017 +152705,2017 +132024,2017 +154119,2017 +198094,2017 +164895,2017 +230305,2017 +223363,2018 +247318,2018 +181817,2018 +180129,2018 +243141,2018 +227561,2018 +106920,2018 +193734,2018 +183834,2018 +224521,2018 +236580,2018 +228625,2018 +223994,2018 +238803,2018 +221747,2018 +225866,2018 +230095,2018 +137693,2018 +193746,2018 +208149,2018 +200258,2018 +125496,2018 +191699,2018 +188378,2018 +216179,2018 +216448,2018 +218911,2018 +249402,2018 +248094,2018 +201593,2018 +243205,2018 +247016,2018 +189688,2018 +246031,2018 +213520,2018 +238398,2018 +212927,2018 +225532,2018 +147058,2018 +190995,2018 +176780,2018 +170743,2018 +197329,2018 +240474,2019 +109739,2019 +236396,2019 +244532,2019 +148983,2019 +137649,2019 +192847,2019 +148189,2019 +169846,2019 +110826,2019 +249817,2019 +212124,2019 +82235,2019 +179407,2019 +247990,2019 +214832,2019 +156391,2019 +240680,2019 +258913,2019 +232146,2019 +185474,2019 +141198,2019 +94819,2019 +216437,2019 +207030,2019 +234392,2019 +213771,2019 +4412,2019 +193917,2019 +177168,2020 +225104,2020 +237101,2020 +232622,2020 +186672,2020 +82235,2020 +205254,2020 +94819,2020 +250998,2020 +236344,2020 +187642,2020 +221641,2020 +51747,2020 +190034,2020 +227711,2020 +241491,2020 +144498,2020 +249752,2020 +241192,2020 +214832,2020 +240474,2020 +227822,2020 +251218,2020 +183763,2020 +228694,2020 +251079,2020 +224100,2020 +250906,2020 +56449,2020 +250210,2020 +196299,2020 +34075,2020 +83733,2020 +181006,2020 +214506,2020 +246935,2020 +242070,2020 +147059,2020 +205842,2021 +170083,2021 +11980,2021 +115254,2021 +246683,2021 +228629,2021 +214763,2021 +212495,2021 +213143,2021 +218226,2021 +129947,2021 +254402,2021 +187782,2021 +208597,2021 +82235,2021 +56000,2021 +64989,2021 +111001,2021 +182850,2021 +254484,2021 +249752,2021 +240080,2021 +249033,2021 +210061,2021 +241733,2021 +240680,2021 +201811,2021 +238619,2021 +226682,2021 +248260,2021 +187428,2021 +236079,2021 +144393,2021 +213806,2021 +207104,2021 +206963,2021 +248943,2021 +206188,2021 +14104,2021 +253671,2021 +249104,2021 +253519,2021 +254745,2021 +255251,2021 +192984,2021 +246540,2021 +253889,2021 +254695,2021 +252762,2021 +243601,2021 +227361,2021 +254747,2021 +240075,2021 +229304,2021 +253719,2021 +232364,2021 +155547,2021 +38009,2021 +211967,2021 +36043,2022 +197787,2022 +185619,2022 +91930,2022 +252746,2022 +98016,2022 +223085,2022 +247273,2022 +236344,2022 +150090,2022 +205092,2022 +250965,2022 +184872,2022 +257726,2022 +39718,2022 +82235,2022 +222924,2022 +94819,2022 +231532,2022 +236079,2022 +198496,2022 +158343,2022 +257749,2022 +195719,2022 +256451,2022 +103261,2022 +238967,2022 +257210,2022 +256821,2022 +219582,2016 +111783,2016 +244792,2016 +229938,2016 +231297,2016 +64400,2016 +21005,2016 +198263,2016 +246358,2016 +204828,2016 +150961,2016 +247285,2017 +190034,2017 +219294,2017 +225051,2017 +177934,2017 +171681,2017 +173830,2017 +235561,2017 +50668,2018 +198658,2018 +248049,2018 +248635,2018 +106968,2018 +248270,2018 +217703,2018 +131972,2018 +98042,2019 +88611,2019 +217403,2019 +234387,2019 +189782,2020 +191094,2020 +191946,2020 +190848,2020 +191610,2020 +190163,2020 +183651,2020 +241193,2020 +235850,2020 +170380,2020 +250450,2020 +247318,2020 +189790,2020 +191186,2020 +201820,2020 +191185,2020 +225053,2020 +601,2020 +191010,2020 +255580,2020 +192315,2020 +191949,2020 +188733,2020 +94530,2021 +256745,2021 +247154,2021 +212124,2021 +225036,2021 +255075,2021 +251420,2021 +205821,2021 +193349,2022 +30647,2022 +242128,2022 +211875,2022 +55683,2022 +255075,2022 +193089,2016 +55815,2016 +197885,2017 +65462,2017 +242643,2017 +143139,2017 +251060,2017 +237083,2018 +95902,2018 +186072,2018 +235247,2018 +141202,2019 +104235,2019 +54936,2020 +247863,2020 +190364,2020 +128661,2020 +130520,2020 +117467,2020 +240432,2021 +128661,2021 +159008,2021 +191201,2021 +212267,2021 +240801,2021 +253527,2021 +256874,2021 +253836,2021 +156005,2022 +182241,2022 +148220,2016 +181306,2016 +190267,2016 +1447,2016 +186831,2016 +141352,2016 +233246,2016 +202361,2016 +187405,2016 +97820,2016 +247811,2016 +177312,2017 +2976,2017 +180895,2017 +170265,2017 +111045,2017 +119705,2018 +182848,2018 +180876,2018 +172606,2018 +78466,2018 +251060,2018 +170375,2018 +152320,2018 +221406,2019 +160877,2019 +181306,2019 +176944,2019 +193816,2019 +240544,2019 +54901,2020 +144780,2020 +38009,2020 +3308,2021 +204846,2021 +134330,2021 +247687,2021 +248409,2021 +200435,2022 +142576,2022 +252761,2022 +158089,2018 +203886,2016 +141944,2016 +222159,2016 +176280,2016 +78825,2018 +248053,2017 +195049,2017 +66214,2017 +173029,2018 +61486,2018 +152112,2018 +2565,2019 +177168,2019 +238213,2019 +245783,2019 +142069,2019 +48702,2020 +146898,2021 +211930,2021 +255454,2021 +197714,2021 +179990,2021 +251967,2022 +195307,2022 +133844,2022 +196375,2022 +141793,2016 +238213,2016 +197953,2016 +1445,2017 +148563,2017 +232513,2017 +68497,2017 +148031,2018 +48352,2018 +131885,2018 +177109,2018 +73371,2018 +230189,2018 +220500,2018 +147945,2019 +125375,2019 +204846,2020 +251178,2020 +239085,2020 +141787,2021 +133572,2021 +66346,2022 +230361,2016 +205150,2016 +177093,2017 +205901,2017 +143455,2017 +111566,2018 +73238,2018 +196589,2019 +162361,2019 +176408,2020 +225626,2020 +50668,2022 +85010,2022 +181829,2022 +170804,2016 +185474,2016 +145147,2016 +148279,2017 +192817,2017 +192208,2017 +212629,2017 +193564,2018 +196078,2018 +162361,2018 +111566,2019 +211619,2021 +53337,2021 +114688,2022 +232822,2022 +204449,2022 +245844,2016 +214503,2017 +205150,2017 +117856,2018 +160306,2019 +143473,2019 +111566,2020 +210571,2021 +187415,2021 +176492,2022 +165461,2022 +189881,2017 +174118,2016 +169685,2016 +50864,2017 +177342,2017 +119031,2017 +88948,2017 +50864,2018 +200278,2018 +176434,2018 +222740,2019 +17874,2020 +204005,2020 +191159,2020 +170786,2021 +229054,2021 +250210,2021 +240535,2017 +188579,2021 +204724,2021 +187283,2021 +191788,2022 +134750,2016 +189543,2017 +147758,2017 +132239,2018 +150367,2018 +177018,2021 +200157,2021 +253719,2022 +191786,2022 +240934,2016 +187736,2016 +208308,2017 +189687,2018 +152759,2018 +56585,2019 +243348,2020 +152574,2021 +181054,2022 +8851,2016 +180127,2016 +108096,2017 +177159,2017 +155932,2017 +187661,2020 +207318,2022 +147111,2017 +212631,2018 +171124,2019 +150609,2020 +147758,2016 +158078,2016 +150489,2016 +177400,2019 +249252,2019 +133572,2020 +137797,2021 +186565,2021 +114174,2022 +161163,2016 +111783,2017 +167280,2018 +166842,2018 +166834,2018 +181663,2018 +166662,2021 +116381,2022 +109204,2022 +152112,2017 +187645,2018 +160973,2019 +164362,2020 +147941,2016 +160947,2016 +192054,2021 +100238,2022 +2437,2016 +94539,2018 +166773,2018 +87296,2017 +186567,2018 +155935,2018 +181663,2020 +232299,2022 +132357,2017 +196888,2020 +180812,2022 +147111,2016 +131864,2017 +70455,2021 +187661,2018 +250085,2021 +111758,2016 +189354,2018 +161165,2022 +190888,2021 +130216,2022 +111758,2017 +190848,2022 +195773,2022 +176120,2021 +246937,2016 +186878,2017 +205740,2017 +133864,2019 +194305,2020 +189227,2020 +189671,2022 +191146,2022 +156062,2021 +190163,2022 +161202,2017 +202936,2018 +143898,2016 +186321,2018 +169951,2021 +115793,2016 +171359,2019 +147073,2021 +205740,2016 +75188,2018 +148881,2019 +193596,2021 +125845,2018 +187872,2016 +170795,2021 +182086,2017 +184257,2018 +161057,2017 +161171,2022 +177880,2016 +193137,2021 +198172,2021 +189690,2022 +161080,2016 +155935,2022 +187750,2021 +161140,2016 +161042,2017 +161067,2017 +147964,2019 +135669,2021 +131915,2017 +189799,2020 +161107,2017 +81941,2020 +111110,2020 +161032,2017 +171359,2018 +158089,2021 +129783,2022 +161202,2016 +171359,2020 +181758,2019 +161089,2016 +137875,2021 +166794,2022 +192887,2016 +130213,2017 +132045,2018 +182191,2017 +131826,2017 +187872,2017 +3379,2017 +131873,2016 +65449,2022 +131826,2018 +212942,2022 +181667,2019 +130749,2018 +175887,2020 +181658,2020 +137965,2022 +147725,2021 +171359,2017 +181658,2022 +138467,2017 +134720,2020 +237887,2022 +188372,2022 +181663,2021 +181658,2016 +181658,2021 +181663,2017 +129783,2018 +193062,2020 +193062,2017 +152133,2017 +181663,2016 +171359,2016 +63579,2020 +118321,2016 +147719,2020 +170831,2017 +131804,2017 +211692,2020 +146632,2022 +180900,2022 +35765,2018 +180900,2021 +136829,2016 +15175,2018 +25545,2016 +219554,2019 +141337,2019 +138368,2019 +164682,2019 +206347,2019 +246159,2020 +154249,2020 +144484,2020 +194570,2021 +148708,2021 +239297,2021 +206033,2021 +162957,2022 +242150,2022 +244629,2022 +105561,2019 +192812,2019 +248836,2019 +223588,2019 +194250,2019 +81810,2019 +35712,2019 +135673,2019 +246203,2019 +138410,2019 +211695,2019 +243955,2019 +249687,2019 +208586,2019 +177336,2019 +232413,2019 +186332,2019 +213481,2019 +158768,2019 +233507,2019 +53220,2019 +95223,2019 +245853,2019 +30558,2020 +200553,2020 +249815,2020 +252604,2020 +256695,2020 +171150,2020 +171170,2020 +251053,2020 +186994,2020 +171082,2020 +102688,2020 +251128,2020 +11678,2020 +230397,2020 +136276,2020 +252824,2020 +91688,2020 +161805,2020 +131844,2020 +235987,2020 +196804,2020 +161297,2020 +246449,2020 +25727,2020 +21005,2020 +131978,2020 +119131,2020 +208000,2020 +241554,2020 +249360,2020 +129809,2020 +313,2020 +130204,2020 +55860,2021 +205103,2021 +205541,2021 +196597,2021 +228286,2021 +245506,2021 +241491,2021 +147743,2021 +252104,2021 +48987,2021 +254977,2021 +244891,2021 +250045,2021 +223855,2021 +77427,2021 +253621,2021 +26131,2021 +246542,2021 +255441,2021 +77932,2021 +83716,2021 +249360,2021 +238126,2021 +141966,2021 +147973,2021 +151594,2021 +196207,2021 +220979,2021 +256331,2021 +234022,2021 +210151,2021 +8526,2022 +190316,2022 +249120,2022 +254103,2022 +194335,2022 +185929,2022 +221846,2022 +228259,2022 +248768,2022 +153686,2022 +259060,2022 +233131,2022 +135791,2022 +189772,2022 +147084,2022 +257664,2022 +180929,2022 +252986,2022 +224118,2022 +196283,2022 +72641,2022 +239085,2022 +62590,2022 +258226,2022 +198573,2022 +258011,2022 +243458,2022 +252566,2022 +171944,2018 +904650,2021 +61410,2019 +6348,2019 +64332,2019 +160196,2019 +150003,2019 +146605,2019 +5982,2019 +101232,2019 +248692,2019 +160749,2019 +25511,2019 +101294,2019 +33900,2019 +85330,2019 +166588,2019 +61612,2019 +119271,2019 +151731,2019 +223983,2018 +3008,2017 +187355,2017 +234878,2018 +131594,2017 +38141,2017 +155834,2017 +239057,2017 +243612,2017 +114495,2017 +205148,2018 +165504,2018 +85763,2018 +27643,2018 +164346,2018 +179198,2018 +179551,2018 +170139,2017 +179363,2018 +179361,2018 +179558,2018 +240578,2017 +69041,2018 +233490,2018 +239473,2018 +234743,2017 +66214,2018 +228981,2018 +201672,2018 +201039,2017 +208320,2017 +177754,2017 +181494,2017 +229886,2017 +225001,2017 +123541,2016 +177877,2017 +36101,2016 +136325,2018 +182069,2017 +179272,2018 +188402,2017 +227433,2018 +174516,2017 +179591,2018 +135998,2018 +64655,2016 +238189,2017 +247897,2018 +186612,2018 +227668,2016 +206142,2016 +187566,2016 +54768,2016 +120051,2017 +164737,2018 +120267,2018 +160193,2016 +204858,2017 +177919,2017 +169355,2017 +150048,2017 +49508,2016 +179660,2018 +204843,2017 +153106,2017 +177835,2017 +144069,2017 +57590,2017 +179597,2018 +136699,2017 +229103,2018 +213972,2017 +179265,2018 +179290,2018 +221285,2017 +214225,2018 +132082,2017 +185706,2017 +5865,2018 +247583,2017 +179240,2018 +177356,2017 +220052,2017 +246845,2018 +247856,2018 +26012,2018 +209841,2017 +3659,2017 +179267,2018 +179372,2018 +179369,2018 +164319,2018 +147045,2018 +179271,2017 +243996,2017 +177993,2017 +183829,2017 +215301,2018 +215892,2017 +95993,2018 +177925,2017 +227642,2017 +134944,2017 +91989,2017 +216575,2017 +152086,2017 +154042,2017 +177936,2017 +205770,2018 +165467,2018 +211645,2017 +201383,2017 +111012,2017 +239644,2018 +205036,2018 +179292,2018 +93943,2017 +191791,2017 +191784,2017 +191116,2017 +244303,2017 +164389,2017 +197932,2017 +118330,2018 +244693,2017 +242586,2017 +215956,2017 +227101,2016 +230503,2018 +246207,2017 +222140,2016 +160805,2016 +209740,2018 +227271,2016 +200941,2016 +190761,2017 +219328,2016 +193631,2017 +186639,2017 +202585,2016 +38553,2016 +176357,2016 +95258,2016 +35877,2016 +179663,2016 +186507,2016 +86973,2016 +132247,2016 +236543,2016 +230363,2016 +228496,2016 +200730,2016 +128399,2016 +221796,2016 +186855,2016 +191871,2016 +142030,2016 +179359,2016 +219855,2016 +136162,2016 +96074,2016 +238336,2016 +230371,2016 +166685,2016 +163887,2016 +232779,2016 +192794,2016 +244983,2016 +155756,2016 +230448,2016 +159782,2016 +221331,2016 +191516,2016 +223655,2016 +199628,2016 +173305,2016 +131847,2016 +162489,2016 +18903,2016 +152762,2016 +99406,2016 +174600,2016 +243734,2016 +197454,2016 +193318,2016 +116086,2016 +88676,2016 +237007,2016 +165563,2022 +203473,2022 +212317,2022 +165750,2022 +258918,2022 +256749,2022 +195907,2022 +141688,2022 +142837,2021 +187625,2021 +141946,2022 +189688,2022 +177270,2021 +211616,2019 +165775,2020 +141758,2019 +209984,2019 +165769,2019 +227061,2018 +34158,2021 +228317,2022 +247746,2022 +34158,2022 +15452,2021 +15452,2022 +71648,2022 +71648,2021 +15452,2020 +71648,2020 +30108,2020 +34158,2020 +243753,2019 +15452,2019 +34158,2019 +204888,2022 +244299,2021 +134158,2017 +134054,2017 +137849,2017 +227934,2017 +227471,2017 +189972,2017 +25545,2017 +133877,2017 +15138,2017 +147719,2017 +134025,2017 +133875,2017 +131872,2017 +182630,2017 +134083,2017 +134057,2017 +134155,2017 +72985,2017 +133834,2017 +134105,2017 +133782,2017 +147733,2017 +134164,2017 +134152,2017 +134279,2017 +134045,2017 +202931,2017 +217558,2017 +134295,2017 +133823,2017 +134168,2017 +133862,2017 +237721,2017 +134138,2017 +38560,2018 +188493,2018 +242588,2018 +38560,2017 +188493,2017 +242588,2017 +185892,2018 +227682,2016 +231373,2016 +219160,2016 +154504,2016 +144163,2016 +170065,2016 +151697,2016 +181006,2016 +184335,2016 +246679,2017 +59643,2016 +238797,2016 +142663,2016 +118366,2019 +120776,2019 +238195,2022 +197262,2019 +245528,2019 +214191,2016 +119702,2016 +49357,2016 +151882,2016 +148279,2016 +72956,2017 +234338,2018 +120869,2017 +213708,2017 +246797,2017 +192128,2021 +130478,2018 +186616,2017 +161731,2018 +192484,2017 +228606,2017 +232644,2017 +232566,2017 +185069,2017 +197454,2018 +230630,2017 +192640,2017 +137668,2017 +246591,2017 +152848,2017 +230150,2016 +205572,2021 +173977,2021 +173924,2020 +173977,2020 +110936,2022 +110936,2021 +240046,2020 +205123,2019 +45469,2021 +45469,2022 +250419,2021 +250418,2020 +45469,2020 +250419,2020 +229622,2020 +111152,2020 +111152,2019 +250084,2021 +250084,2022 +111152,2022 +214068,2022 +170159,2022 +141858,2021 +111152,2021 +187745,2020 +141782,2020 +205272,2019 +250084,2020 +172606,2020 +203196,2019 +247784,2022 +225015,2022 +182626,2021 +200207,2021 +199369,2020 +175289,2019 +191782,2019 +43755,2019 +183990,2019 +252699,2020 +221010,2017 +207338,2017 +6075,2017 +133904,2017 +194426,2017 +56843,2017 +29637,2017 +246936,2017 +196273,2017 +70485,2020 +187338,2021 +250418,2021 +173977,2022 +187338,2022 +174341,2021 +110936,2020 +250838,2020 +141477,2021 +228837,2022 +141782,2022 +242213,2022 +242213,2021 +141782,2021 +152638,2016 +182173,2021 +141541,2021 +230565,2021 +230565,2020 +142028,2022 +86892,2021 +183243,2021 +245053,2021 +203580,2022 +37904,2021 +183243,2022 +39290,2022 +211851,2022 +138363,2022 +39295,2022 +43453,2022 +39295,2021 +39026,2022 +39026,2021 +211851,2021 +195481,2021 +43453,2021 +39290,2021 +138363,2021 +251297,2020 +197187,2019 +39290,2020 +248502,2020 +43453,2020 +186039,2020 +36393,2020 +245921,2020 +187723,2020 +39295,2020 +172651,2020 +241523,2020 +241944,2020 +241185,2020 +39026,2020 +11543,2020 +183243,2020 +186039,2021 +37904,2020 +18603,2019 +191692,2019 +247880,2020 +86892,2020 +138363,2020 +211851,2020 +165469,2019 +165467,2019 +246281,2019 +43453,2019 +118672,2019 +241144,2019 +138667,2019 +248503,2020 +30108,2019 +39295,2019 +138363,2019 +211851,2019 +204977,2019 +182458,2019 +183243,2019 +136113,2019 +186039,2019 +110926,2019 +187723,2019 +165468,2019 +229234,2019 +37904,2019 +165488,2019 +200326,2019 +86892,2019 +39026,2019 +216106,2019 +249647,2019 +241523,2019 +203580,2019 +145908,2019 +200191,2020 +215459,2020 +120130,2021 +216160,2021 +138666,2021 +258219,2022 +130575,2022 +131821,2022 +133942,2022 +137050,2022 +144156,2022 +155930,2022 +178024,2022 +182362,2022 +182955,2022 +199759,2022 +199799,2022 +221412,2022 +225564,2022 +232747,2022 +238531,2022 +245252,2022 +250417,2022 +256322,2022 +257491,2022 +258492,2022 +4184,2022 +66058,2022 +7633,2022 +133623,2022 +133922,2022 +133950,2022 +133959,2022 +134435,2022 +136939,2022 +144315,2022 +158000,2022 +158350,2022 +162511,2022 +170749,2022 +171967,2022 +180356,2022 +181243,2022 +182347,2022 +183607,2022 +183955,2022 +184907,2022 +192692,2022 +193190,2022 +198212,2022 +205292,2022 +213657,2022 +226095,2022 +228034,2022 +233660,2022 +243901,2022 +258159,2022 +259083,2022 +48219,2022 +184907,2021 +186200,2021 +187327,2021 +133578,2021 +144156,2021 +158350,2021 +170749,2021 +180879,2021 +197034,2021 +199945,2021 +208670,2021 +246036,2021 +246139,2021 +248482,2021 +248483,2021 +251582,2021 +254332,2021 +255073,2021 +257009,2021 +258544,2021 +7633,2021 +904721,2021 +130575,2021 +131821,2021 +131886,2021 +132580,2021 +128390,2020 +134446,2020 +144156,2020 +144315,2020 +156539,2020 +180356,2020 +184907,2020 +18842,2020 +193962,2020 +195097,2020 +198076,2020 +200095,2020 +211951,2020 +215924,2020 +238203,2020 +239561,2020 +240828,2020 +242384,2020 +243901,2020 +249987,2020 +250219,2020 +250420,2020 +250438,2020 +250488,2020 +250610,2020 +250882,2020 +250927,2020 +251948,2020 +252508,2020 +254501,2020 +130575,2020 +141279,2020 +158537,2020 +174205,2020 +182226,2020 +182362,2020 +191838,2020 +199485,2020 +224063,2020 +228034,2020 +244738,2020 +249855,2020 +250303,2020 +250324,2020 +250440,2020 +250683,2020 +250899,2020 +251245,2020 +252693,2020 +254776,2020 +256729,2020 +66058,2020 +66064,2020 +66070,2020 +7505,2020 +904475,2020 +129783,2019 +132580,2019 +136308,2019 +162507,2019 +131821,2019 +147944,2019 +180337,2019 +182362,2019 +183394,2019 +184907,2019 +188848,2019 +193190,2019 +196436,2019 +197680,2019 +208670,2019 +211951,2019 +223594,2019 +238531,2019 +245252,2019 +249141,2019 +249724,2019 +250001,2019 +251266,2019 +251840,2019 +136939,2021 +144315,2021 +155930,2021 +162181,2021 +182362,2021 +193962,2021 +194914,2021 +199759,2021 +208014,2021 +221412,2021 +221809,2021 +251782,2021 +252508,2021 +252820,2021 +252887,2021 +254173,2021 +254898,2021 +255262,2021 +255833,2021 +256127,2021 +904604,2021 +170831,2019 +185678,2019 +18842,2019 +192079,2019 +201962,2019 +250845,2019 +250927,2019 +251867,2019 +256729,2019 +162181,2020 +162442,2019 +59154,2020 +177867,2019 +183227,2019 +149997,2019 +234848,2019 +189799,2019 +243307,2020 +207609,2021 +211578,2017 +211578,2016 +200598,2022 +169236,2016 +160549,2022 +169236,2018 +160609,2022 +97814,2021 +97814,2019 +160439,2022 +19913,2022 +219415,2020 +250957,2020 +36465,2019 +169236,2017 +19002,2022 +97814,2022 +128545,2021 +231040,2019 +34371,2021 +232307,2022 +212452,2021 +97814,2020 +98002,2017 +61372,2017 +213144,2019 +9900,2019 +9389,2019 +220293,2019 +98002,2018 +255789,2022 +252211,2022 +232700,2022 +179933,2021 +252671,2021 +250347,2020 +227975,2020 +227404,2019 +240535,2018 +231141,2019 +184882,2017 +201754,2017 +97978,2018 +245353,2016 +201754,2016 +256075,2021 +242628,2021 +254662,2022 +237472,2018 +242898,2021 +231040,2021 +231040,2020 +250571,2020 +88591,2020 +98002,2020 +195952,2020 +185776,2019 +246532,2018 +105747,2018 +98002,2019 +226845,2016 +237472,2017 +246658,2017 +219895,2017 +244340,2017 +224186,2016 +175278,2016 +238093,2016 +198009,2016 +203015,2021 +234451,2020 +234451,2019 +96927,2020 +176362,2020 +205298,2018 +186063,2017 +148755,2016 +77803,2021 +78068,2022 +87913,2018 +237832,2020 +126995,2017 +25515,2020 +80052,2018 +87913,2019 +127155,2017 +87913,2016 +2854,2019 +237832,2019 +2854,2021 +2854,2020 +237832,2018 +179684,2021 +77673,2017 +25515,2018 +191719,2021 +150053,2020 +134043,2019 +188008,2021 +201693,2018 +188008,2020 +188008,2018 +188008,2019 +147725,2022 +81328,2021 +133933,2019 +198499,2020 +125011,2020 +181663,2019 +243753,2018 +218330,2017 +195177,2021 +249608,2022 +246694,2017 +68000,2017 +91983,2016 +133993,2019 +87913,2017 +206851,2019 +134208,2021 +133883,2022 +68000,2020 +210107,2016 +118338,2018 +170653,2017 +201083,2016 +5982,2020 +197622,2022 +134200,2021 +5982,2021 +245327,2017 +131879,2021 +68000,2019 +68000,2021 +93398,2018 +93398,2017 +131879,2022 +68000,2022 +120244,2022 +119104,2018 +195408,2016 +25515,2019 +245299,2016 +25515,2021 +15491,2021 +179604,2020 +224465,2018 +42338,2020 +63509,2017 +97826,2016 +237832,2017 +251210,2020 +182165,2022 +171382,2019 +184315,2019 +205745,2021 +181178,2017 +132044,2022 +136772,2019 +249397,2020 +32660,2016 +202,2022 +171424,2019 +181178,2019 +175013,2022 +182926,2020 +180985,2021 +63579,2022 +162412,2019 +154226,2017 +182454,2020 +183764,2019 +250736,2022 +180900,2017 +72985,2019 +138590,2021 +175013,2020 +123466,2019 +136772,2022 +213388,2020 +32660,2018 +171977,2018 +195724,2019 +18081,2021 +32660,2017 +77883,2021 +171421,2019 +196737,2020 +15175,2020 +15175,2021 +92331,2019 +148727,2018 +123694,2019 +219107,2019 +245476,2019 +192928,2021 +32660,2019 +176851,2020 +147719,2018 +6735,2022 +136772,2021 +170808,2019 +69267,2022 +162405,2021 +162405,2020 +248888,2018 +198478,2019 +2131,2019 +194793,2020 +122370,2022 +120172,2022 +120172,2021 +211897,2019 +80598,2020 +181758,2018 +203870,2022 +72948,2020 +193986,2018 +72948,2021 +148729,2018 +36348,2018 +195222,2022 +182396,2018 +162405,2019 +195222,2021 +69267,2021 +194704,2021 +162405,2022 +2961,2020 +136308,2022 +131847,2021 +137921,2021 +161052,2020 +25395,2021 +213388,2022 +187424,2019 +185929,2019 +181661,2019 +233569,2021 +6735,2019 +166795,2019 +158391,2018 +213388,2021 +131807,2022 +166054,2020 +189972,2022 +162414,2022 +45510,2020 +131847,2020 +258457,2022 +6735,2018 +93230,2021 +158380,2018 +65449,2018 +166054,2022 +2961,2021 +155652,2019 +176088,2019 +6735,2021 +181758,2021 +148751,2018 +129783,2020 +42780,2021 +182884,2020 +123694,2020 +78408,2022 +72712,2018 +1465,2022 +188084,2018 +91200,2019 +136063,2022 +176379,2018 +152133,2020 +166019,2021 +145856,2018 +52086,2020 +181667,2022 +194305,2019 +130209,2022 +37861,2022 +258472,2022 +181554,2019 +72105,2019 +179223,2020 +100050,2018 +189201,2018 +18064,2017 +250736,2020 +221192,2018 +8827,2018 +162411,2022 +195537,2021 +124244,2021 +244592,2019 +124244,2022 +78063,2022 +156529,2018 +131871,2020 +80286,2022 +80286,2020 +80286,2021 +136521,2021 +181554,2017 +182074,2021 +233569,2018 +186914,2021 +200449,2018 +33299,2017 +193947,2020 +10724,2019 +258431,2022 +80286,2019 +118676,2022 +212928,2019 +17157,2019 +18565,2022 +187507,2018 +170309,2020 +185800,2020 +147731,2022 +201412,2021 +166477,2022 +132057,2022 +181658,2019 +170804,2019 +182968,2019 +179867,2020 +29193,2018 +147731,2021 +182191,2022 +160989,2019 +193634,2022 +185929,2021 +170831,2020 +187704,2018 +132155,2021 +174986,2021 +100050,2020 +156972,2020 +204769,2017 +17122,2021 +187317,2021 +166399,2021 +185800,2021 +182755,2022 +78063,2021 +187704,2019 +143605,2021 +199484,2018 +85608,2020 +182454,2022 +170263,2017 +141524,2022 +101742,2022 +144797,2021 +182916,2019 +70282,2017 +181427,2022 +152133,2019 +176394,2018 +123466,2021 +160969,2019 +211695,2022 +65853,2019 +179712,2018 +121369,2018 +135840,2021 +166670,2019 +174778,2022 +197909,2022 +220190,2020 +186438,2021 +190313,2022 +146779,2020 +232373,2021 +134126,2019 +176254,2021 +220190,2022 +185406,2020 +134136,2020 +202341,2021 +63212,2016 +209451,2020 +135760,2020 +205106,2021 +133895,2020 +161094,2021 +177931,2017 +171698,2021 +209663,2021 +186474,2016 +245496,2021 +157629,2022 +133870,2021 +135828,2022 +174774,2022 +188181,2021 +197194,2022 +170805,2022 +81346,2020 +157844,2022 +181050,2021 +15322,2022 +181657,2022 +200266,2022 +231939,2022 +243857,2021 +157792,2022 +161067,2020 +161151,2022 +227662,2021 +133911,2022 +232433,2019 +188065,2022 +133911,2020 +169988,2016 +92979,2019 +92979,2018 +161027,2021 +157786,2022 +161177,2022 +92979,2020 +186157,2022 +161042,2022 +161032,2020 +158417,2019 +161018,2020 +161067,2021 +150467,2021 +198496,2020 +161018,2019 +157757,2022 +219566,2022 +160998,2021 +133767,2022 +157852,2022 +247901,2021 +139184,2021 +211897,2021 +211897,2022 +134122,2020 +151697,2017 +151697,2018 +248480,2016 +248480,2018 +164868,2016 +212625,2018 +219129,2018 +227927,2018 +232762,2018 +212625,2017 +227927,2017 +219113,2018 +164868,2017 +219129,2017 +232762,2017 +212625,2016 +219129,2016 +227927,2016 +232762,2016 +223003,2016 +181006,2017 +214191,2018 +243110,2018 +141321,2017 +214191,2017 +223003,2018 +38675,2018 +108177,2018 +125014,2018 +209721,2018 +218813,2018 +209721,2017 +191960,2018 +216877,2017 +202715,2018 +216877,2018 +164912,2017 +249560,2018 +218813,2017 +224092,2018 +202715,2017 +38675,2017 +223003,2017 +151882,2017 +68500,2016 +164912,2016 +202715,2016 +181423,2016 +38675,2016 +213293,2016 +170120,2020 +222514,2020 +206360,2020 +247298,2022 +174545,2019 +245949,2022 +169637,2020 +174271,2022 +166556,2021 +231060,2022 +247713,2022 +188608,2020 +249073,2020 +257557,2022 +166559,2021 +186212,2021 +253539,2020 +207099,2022 +171671,2022 +96959,2019 +182940,2022 +247095,2022 +161152,2022 +133605,2021 +186212,2022 +249737,2019 +257848,2021 +193581,2019 +242983,2020 +166039,2021 +165674,2020 +166480,2022 +223389,2022 +133605,2022 +204031,2022 +197909,2019 +249439,2020 +162437,2022 +146779,2019 +49685,2022 +195929,2021 +249439,2019 +250934,2020 +166761,2021 +133602,2022 +166761,2022 +157547,2021 +166536,2021 +192862,2021 +187723,2021 +166604,2021 +131902,2019 +197909,2020 +238203,2019 +195487,2022 +160983,2020 +161186,2021 +160989,2022 +157850,2022 +249505,2019 +133882,2021 +204417,2022 +161082,2021 +229253,2020 +161093,2019 +157784,2022 +160946,2021 +161160,2020 +161089,2021 +219113,2017 +219113,2016 +202121,2016 +181006,2018 +177452,2017 +217833,2017 +218813,2016 +224092,2017 +221304,2018 +29466,2017 +164882,2016 +234631,2021 +141226,2019 +246870,2022 +232386,2019 +157761,2022 +170349,2019 +170349,2020 +205409,2021 +136080,2019 +249073,2019 +141542,2022 +3830,2022 +206688,2020 +205409,2020 +185443,2021 +245887,2022 +161156,2022 +174996,2020 +177408,2018 +176278,2022 +193043,2016 +144797,2020 +158024,2019 +108096,2020 +45510,2022 +220190,2021 +242168,2018 +224092,2016 +161099,2021 +248803,2022 +232041,2022 +205536,2022 +223716,2022 +230486,2021 +157642,2021 +145507,2019 +242181,2021 +14842,2021 +134269,2021 +231945,2022 +226279,2019 +160498,2021 +54219,2021 +144373,2022 +19530,2021 +177052,2020 +161166,2021 +158396,2018 +161075,2021 +137901,2018 +86512,2022 +196861,2017 +164845,2017 +154077,2019 +162449,2018 +135829,2022 +252761,2021 +250108,2022 +236889,2020 +191833,2019 +133943,2019 +133870,2020 +197291,2021 +175159,2021 +252076,2021 +190779,2022 +133870,2019 +236161,2020 +131876,2016 +188059,2022 +188059,2021 +133827,2022 +133937,2019 +133828,2022 +134142,2022 +133937,2022 +133845,2021 +136123,2019 +241521,2019 +133937,2020 +134077,2021 +188059,2020 +232604,2022 +134144,2020 +176289,2022 +216437,2018 +186480,2022 +186661,2020 +206178,2022 +187886,2018 +185341,2021 +240953,2019 +187338,2019 +134096,2022 +134580,2022 +255765,2022 +193543,2022 +186388,2020 +181086,2022 +134142,2021 +133886,2022 +157646,2021 +133950,2019 +133557,2019 +134586,2022 +134144,2022 +134130,2019 +201641,2020 +134130,2020 +134045,2022 +197291,2020 +134142,2019 +241356,2020 +133943,2022 +133845,2020 +134045,2020 +133882,2020 +133885,2019 +133845,2022 +160178,2022 +182678,2019 +133943,2020 +134139,2020 +156039,2019 +258861,2022 +176362,2018 +244461,2022 +135836,2020 +151898,2021 +1815,2022 +134139,2019 +134431,2020 +134586,2019 +134096,2019 +44083,2021 +133885,2022 +133882,2019 +188059,2019 +133882,2022 +157481,2021 +161129,2022 +2619,2020 +134431,2019 +133950,2020 +157662,2021 +86512,2021 +134431,2022 +141785,2022 +237824,2021 +134132,2021 +133885,2020 +197909,2021 +201528,2021 +242303,2019 +242303,2020 +134104,2021 +145635,2019 +247711,2019 +134048,2021 +134182,2022 +134136,2019 +134182,2020 +134146,2022 +133830,2019 +187036,2021 +188608,2022 +134096,2020 +193698,2018 +247972,2021 +223527,2020 +223527,2021 +201017,2021 +133830,2020 +233179,2019 +157663,2021 +176283,2022 +156590,2022 +157507,2020 +133839,2022 +134092,2020 +144816,2020 +151705,2020 +133911,2019 +258552,2021 +161070,2021 +92997,2022 +244898,2021 +237824,2022 +131874,2021 +146878,2022 +134136,2021 +134136,2022 +134104,2020 +196951,2022 +134142,2020 +134048,2022 +144816,2019 +197291,2022 +230344,2020 +176362,2016 +211695,2021 +230344,2021 +135840,2020 +249505,2022 +158405,2018 +186157,2021 +132225,2021 +205143,2022 +195146,2019 +157472,2021 +207821,2020 +134092,2019 +157670,2022 +134402,2022 +157448,2022 +156753,2021 +134132,2020 +156753,2019 +131874,2022 +91317,2018 +197070,2019 +176362,2017 +200579,2018 +248411,2022 +194244,2017 +182355,2016 +59870,2016 +25950,2022 +214309,2016 +255101,2021 +255101,2022 +125765,2019 +125765,2020 +166849,2017 +166849,2018 +206033,2016 +166851,2018 +147871,2020 +131882,2018 +22967,2022 +2227,2022 +158396,2021 +131886,2017 +134589,2017 +190859,2016 +191092,2016 +190159,2016 +190163,2016 +143452,2016 +184588,2016 +190847,2016 +184590,2016 +194229,2016 +187222,2016 +189685,2016 +189687,2016 +194335,2016 +192109,2016 +188726,2016 +190848,2016 +194314,2016 +191415,2016 +191768,2016 +232308,2016 +192315,2016 +192108,2016 +190979,2016 +154502,2016 +191950,2016 +190247,2016 +191012,2016 +188248,2016 +186065,2016 +191610,2016 +191146,2016 +189117,2016 +189782,2016 +189790,2016 +191409,2016 +190204,2016 +190318,2016 +188733,2016 +202163,2016 +227779,2016 +188727,2016 +192173,2016 +223387,2016 +223747,2016 +143586,2016 +190177,2016 +191949,2016 +211681,2016 +191727,2016 +202938,2016 +190245,2016 +186838,2016 +192677,2016 +190180,2016 +143329,2016 +190995,2016 +188565,2016 +191104,2016 +192675,2016 +143546,2016 +189735,2016 +211158,2016 +188734,2016 +189688,2016 +188340,2016 +191416,2016 +191708,2016 +190978,2016 +190846,2016 +143343,2016 +191090,2016 +191186,2016 +188735,2016 +189732,2016 +189791,2016 +207267,2016 +191213,2016 +190986,2016 +190364,2016 +191159,2016 +191021,2016 +189775,2016 +191621,2016 +191407,2016 +201915,2016 +191951,2016 +227561,2016 +189690,2016 +243869,2016 +189819,2016 +190845,2016 +232619,2016 +193682,2016 +189772,2016 +197685,2016 +201820,2016 +190175,2016 +191152,2016 +192111,2016 +190164,2016 +191412,2016 +190182,2016 +189774,2016 +191881,2016 +94572,2017 +190861,2016 +196899,2016 +191141,2016 +191010,2016 +187492,2016 +191210,2016 +211277,2016 +192796,2016 +192171,2016 +94572,2016 +191145,2016 +233155,2016 +232929,2016 +190989,2016 +191205,2016 +186781,2016 +227614,2016 +232794,2016 +190210,2016 +191102,2016 +94770,2016 +94770,2017 +190983,2016 +212212,2016 +211510,2016 +191948,2016 +191094,2016 +202046,2016 +191724,2016 +190521,2016 +190316,2016 +232299,2016 +190862,2016 +197740,2016 +232617,2016 +197735,2016 +202048,2016 +212220,2016 +232847,2016 +232910,2016 +187688,2016 +232930,2016 +211666,2016 +213650,2016 +188657,2016 +191946,2016 +232792,2016 +211511,2016 +202876,2016 +94572,2018 +191147,2016 +202047,2016 +94770,2018 +184271,2018 +184271,2017 +184271,2016 +211062,2016 +74813,2016 +147919,2016 +200682,2017 +231770,2016 +82017,2016 +245399,2016 +240674,2018 +184725,2017 +184725,2018 +206194,2018 +184722,2018 +184722,2017 +206194,2017 +184723,2018 +184726,2018 +240674,2017 +221412,2017 +184828,2018 +184720,2018 +184723,2017 +184726,2017 +184828,2017 +184720,2017 +234415,2016 +240674,2016 +166670,2017 +221412,2016 +221412,2018 +184725,2016 +246804,2016 +248904,2016 +246205,2016 +184723,2016 +184720,2016 +184722,2016 +184726,2016 +184828,2016 +206194,2016 +224263,2016 +246525,2017 +245657,2016 +11674,2017 +134101,2017 +235343,2017 +191936,2019 +231334,2019 +251231,2019 +82834,2019 +203817,2019 +13836,2018 +184525,2021 +248282,2018 +115801,2021 +32699,2019 +218847,2021 +240506,2021 +195558,2020 +180133,2022 +223772,2021 +223772,2020 +258158,2022 +258074,2021 +180177,2018 +230558,2020 +257745,2022 +904751,2022 +904753,2022 +904739,2022 +904748,2022 +904749,2022 +904752,2022 +257168,2022 +904706,2022 +904750,2022 +904763,2022 +904764,2022 +904776,2022 +239186,2021 +239456,2021 +243104,2021 +75839,2021 +904769,2021 +904774,2021 +149070,2021 +154246,2021 +154247,2021 +215977,2021 +904639,2021 +904644,2021 +255084,2021 +255204,2021 +255900,2021 +255956,2021 +256443,2021 +904573,2021 +904581,2021 +904603,2021 +904605,2021 +904610,2021 +904667,2021 +904676,2021 +242836,2021 +77249,2021 +904686,2021 +904715,2021 +904653,2021 +904654,2021 +904660,2021 +251089,2021 +251650,2021 +252346,2021 +252461,2021 +904647,2021 +904648,2021 +904777,2021 +904779,2021 +235314,2021 +250946,2021 +251025,2021 +251076,2021 +254211,2021 +254414,2021 +254122,2021 +254714,2021 +252358,2020 +252394,2020 +252832,2020 +252834,2020 +252846,2020 +904559,2020 +225532,2020 +220815,2020 +237347,2020 +904551,2020 +904666,2020 +904690,2020 +251235,2020 +252835,2020 +252847,2020 +252893,2020 +252950,2020 +253125,2020 +249245,2020 +249479,2020 +250294,2020 +251197,2020 +904484,2020 +904699,2020 +154277,2020 +154246,2020 +154247,2020 +191445,2020 +904555,2020 +904618,2020 +212403,2020 +244421,2020 +251253,2020 +251299,2020 +251746,2020 +251860,2020 +252203,2020 +904566,2020 +904727,2020 +192416,2020 +904608,2020 +904615,2020 +175190,2019 +193973,2019 +199272,2019 +215660,2019 +242795,2019 +247742,2019 +39290,2019 +97795,2019 +247820,2019 +248449,2019 +248665,2019 +249016,2019 +250537,2019 +250771,2019 +904593,2019 +904594,2019 +904719,2019 +904788,2019 +904451,2019 +904459,2019 +230793,2019 +904471,2019 +904325,2019 +904368,2019 +904445,2019 +904479,2019 +904500,2019 +904512,2019 +235223,2018 +224327,2018 +223089,2018 +197898,2018 +191929,2018 +172153,2018 +249006,2018 +250448,2018 +248685,2018 +248511,2018 +248837,2018 +248198,2018 +248119,2018 +247095,2018 +246668,2018 +242635,2018 +236124,2018 +183892,2018 +216261,2018 +158744,2018 +214227,2018 +216431,2018 +904633,2018 +904137,2018 +904329,2018 +904391,2018 +904433,2018 +904468,2018 +904536,2018 +904580,2018 +904101,2018 +904773,2018 +199802,2017 +160841,2017 +160839,2017 +170077,2017 +140482,2017 +22798,2017 +904393,2017 +904549,2017 +904675,2017 +904088,2017 +214227,2017 +903932,2017 +246862,2017 +246823,2017 +247262,2017 +246762,2017 +247342,2017 +246691,2017 +247743,2017 +246478,2017 +246430,2017 +246272,2017 +244887,2017 +247788,2017 +247208,2017 +248837,2017 +247015,2017 +247658,2016 +245687,2016 +255666,2016 +245797,2016 +243781,2016 +241142,2016 +220881,2016 +199903,2016 +244464,2016 +245891,2016 +246032,2016 +246503,2016 +247345,2016 +182108,2016 +212404,2016 +214227,2016 +62641,2016 +903732,2016 +904350,2016 +203150,2016 +212516,2016 +92172,2018 +200449,2021 +180912,2020 +147633,2020 +229886,2019 +147635,2019 +236473,2020 +200130,2019 +159651,2021 +159651,2020 +195758,2020 +203339,2018 +250336,2020 +192787,2019 +139446,2022 +257528,2022 +97378,2018 +225260,2022 +184025,2022 +33364,2021 +134414,2018 +163998,2018 +191533,2022 +151109,2022 +123976,2022 +40858,2020 +200216,2020 +255001,2022 +216555,2020 +240234,2022 +233859,2021 +232538,2018 +144893,2020 +6196,2017 +195030,2020 +160733,2021 +160462,2018 +222798,2018 +128408,2021 +104819,2022 +201778,2020 +70729,2018 +208597,2020 +19265,2020 +108729,2019 +131402,2022 +188167,2021 +135888,2021 +181503,2020 +233807,2020 +230853,2020 +202067,2021 +241501,2019 +204581,2021 +169234,2020 +246395,2020 +257022,2021 +248547,2022 +227083,2019 +242286,2021 +251249,2020 +237091,2020 +241618,2020 +73966,2022 +24133,2022 +251385,2020 +219756,2021 +43419,2020 +113129,2020 +231180,2021 +221722,2022 +230608,2019 +210294,2019 +131011,2020 +237731,2022 +174480,2021 +245059,2021 +231494,2021 +146766,2022 +195203,2020 +179942,2021 +186894,2020 +110766,2022 +225705,2022 +177909,2021 +181726,2021 +243181,2019 +250712,2022 +236351,2021 +203116,2020 +230697,2021 +170743,2020 +229536,2022 +170118,2020 +136561,2021 +37007,2018 +197793,2020 +234426,2021 +76857,2018 +110982,2021 +247762,2021 +190281,2020 +184556,2018 +251927,2022 +237943,2021 +154954,2021 +154944,2020 +244854,2020 +237943,2020 +215526,2020 +215047,2020 +188745,2020 +24463,2021 +24463,2020 +185706,2020 +202420,2018 +250153,2021 +187680,2021 +164198,2021 +164152,2021 +245397,2020 +42733,2022 +42733,2021 +239520,2021 +164900,2021 +208658,2021 +257447,2021 +254217,2021 +203814,2018 +253902,2021 +208658,2018 +164845,2021 +198160,2021 +164845,2020 +220858,2020 +150536,2020 +211102,2019 +232770,2020 +238919,2020 +26082,2017 +205353,2022 +230738,2022 +373,2021 +169938,2022 +169938,2021 +238459,2021 +211489,2021 +226427,2021 +220696,2022 +148751,2020 +248547,2021 +248547,2018 +248547,2020 +194697,2021 +229628,2020 +195438,2020 +181769,2020 +232265,2022 +228358,2018 +158282,2018 +227168,2018 +147662,2022 +147662,2021 +225245,2021 +184789,2020 +247164,2018 +247572,2018 +227076,2021 +181692,2022 +170586,2021 +170828,2021 +170828,2022 +242889,2021 +248772,2018 +241339,2017 +114257,2020 +252871,2022 +214841,2020 +130587,2019 +196979,2020 +220158,2022 +188759,2020 +179978,2021 +14123,2021 +175391,2022 +250290,2018 +250290,2017 +225466,2019 +114138,2022 +114138,2021 +114138,2020 +252236,2020 +82175,2022 +246769,2021 +105840,2020 +249121,2022 +250085,2020 +181024,2022 +191284,2022 +85483,2020 +242333,2022 +251203,2020 +259056,2021 +60442,2021 +257529,2022 +81410,2022 +256142,2022 +254775,2021 +252793,2021 +152295,2019 +174235,2021 +175289,2018 +255465,2022 +192840,2021 +192840,2020 +75117,2020 +198496,2019 +182926,2019 +138590,2017 +162392,2017 +138590,2018 +193698,2016 +138590,2016 +204066,2022 +259010,2022 +250980,2022 +132184,2022 +132234,2022 +132038,2022 +207269,2022 +149112,2022 +201400,2022 +221748,2022 +186845,2022 +221258,2022 +241217,2022 +132027,2022 +250646,2022 +129089,2022 +257730,2022 +155962,2022 +143705,2022 +258704,2022 +219545,2022 +186571,2022 +235543,2022 +258453,2022 +176503,2022 +258687,2022 +162699,2022 +194518,2022 +199286,2022 +206372,2022 +258071,2022 +257727,2022 +258078,2022 +188000,2022 +208001,2022 +214268,2022 +149236,2022 +234363,2022 +257535,2022 +190659,2022 +258833,2022 +138899,2022 +258076,2022 +258370,2022 +258820,2022 +211400,2022 +246054,2022 +183688,2022 +254036,2022 +240266,2022 +258418,2022 +258461,2022 +131924,2022 +202985,2022 +258557,2022 +250842,2022 +243260,2022 +257199,2022 +190140,2022 +199660,2022 +258898,2022 +198108,2022 +258755,2022 +201487,2022 +250623,2022 +240530,2022 +137066,2022 +211071,2022 +212437,2022 +172852,2022 +221301,2022 +203303,2022 +161576,2022 +205843,2022 +172208,2022 +200715,2022 +151230,2022 +225312,2022 +257315,2022 +257099,2022 +143646,2022 +188783,2022 +258009,2022 +178453,2022 +222579,2022 +175073,2022 +258996,2022 +251006,2022 +137166,2022 +196421,2022 +193815,2022 +202966,2022 +194096,2022 +248634,2022 +259135,2022 +259062,2022 +258455,2022 +218763,2022 +208429,2022 +136270,2022 +205911,2022 +131963,2022 +191922,2022 +258298,2022 +194443,2022 +258585,2022 +219968,2022 +145265,2022 +240994,2022 +248310,2022 +181475,2022 +252603,2022 +251063,2022 +258837,2022 +258360,2022 +259016,2022 +132072,2022 +148743,2022 +191240,2022 +206941,2022 +136926,2022 +259145,2022 +149886,2022 +210920,2022 +193361,2022 +234662,2022 +258646,2022 +208741,2022 +258560,2022 +258208,2022 +195915,2022 +257164,2022 +258561,2022 +250974,2022 +213361,2022 +224358,2022 +236558,2022 +200634,2022 +257925,2022 +241776,2022 +228509,2022 +190278,2022 +132288,2022 +211185,2022 +211455,2022 +234558,2022 +191238,2022 +152819,2022 +201282,2022 +258602,2022 +238979,2022 +259132,2022 +202423,2022 +204362,2022 +132042,2022 +136469,2022 +201545,2022 +129048,2022 +144837,2022 +238710,2022 +179798,2022 +176495,2022 +132179,2022 +250287,2022 +251068,2022 +200061,2022 +251872,2022 +194819,2022 +228680,2022 +195437,2022 +239269,2022 +144931,2022 +228517,2022 +219133,2022 +257233,2022 +204225,2022 +245233,2022 +194371,2022 +257993,2022 +144808,2022 +192631,2022 +139128,2022 +238653,2022 +247680,2022 +256295,2022 +231932,2022 +134801,2022 +213224,2022 +252166,2022 +258818,2022 +166191,2022 +147753,2022 +209819,2022 +217386,2022 +258412,2022 +238604,2022 +250766,2022 +225071,2022 +187139,2022 +188863,2022 +257610,2022 +250128,2022 +252204,2022 +228346,2022 +258543,2022 +198067,2022 +228903,2022 +243574,2022 +149411,2022 +258507,2022 +258583,2022 +211907,2022 +243081,2022 +199893,2022 +258113,2022 +143718,2022 +228361,2022 +216457,2022 +227928,2022 +176507,2022 +192673,2022 +201610,2022 +152817,2022 +257304,2022 +161643,2022 +258982,2022 +194082,2022 +222305,2022 +251069,2022 +250008,2022 +192884,2022 +203336,2022 +258359,2022 +248381,2022 +239869,2022 +258951,2022 +249629,2022 +170426,2022 +252758,2022 +248426,2022 +256467,2022 +212668,2022 +209387,2022 +216737,2022 +257920,2022 +137221,2022 +225092,2022 +250825,2022 +190169,2022 +258075,2022 +230797,2022 +248202,2022 +132168,2022 +258706,2022 +258423,2022 +257641,2022 +187205,2022 +202995,2022 +208871,2022 +250793,2022 +239050,2022 +213495,2022 +258956,2022 +179813,2022 +191840,2022 +256840,2022 +136340,2022 +238725,2022 +244655,2022 +250398,2022 +231240,2022 +225181,2022 +195127,2022 +237027,2022 +162674,2022 +175116,2022 +132025,2022 +250402,2022 +136475,2022 +258130,2022 +241102,2022 +199892,2022 +240729,2022 +258748,2022 +228246,2022 +258321,2022 +258658,2022 +149218,2022 +213558,2022 +202545,2022 +219387,2022 +250953,2022 +145911,2022 +212627,2022 +255986,2022 +244191,2022 +176425,2022 +192240,2022 +250022,2022 +208436,2022 +181477,2022 +214629,2022 +251931,2022 +209917,2022 +258121,2022 +257914,2022 +234296,2022 +231306,2022 +178441,2022 +220403,2022 +152806,2022 +162610,2022 +208032,2022 +258135,2022 +143655,2022 +258488,2022 +191598,2022 +904756,2022 +131916,2017 +171944,2017 \ No newline at end of file diff --git a/backend/tools/setup_cgov_env.sh b/backend/tools/setup_cgov_env.sh index f944100f8b..553a3c636f 100644 --- a/backend/tools/setup_cgov_env.sh +++ b/backend/tools/setup_cgov_env.sh @@ -1,5 +1,7 @@ source tools/util_startup.sh +# Aliases need to be outside of function scope + function setup_cgov_env { set -e @@ -38,5 +40,12 @@ function setup_cgov_env { export NEW_RELIC_HOST="gov-collector.newrelic.com" # https://docs.newrelic.com/docs/apm/agents/python-agent/configuration/python-agent-configuration/#proxy export NEW_RELIC_PROXY_HOST="$https_proxy" + + # For database work + export FAC_DB_URI="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "fac-db" ".[][] | select(.name == \$service_name) | .credentials.uri")" + export FAC_SNAPSHOT_URI="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "fac-snapshot-db" ".[][] | select(.name == \$service_name) | .credentials.uri")" + # https://stackoverflow.com/questions/37072245/check-return-status-of-psql-command-in-unix-shell-scripting + export PSQL_EXE='/home/vcap/deps/0/apt/usr/lib/postgresql/*/bin/psql' + return 0 } diff --git a/backend/tools/setup_local_env.sh b/backend/tools/setup_local_env.sh index 5cf30af2f0..b6d6c0f05f 100644 --- a/backend/tools/setup_local_env.sh +++ b/backend/tools/setup_local_env.sh @@ -1,5 +1,20 @@ source tools/util_startup.sh +function install_local_sling { + curl -LO 'https://github.com/slingdata-io/sling-cli/releases/latest/download/sling_linux_amd64.tar.gz' + gonogo "curl sling" + tar xf sling_linux_amd64.tar.gz + gonogo "tar xf sling" + rm -f sling_linux_amd64.tar.gz + gonogo "rm sling.tar" + chmod +x sling + gonogo "chmod sling" + mv sling /usr/local/bin/sling + gonogo "mv sling" + export SLING_EXE='/usr/local/bin/sling' + return 0 +} + function setup_local_env { if [[ "${ENV}" == "LOCAL" || "${ENV}" == "TESTING" ]]; then @@ -12,6 +27,13 @@ function setup_local_env { # https://min.io/docs/minio/linux/reference/minio-mc/mc-mb.html mc mb --ignore-existing myminio/gsa-fac-private-s3 mc admin user svcacct add --access-key="${AWS_PRIVATE_ACCESS_KEY_ID}" --secret-key="${AWS_PRIVATE_SECRET_ACCESS_KEY}" myminio minioadmin + + # For database work + export FAC_DB_URI=${DATABASE_URL}?sslmode=disable + export FAC_SNAPSHOT_URI=${SNAPSHOT_URL}?sslmode=disable + export PSQL_EXE='psql -v ON_ERROR_STOP=on' + install_local_sling + return 0 fi; } diff --git a/backend/tools/sling_public_tables.sh b/backend/tools/sling_public_tables.sh new file mode 100644 index 0000000000..fc4a6b27b4 --- /dev/null +++ b/backend/tools/sling_public_tables.sh @@ -0,0 +1,9 @@ +source tools/util_startup.sh + +function sling_public_tables () { + # run_sql has its own gonogo built in. + run_sql $FAC_DB_URI 'dissemination/sql' 'sling' 'public_data_v1_0_0' 'teardown.sql' + $SLING_EXE run -r dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml + local result=$? + return $result +} diff --git a/backend/tools/util_startup.sh b/backend/tools/util_startup.sh index ca54205ab9..923f52893e 100644 --- a/backend/tools/util_startup.sh +++ b/backend/tools/util_startup.sh @@ -1,3 +1,14 @@ +declare -a public_api_versions=( + "api_v1_0_3" + "api_v1_1_0" + "public_api_v2_0_0_alpha" + ) + +declare -a admin_api_versions=( + "admin_api_v1_1_0" + "admin_api_v1_1_1" + ) + function startup_log { local tag="$1" local msg="$2" @@ -14,3 +25,49 @@ function gonogo { exit -1 fi } + +function check_table_exists() { + local db_uri="$1" + local dbname="$2" + $PSQL_EXE $db_uri -c "SELECT '$dbname'::regclass" >/dev/null 2>&1 + result=$? + return $result +} + +function run_sql () { + local db_uri="$1" + local base_path="$2" + local location="$3" + local api_version="$4" + local sql_file="$5" + + $PSQL_EXE $db_uri < ${base_path}/${location}/${api_version}/${sql_file} + gonogo "run_sql < ${base_path}/${location}/${api_version}/${sql_file}" +} + +function run_sql_for_public_apis () { + local location="$1" + local sql_file="$2" + local base_path='dissemination/sql' + + for api_version in "${public_api_versions[@]}" + do + if [ -f ${base_path}/${location}/${api_version}/${sql_file} ]; then + # $PSQL_EXE $FAC_DB_URI < ${base_path}/${location}/${api_version}/${sql_file} + run_sql $FAC_DB_URI $base_path $location $api_version $sql_file + else + echo "API FILE NOT FOUND/SKIPPED ${location}/${api_version}/${sql_file}" + fi + done + + # for api_version in "${admin_api_versions[@]}" + # do + # echo "VERSION $api_version" + # if [ -f ${base_path}/${location}/${api_version}/${sql_file} ]; then + # $PSQL_EXE $FAC_DB_URI < ${base_path}/${location}/${api_version}/${sql_file} + # gonogo "psql < ${location}/${api_version}/${sql_file}" + # else + # echo "API FILE NOT FOUND/SKIPPED ${location}/${api_version}/${sql_file}" + # fi + # done +} From 40158bb0617c86803739cf77a19289f597e59067 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Fri, 27 Sep 2024 13:57:47 -0400 Subject: [PATCH 02/89] Updating/cleaning up for testing --- .../api/api_v1_1_0/create_functions.sql | 59 --- .../api/api_v1_1_0/create_schema.sql | 48 --- .../api/api_v1_1_0/create_views.sql | 306 ---------------- .../dissemination/api/api_v1_1_0/teardown.sql | 1 + .../dissemination/sql/api/api_v1_0_3/base.sql | 29 -- .../sql/api/api_v1_0_3/create_functions.sql | 24 -- .../sql/api/api_v1_0_3/create_schema.sql | 52 --- .../sql/api/api_v1_0_3/create_views.sql | 335 ------------------ .../sql/api/api_v1_0_3/drop_schema.sql | 11 - .../sql/api/api_v1_0_3/drop_views.sql | 17 - .../dissemination/sql/api/api_v1_1_0/base.sql | 29 -- .../sql/api/api_v1_1_0/config1.yaml | 43 --- .../sql/api/api_v1_1_0/create_functions.sql | 126 ------- .../sql/api/api_v1_1_0/create_schema.sql | 60 ---- .../sql/api/api_v1_1_0/create_views.sql | 308 ---------------- .../sql/api/api_v1_1_0/db_to_db.yaml | 316 ----------------- .../sql/api/api_v1_1_0/drop_schema.sql | 11 - .../sql/api/api_v1_1_0/drop_views.sql | 17 - .../sql/api/api_v1_1_0/performance.sql | 36 -- .../sql/api/api_v1_1_0/public_data.general | 314 ---------------- .../dissemination/sql/api/api_v1_1_0/sling.md | 41 --- .../sql/api/public_api_v2_0_0_alpha/base.sql | 36 -- .../create_functions.sql | 4 - .../public_api_v2_0_0_alpha/create_schema.sql | 42 --- .../public_api_v2_0_0_alpha/create_views.sql | 182 ---------- .../public_api_v2_0_0_alpha/drop_schema.sql | 12 - .../public_api_v2_0_0_alpha/drop_views.sql | 18 - .../sql/api/public_api_v2_0_0_alpha/setup.sql | 0 .../public_data_v1_0_0.yaml | 2 +- .../sql/sling/public_data_v1_0_0/standup.sql | 6 + backend/tools/setup_cgov_env.sh | 2 +- 31 files changed, 9 insertions(+), 2478 deletions(-) delete mode 100644 backend/dissemination/api/api_v1_1_0/create_functions.sql delete mode 100644 backend/dissemination/api/api_v1_1_0/create_schema.sql delete mode 100644 backend/dissemination/api/api_v1_1_0/create_views.sql create mode 100644 backend/dissemination/api/api_v1_1_0/teardown.sql delete mode 100644 backend/dissemination/sql/api/api_v1_0_3/base.sql delete mode 100644 backend/dissemination/sql/api/api_v1_0_3/create_functions.sql delete mode 100644 backend/dissemination/sql/api/api_v1_0_3/create_schema.sql delete mode 100644 backend/dissemination/sql/api/api_v1_0_3/create_views.sql delete mode 100644 backend/dissemination/sql/api/api_v1_0_3/drop_schema.sql delete mode 100644 backend/dissemination/sql/api/api_v1_0_3/drop_views.sql delete mode 100644 backend/dissemination/sql/api/api_v1_1_0/base.sql delete mode 100644 backend/dissemination/sql/api/api_v1_1_0/config1.yaml delete mode 100644 backend/dissemination/sql/api/api_v1_1_0/create_functions.sql delete mode 100644 backend/dissemination/sql/api/api_v1_1_0/create_schema.sql delete mode 100644 backend/dissemination/sql/api/api_v1_1_0/create_views.sql delete mode 100644 backend/dissemination/sql/api/api_v1_1_0/db_to_db.yaml delete mode 100644 backend/dissemination/sql/api/api_v1_1_0/drop_schema.sql delete mode 100644 backend/dissemination/sql/api/api_v1_1_0/drop_views.sql delete mode 100644 backend/dissemination/sql/api/api_v1_1_0/performance.sql delete mode 100644 backend/dissemination/sql/api/api_v1_1_0/public_data.general delete mode 100644 backend/dissemination/sql/api/api_v1_1_0/sling.md delete mode 100644 backend/dissemination/sql/api/public_api_v2_0_0_alpha/base.sql delete mode 100644 backend/dissemination/sql/api/public_api_v2_0_0_alpha/create_functions.sql delete mode 100644 backend/dissemination/sql/api/public_api_v2_0_0_alpha/create_schema.sql delete mode 100644 backend/dissemination/sql/api/public_api_v2_0_0_alpha/create_views.sql delete mode 100644 backend/dissemination/sql/api/public_api_v2_0_0_alpha/drop_schema.sql delete mode 100644 backend/dissemination/sql/api/public_api_v2_0_0_alpha/drop_views.sql delete mode 100644 backend/dissemination/sql/api/public_api_v2_0_0_alpha/setup.sql diff --git a/backend/dissemination/api/api_v1_1_0/create_functions.sql b/backend/dissemination/api/api_v1_1_0/create_functions.sql deleted file mode 100644 index 248a259cc2..0000000000 --- a/backend/dissemination/api/api_v1_1_0/create_functions.sql +++ /dev/null @@ -1,59 +0,0 @@ --- WARNING --- Under PostgreSQL 12, the functions below work. --- Under PostgreSQL 14, these will break. --- --- Note the differences: --- --- raise info 'Works under PostgreSQL 12'; --- raise info 'request.header.x-magic %', (SELECT current_setting('request.header.x-magic', true)); --- raise info 'request.jwt.claim.expires %', (SELECT current_setting('request.jwt.claim.expires', true)); --- raise info 'Works under PostgreSQL 14'; --- raise info 'request.headers::json->>x-magic %', (SELECT current_setting('request.headers', true)::json->>'x-magic'); --- raise info 'request.jwt.claims::json->expires %', (SELECT current_setting('request.jwt.claims', true)::json->>'expires'); --- --- To quote the work of Dav Pilkey, "remember this now." - - -CREATE OR REPLACE FUNCTION api_v1_1_0_functions.get_header(item text) RETURNS text - AS $get_header$ - declare res text; - begin - SELECT (current_setting('request.headers', true)::json)->>item into res; - return res; - end; -$get_header$ LANGUAGE plpgsql; - -create or replace function api_v1_1_0_functions.get_api_key_uuid() returns TEXT -as $gaku$ -declare uuid text; -begin - select api_v1_1_0_functions.get_header('x-api-user-id') into uuid; - return uuid; -end; -$gaku$ LANGUAGE plpgsql; - -create or replace function api_v1_1_0_functions.has_tribal_data_access() -returns boolean -as $has_tribal_data_access$ -DECLARE - uuid_header UUID; - key_exists boolean; -BEGIN - - SELECT api_v1_1_0_functions.get_api_key_uuid() INTO uuid_header; - SELECT - CASE WHEN EXISTS ( - SELECT key_id - FROM public.dissemination_TribalApiAccessKeyIds taaki - WHERE taaki.key_id = uuid_header::TEXT) - THEN 1::BOOLEAN - ELSE 0::BOOLEAN - END - INTO key_exists; - RAISE INFO 'api_v1_1_0 has_tribal % %', uuid_header, key_exists; - RETURN key_exists; -END; -$has_tribal_data_access$ LANGUAGE plpgsql; - - -NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/api/api_v1_1_0/create_schema.sql b/backend/dissemination/api/api_v1_1_0/create_schema.sql deleted file mode 100644 index 33b9bd4161..0000000000 --- a/backend/dissemination/api/api_v1_1_0/create_schema.sql +++ /dev/null @@ -1,48 +0,0 @@ -begin; - -do -$$ -begin - DROP SCHEMA IF EXISTS api_v1_1_0 CASCADE; - DROP SCHEMA IF EXISTS api_v1_1_0_functions CASCADE; - - if not exists (select schema_name from information_schema.schemata where schema_name = 'api_v1_1_0') then - create schema api_v1_1_0; - create schema api_v1_1_0_functions; - - grant usage on schema api_v1_1_0_functions to api_fac_gov; - - -- Grant access to tables and views - alter default privileges - in schema api_v1_1_0 - grant select - -- this includes views - on tables - to api_fac_gov; - - -- Grant access to sequences, if we have them - grant usage on schema api_v1_1_0 to api_fac_gov; - grant select, usage on all sequences in schema api_v1_1_0 to api_fac_gov; - alter default privileges - in schema api_v1_1_0 - grant select, usage - on sequences - to api_fac_gov; - end if; -end -$$ -; - --- https://postgrest.org/en/stable/references/api/openapi.html --- This is the title (version number) and description (text). -COMMENT ON SCHEMA api_v1_1_0 IS -$$v1.1.0 - -A RESTful API that serves data from the SF-SAC.$$; - - -commit; - -notify pgrst, - 'reload schema'; - diff --git a/backend/dissemination/api/api_v1_1_0/create_views.sql b/backend/dissemination/api/api_v1_1_0/create_views.sql deleted file mode 100644 index f14f58d50e..0000000000 --- a/backend/dissemination/api/api_v1_1_0/create_views.sql +++ /dev/null @@ -1,306 +0,0 @@ -begin; - ---------------------------------------- --- finding_text ---------------------------------------- -create view api_v1_1_0.findings_text as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - ft.finding_ref_number, - ft.contains_chart_or_table, - ft.finding_text - from - dissemination_findingtext ft, - dissemination_general gen - where - ft.report_id = gen.report_id - and - (gen.is_public = true - or (gen.is_public = false and api_v1_1_0_functions.has_tribal_data_access())) - order by ft.id -; - ---------------------------------------- --- additional_ueis ---------------------------------------- -create view api_v1_1_0.additional_ueis as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - uei.additional_uei - from - dissemination_general gen, - dissemination_additionaluei uei - where - gen.report_id = uei.report_id - order by uei.id -; - ---------------------------------------- --- finding ---------------------------------------- -create view api_v1_1_0.findings as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - finding.award_reference, - finding.reference_number, - finding.is_material_weakness, - finding.is_modified_opinion, - finding.is_other_findings, - finding.is_other_matters, - finding.prior_finding_ref_numbers, - finding.is_questioned_costs, - finding.is_repeat_finding, - finding.is_significant_deficiency, - finding.type_requirement - from - dissemination_finding finding, - dissemination_general gen - where - finding.report_id = gen.report_id - order by finding.id -; - ---------------------------------------- --- federal award ---------------------------------------- -create view api_v1_1_0.federal_awards as - select - award.report_id, - gen.auditee_uei, - gen.audit_year, - --- - award.award_reference, - award.federal_agency_prefix, - award.federal_award_extension, - award.additional_award_identification, - award.federal_program_name, - award.amount_expended, - award.cluster_name, - award.other_cluster_name, - award.state_cluster_name, - award.cluster_total, - award.federal_program_total, - award.is_major, - award.is_loan, - award.loan_balance, - award.is_direct, - award.audit_report_type, - award.findings_count, - award.is_passthrough_award, - award.passthrough_amount - from - dissemination_federalaward award, - dissemination_general gen - where - award.report_id = gen.report_id - order by award.id -; - - ---------------------------------------- --- corrective_action_plan ---------------------------------------- -create view api_v1_1_0.corrective_action_plans as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - ct.finding_ref_number, - ct.contains_chart_or_table, - ct.planned_action - from - dissemination_CAPText ct, - dissemination_General gen - where - ct.report_id = gen.report_id - and - (gen.is_public = true - or (gen.is_public = false and api_v1_1_0_functions.has_tribal_data_access())) - order by ct.id -; - ---------------------------------------- --- notes_to_sefa ---------------------------------------- -create view api_v1_1_0.notes_to_sefa as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - note.note_title as title, - note.accounting_policies, - note.is_minimis_rate_used, - note.rate_explained, - note.content, - note.contains_chart_or_table - from - dissemination_general gen, - dissemination_note note - where - note.report_id = gen.report_id - and - (gen.is_public = true - or (gen.is_public = false and api_v1_1_0_functions.has_tribal_data_access())) - order by note.id -; - ---------------------------------------- --- passthrough ---------------------------------------- -create view api_v1_1_0.passthrough as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - pass.award_reference, - pass.passthrough_id, - pass.passthrough_name - from - dissemination_general as gen, - dissemination_passthrough as pass - where - gen.report_id = pass.report_id - order by pass.id -; - - ---------------------------------------- --- general ---------------------------------------- -create view api_v1_1_0.general as - select - -- every table starts with report_id, UEI, and year - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - gen.auditee_certify_name, - gen.auditee_certify_title, - gen.auditee_contact_name, - gen.auditee_email, - gen.auditee_name, - gen.auditee_phone, - gen.auditee_contact_title, - gen.auditee_address_line_1, - gen.auditee_city, - gen.auditee_state, - gen.auditee_ein, - gen.auditee_zip, - -- auditor - gen.auditor_certify_name, - gen.auditor_certify_title, - gen.auditor_phone, - gen.auditor_state, - gen.auditor_city, - gen.auditor_contact_title, - gen.auditor_address_line_1, - gen.auditor_zip, - gen.auditor_country, - gen.auditor_contact_name, - gen.auditor_email, - gen.auditor_firm_name, - gen.auditor_foreign_address, - gen.auditor_ein, - -- agency - gen.cognizant_agency, - gen.oversight_agency, - -- dates - gen.date_created, - gen.ready_for_certification_date, - gen.auditor_certified_date, - gen.auditee_certified_date, - gen.submitted_date, - gen.fac_accepted_date, - gen.fy_end_date, - gen.fy_start_date, - gen.audit_type, - gen.gaap_results, - gen.sp_framework_basis, - gen.is_sp_framework_required, - gen.sp_framework_opinions, - gen.is_going_concern_included, - gen.is_internal_control_deficiency_disclosed, - gen.is_internal_control_material_weakness_disclosed, - gen.is_material_noncompliance_disclosed, - gen.dollar_threshold, - gen.is_low_risk_auditee, - gen.agencies_with_prior_findings, - gen.entity_type, - gen.number_months, - gen.audit_period_covered, - gen.total_amount_expended, - gen.type_audit_code, - gen.is_public, - gen.data_source, - gen.is_aicpa_audit_guide_included, - gen.is_additional_ueis, - CASE EXISTS(SELECT ein.report_id FROM dissemination_additionalein ein WHERE ein.report_id = gen.report_id) - WHEN FALSE THEN 'No' - ELSE 'Yes' - END AS is_multiple_eins, - CASE EXISTS(SELECT aud.report_id FROM dissemination_secondaryauditor aud WHERE aud.report_id = gen.report_id) - WHEN FALSE THEN 'No' - ELSE 'Yes' - END AS is_secondary_auditors - from - dissemination_general gen - order by gen.id -; - ---------------------------------------- --- auditor (secondary auditor) ---------------------------------------- -create view api_v1_1_0.secondary_auditors as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - sa.auditor_ein, - sa.auditor_name, - sa.contact_name, - sa.contact_title, - sa.contact_email, - sa.contact_phone, - sa.address_street, - sa.address_city, - sa.address_state, - sa.address_zipcode - from - dissemination_General gen, - dissemination_SecondaryAuditor sa - where - sa.report_id = gen.report_id - order by sa.id -; - -create view api_v1_1_0.additional_eins as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - ein.additional_ein - from - dissemination_general gen, - dissemination_additionalein ein - where - gen.report_id = ein.report_id - order by ein.id -; - -commit; - -notify pgrst, - 'reload schema'; - diff --git a/backend/dissemination/api/api_v1_1_0/teardown.sql b/backend/dissemination/api/api_v1_1_0/teardown.sql new file mode 100644 index 0000000000..99b1d27eb9 --- /dev/null +++ b/backend/dissemination/api/api_v1_1_0/teardown.sql @@ -0,0 +1 @@ +\i drop_schema.sql diff --git a/backend/dissemination/sql/api/api_v1_0_3/base.sql b/backend/dissemination/sql/api/api_v1_0_3/base.sql deleted file mode 100644 index dedabe0cb7..0000000000 --- a/backend/dissemination/sql/api/api_v1_0_3/base.sql +++ /dev/null @@ -1,29 +0,0 @@ -DO -$do$ -BEGIN - IF EXISTS ( - SELECT FROM pg_catalog.pg_roles - WHERE rolname = 'authenticator') THEN - RAISE NOTICE 'Role "authenticator" already exists. Skipping.'; - ELSE - CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; - END IF; -END -$do$; - -DO -$do$ -BEGIN - IF EXISTS ( - SELECT FROM pg_catalog.pg_roles - WHERE rolname = 'api_fac_gov') THEN - RAISE NOTICE 'Role "api_fac_gov" already exists. Skipping.'; - ELSE - CREATE ROLE api_fac_gov NOLOGIN; - END IF; -END -$do$; - -GRANT api_fac_gov TO authenticator; - -NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/api/api_v1_0_3/create_functions.sql b/backend/dissemination/sql/api/api_v1_0_3/create_functions.sql deleted file mode 100644 index 62d2b400e7..0000000000 --- a/backend/dissemination/sql/api/api_v1_0_3/create_functions.sql +++ /dev/null @@ -1,24 +0,0 @@ --- WARNING --- Under PostgreSQL 12, the functions below work. --- Under PostgreSQL 14, these will break. --- --- Note the differences: --- --- raise info 'Works under PostgreSQL 12'; --- raise info 'request.header.x-magic %', (SELECT current_setting('request.header.x-magic', true)); --- raise info 'request.jwt.claim.expires %', (SELECT current_setting('request.jwt.claim.expires', true)); --- raise info 'Works under PostgreSQL 14'; --- raise info 'request.headers::json->>x-magic %', (SELECT current_setting('request.headers', true)::json->>'x-magic'); --- raise info 'request.jwt.claims::json->expires %', (SELECT current_setting('request.jwt.claims', true)::json->>'expires'); --- --- To quote the work of Dav Pilkey, "remember this now." - --- We don't grant tribal access (yet) -create or replace function api_v1_0_3_functions.has_tribal_data_access() returns boolean -as $has_tribal_data_access$ -BEGIN - RETURN 0::BOOLEAN; -END; -$has_tribal_data_access$ LANGUAGE plpgsql; - -NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/api/api_v1_0_3/create_schema.sql b/backend/dissemination/sql/api/api_v1_0_3/create_schema.sql deleted file mode 100644 index 089e746f2f..0000000000 --- a/backend/dissemination/sql/api/api_v1_0_3/create_schema.sql +++ /dev/null @@ -1,52 +0,0 @@ -begin; - -do -$$ -begin - DROP SCHEMA IF EXISTS api_v1_0_3 CASCADE; - DROP SCHEMA IF EXISTS api_v1_0_3_functions CASCADE; - - if not exists (select schema_name from information_schema.schemata where schema_name = 'api_v1_0_3') then - create schema api_v1_0_3; - create schema api_v1_0_3_functions; - - grant usage on schema api_v1_0_3_functions to api_fac_gov; - - -- Grant access to tables and views - alter default privileges - in schema api_v1_0_3 - grant select - -- this includes views - on tables - to api_fac_gov; - - -- Grant access to sequences, if we have them - grant usage on schema api_v1_0_3 to api_fac_gov; - grant select, usage on all sequences in schema api_v1_0_3 to api_fac_gov; - alter default privileges - in schema api_v1_0_3 - grant select, usage - on sequences - to api_fac_gov; - end if; -end -$$ -; - --- This is the description -COMMENT ON SCHEMA api_v1_0_3 IS - 'The FAC dissemation API version 1.0.3.' -; - --- https://postgrest.org/en/stable/references/api/openapi.html --- This is the title -COMMENT ON SCHEMA api_v1_0_3 IS -$$v1.0.3 - -A RESTful API that serves data from the SF-SAC.$$; - -commit; - -notify pgrst, - 'reload schema'; - diff --git a/backend/dissemination/sql/api/api_v1_0_3/create_views.sql b/backend/dissemination/sql/api/api_v1_0_3/create_views.sql deleted file mode 100644 index 598701bede..0000000000 --- a/backend/dissemination/sql/api/api_v1_0_3/create_views.sql +++ /dev/null @@ -1,335 +0,0 @@ -begin; - ---------------------------------------- --- finding_text ---------------------------------------- -create view api_v1_0_3.findings_text as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - ft.finding_ref_number, - ft.contains_chart_or_table, - ft.finding_text - from - public.dissemination_findingtext ft, - public.dissemination_general gen - where - ft.report_id = gen.report_id - and - (gen.is_public = true - or - (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) - order by ft.id -; - ---------------------------------------- --- additional_ueis ---------------------------------------- -create view api_v1_0_3.additional_ueis as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - uei.additional_uei - from - public.dissemination_general gen, - public.dissemination_additionaluei uei - where - gen.report_id = uei.report_id - and - (gen.is_public = true - or - (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) - order by uei.id -; - ---------------------------------------- --- finding ---------------------------------------- -create view api_v1_0_3.findings as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - finding.award_reference, - finding.reference_number, - finding.is_material_weakness, - finding.is_modified_opinion, - finding.is_other_findings, - finding.is_other_matters, - finding.prior_finding_ref_numbers, - finding.is_questioned_costs, - finding.is_repeat_finding, - finding.is_significant_deficiency, - finding.type_requirement - from - public.dissemination_finding finding, - public.dissemination_general gen - where - finding.report_id = gen.report_id - and - (gen.is_public = true - or - (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) - order by finding.id -; - ---------------------------------------- --- federal award ---------------------------------------- -create view api_v1_0_3.federal_awards as - select - award.report_id, - gen.auditee_uei, - gen.audit_year, - --- - award.award_reference, - award.federal_agency_prefix, - award.federal_award_extension, - award.additional_award_identification, - award.federal_program_name, - award.amount_expended, - award.cluster_name, - award.other_cluster_name, - award.state_cluster_name, - award.cluster_total, - award.federal_program_total, - award.is_major, - award.is_loan, - award.loan_balance, - award.is_direct, - award.audit_report_type, - award.findings_count, - award.is_passthrough_award, - award.passthrough_amount - from - public.dissemination_federalaward award, - public.dissemination_general gen - where - award.report_id = gen.report_id - and - (gen.is_public = true - or - (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) - order by award.id -; - - ---------------------------------------- --- corrective_action_plan ---------------------------------------- -create view api_v1_0_3.corrective_action_plans as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - ct.finding_ref_number, - ct.contains_chart_or_table, - ct.planned_action - from - public.dissemination_CAPText ct, - public.dissemination_General gen - where - ct.report_id = gen.report_id - and - (gen.is_public = true - or - (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) - order by ct.id -; - ---------------------------------------- --- notes_to_sefa ---------------------------------------- -create view api_v1_0_3.notes_to_sefa as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - note.note_title as title, - note.accounting_policies, - note.is_minimis_rate_used, - note.rate_explained, - note.content, - note.contains_chart_or_table - from - public.dissemination_general gen, - public.dissemination_note note - where - note.report_id = gen.report_id - and - (gen.is_public = true - or - (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) - order by note.id -; - ---------------------------------------- --- passthrough ---------------------------------------- -create view api_v1_0_3.passthrough as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - pass.award_reference, - pass.passthrough_id, - pass.passthrough_name - from - public.dissemination_general as gen, - public.dissemination_passthrough as pass - where - gen.report_id = pass.report_id - and - (gen.is_public = true - or - (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) - order by pass.id -; - - ---------------------------------------- --- general ---------------------------------------- -create view api_v1_0_3.general as - select - -- every table starts with report_id, UEI, and year - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - gen.auditee_certify_name, - gen.auditee_certify_title, - gen.auditee_contact_name, - gen.auditee_email, - gen.auditee_name, - gen.auditee_phone, - gen.auditee_contact_title, - gen.auditee_address_line_1, - gen.auditee_city, - gen.auditee_state, - gen.auditee_ein, - gen.auditee_zip, - -- auditor - gen.auditor_phone, - gen.auditor_state, - gen.auditor_city, - gen.auditor_contact_title, - gen.auditor_address_line_1, - gen.auditor_zip, - gen.auditor_country, - gen.auditor_contact_name, - gen.auditor_email, - gen.auditor_firm_name, - gen.auditor_foreign_address, - gen.auditor_ein, - -- agency - gen.cognizant_agency, - gen.oversight_agency, - -- dates - gen.date_created, - gen.ready_for_certification_date, - gen.auditor_certified_date, - gen.auditee_certified_date, - gen.submitted_date, - gen.fac_accepted_date, - gen.fy_end_date, - gen.fy_start_date, - gen.audit_type, - gen.gaap_results, - gen.sp_framework_basis, - gen.is_sp_framework_required, - gen.sp_framework_opinions, - gen.is_going_concern_included, - gen.is_internal_control_deficiency_disclosed, - gen.is_internal_control_material_weakness_disclosed, - gen.is_material_noncompliance_disclosed, - gen.dollar_threshold, - gen.is_low_risk_auditee, - gen.agencies_with_prior_findings, - gen.entity_type, - gen.number_months, - gen.audit_period_covered, - gen.total_amount_expended, - gen.type_audit_code, - gen.is_public, - gen.data_source, - gen.is_aicpa_audit_guide_included, - gen.is_additional_ueis, - CASE EXISTS(SELECT ein.report_id FROM dissemination_additionalein ein WHERE ein.report_id = gen.report_id) - WHEN FALSE THEN 'No' - ELSE 'Yes' - END AS is_multiple_eins, - CASE EXISTS(SELECT aud.report_id FROM dissemination_secondaryauditor aud WHERE aud.report_id = gen.report_id) - WHEN FALSE THEN 'No' - ELSE 'Yes' - END AS is_secondary_auditors - from - public.dissemination_general gen - where - gen.is_public = true - or - (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access()) - order by gen.id -; - ---------------------------------------- --- auditor (secondary auditor) ---------------------------------------- -create view api_v1_0_3.secondary_auditors as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - sa.auditor_ein, - sa.auditor_name, - sa.contact_name, - sa.contact_title, - sa.contact_email, - sa.contact_phone, - sa.address_street, - sa.address_city, - sa.address_state, - sa.address_zipcode - from - public.dissemination_General gen, - public.dissemination_SecondaryAuditor sa - where - sa.report_id = gen.report_id - and - (gen.is_public=True - or - (gen.is_public=false and api_v1_0_3_functions.has_tribal_data_access())) - order by sa.id -; - -create view api_v1_0_3.additional_eins as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - ein.additional_ein - from - public.dissemination_general gen, - public.dissemination_additionalein ein - where - gen.report_id = ein.report_id - and - (gen.is_public = true - or - (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) - order by ein.id -; - -commit; - -notify pgrst, - 'reload schema'; - diff --git a/backend/dissemination/sql/api/api_v1_0_3/drop_schema.sql b/backend/dissemination/sql/api/api_v1_0_3/drop_schema.sql deleted file mode 100644 index cf1aca6d91..0000000000 --- a/backend/dissemination/sql/api/api_v1_0_3/drop_schema.sql +++ /dev/null @@ -1,11 +0,0 @@ - -begin; - -DROP SCHEMA IF EXISTS api_v1_0_3 CASCADE; --- DROP ROLE IF EXISTS authenticator; --- DROP ROLE IF EXISTS api_fac_gov; - -commit; - -notify pgrst, - 'reload schema'; diff --git a/backend/dissemination/sql/api/api_v1_0_3/drop_views.sql b/backend/dissemination/sql/api/api_v1_0_3/drop_views.sql deleted file mode 100644 index 2775087977..0000000000 --- a/backend/dissemination/sql/api/api_v1_0_3/drop_views.sql +++ /dev/null @@ -1,17 +0,0 @@ -begin; - drop table if exists api_v1_0_3.metadata; - drop view if exists api_v1_0_3.general; - drop view if exists api_v1_0_3.auditor; - drop view if exists api_v1_0_3.federal_awards; - drop view if exists api_v1_0_3.findings; - drop view if exists api_v1_0_3.findings_text; - drop view if exists api_v1_0_3.corrective_action_plans; - drop view if exists api_v1_0_3.additional_ueis; - drop view if exists api_v1_0_3.notes_to_sefa; - drop view if exists api_v1_0_3.passthrough; - drop view if exists api_v1_0_3.secondary_auditors; - drop view if exists api_v1_0_3.additional_eins; -commit; - -notify pgrst, - 'reload schema'; diff --git a/backend/dissemination/sql/api/api_v1_1_0/base.sql b/backend/dissemination/sql/api/api_v1_1_0/base.sql deleted file mode 100644 index dedabe0cb7..0000000000 --- a/backend/dissemination/sql/api/api_v1_1_0/base.sql +++ /dev/null @@ -1,29 +0,0 @@ -DO -$do$ -BEGIN - IF EXISTS ( - SELECT FROM pg_catalog.pg_roles - WHERE rolname = 'authenticator') THEN - RAISE NOTICE 'Role "authenticator" already exists. Skipping.'; - ELSE - CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; - END IF; -END -$do$; - -DO -$do$ -BEGIN - IF EXISTS ( - SELECT FROM pg_catalog.pg_roles - WHERE rolname = 'api_fac_gov') THEN - RAISE NOTICE 'Role "api_fac_gov" already exists. Skipping.'; - ELSE - CREATE ROLE api_fac_gov NOLOGIN; - END IF; -END -$do$; - -GRANT api_fac_gov TO authenticator; - -NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/api/api_v1_1_0/config1.yaml b/backend/dissemination/sql/api/api_v1_1_0/config1.yaml deleted file mode 100644 index 09d55d7a06..0000000000 --- a/backend/dissemination/sql/api/api_v1_1_0/config1.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# This uses an env var called PG to -# set the connection string. -source: PG -target: LOCAL - -streams: - - public.dissemination_general.xlsx: - mode: full-refresh - target_options: - format: xlsx - object: file:///tmp/sling/dg.xlsx - sql: | - SELECT * from public.dissemination_general - WHERE is_public = true - - public.dissemination_general.csv: - mode: full-refresh - object: file:///tmp/sling/dg.csv - sql: | - SELECT * from public.dissemination_general - WHERE is_public = true - target_options: - format: csv - - - public.dissemination_federalaward: - mode: full-refresh - object: file:///tmp/sling/dfa/*.csv - target_options: - file_max_rows: 1000 - format: csv - sql: | - SELECT * FROM - public.dissemination_general dg, - public.dissemination_federalaward dfa - WHERE - dfa.report_id = dg.report_id - AND - dg.is_public = true - - - diff --git a/backend/dissemination/sql/api/api_v1_1_0/create_functions.sql b/backend/dissemination/sql/api/api_v1_1_0/create_functions.sql deleted file mode 100644 index b39ac337fb..0000000000 --- a/backend/dissemination/sql/api/api_v1_1_0/create_functions.sql +++ /dev/null @@ -1,126 +0,0 @@ --- WARNING --- Under PostgreSQL 12, the functions below work. --- Under PostgreSQL 14, these will break. --- --- Note the differences: --- --- raise info 'Works under PostgreSQL 12'; --- raise info 'request.header.x-magic %', (SELECT current_setting('request.header.x-magic', true)); --- raise info 'request.jwt.claim.expires %', (SELECT current_setting('request.jwt.claim.expires', true)); --- raise info 'Works under PostgreSQL 14'; --- raise info 'request.headers::json->>x-magic %', (SELECT current_setting('request.headers', true)::json->>'x-magic'); --- raise info 'request.jwt.claims::json->expires %', (SELECT current_setting('request.jwt.claims', true)::json->>'expires'); --- --- To quote the work of Dav Pilkey, "remember this now." - - -CREATE OR REPLACE FUNCTION api_v1_1_0_functions.get_header(item text) RETURNS text - AS $get_header$ - declare res text; - begin - SELECT (current_setting('request.headers', true)::json)->>item into res; - return res; - end; -$get_header$ LANGUAGE plpgsql; - -create or replace function api_v1_1_0_functions.get_api_key_uuid() returns TEXT -as $gaku$ -declare uuid text; -begin - select api_v1_1_0_functions.get_header('x-api-user-id') into uuid; - return uuid; -end; -$gaku$ LANGUAGE plpgsql; - -create or replace function api_v1_1_0_functions.has_tribal_data_access() -returns boolean -as $has_tribal_data_access$ -DECLARE - uuid_header UUID; - key_exists boolean; -BEGIN - - SELECT api_v1_1_0_functions.get_api_key_uuid() INTO uuid_header; - SELECT - CASE WHEN EXISTS ( - SELECT key_id - FROM public.dissemination_TribalApiAccessKeyIds taaki - WHERE taaki.key_id = uuid_header::TEXT) - THEN 1::BOOLEAN - ELSE 0::BOOLEAN - END - INTO key_exists; - RAISE INFO 'api_v1_1_0 has_tribal % %', uuid_header, key_exists; - RETURN key_exists; -END; -$has_tribal_data_access$ LANGUAGE plpgsql; - --- If you change the constant defined by this function, --- you must regenerate the index. -CREATE OR REPLACE FUNCTION api_v1_1_0_functions.batch_size() - RETURNS int - LANGUAGE sql IMMUTABLE PARALLEL SAFE AS -'SELECT 20000'; -GRANT EXECUTE ON FUNCTION api_v1_1_0_functions.batch_size() TO api_fac_gov; - -CREATE OR REPLACE FUNCTION api_v1_1_0_functions.batch (id bigint) -returns bigint -as $batch$ -declare num bigint; -begin - select div(id, api_v1_1_0_functions.batch_size()) into num; - return num; -end; -$batch$ -language plpgsql immutable; -GRANT EXECUTE ON FUNCTION api_v1_1_0_functions.batch(bigint) TO api_fac_gov; - -CREATE OR REPLACE FUNCTION api_v1_1_0.batches (in_table text) -returns integer -as $batches$ -declare count integer; -declare batch_size bigint; -begin - select api_v1_1_0_functions.batch_size() into batch_size; - case - when in_table = 'general' then - select div(count(*), batch_size) into count - from public.dissemination_general; - when in_table = 'federal_awards' then - select div(count(*), batch_size) into count - from public.dissemination_federalaward; - else - count := 0; - end case; - RETURN count; -end; -$batches$ -language plpgsql; -GRANT EXECUTE ON FUNCTION api_v1_1_0.batches(text) TO api_fac_gov; - -CREATE OR REPLACE FUNCTION api_v1_1_0.get_general_batch (batch_no bigint) -returns setof dissemination_general -as $batches$ - select * from public.dissemination_general where api_v1_1_0_functions.batch(id) = batch_no; -$batches$ -language sql; -GRANT EXECUTE ON FUNCTION api_v1_1_0.get_general_batch(bigint) TO api_fac_gov; - -CREATE OR REPLACE FUNCTION api_v1_1_0.get_federal_award_batch (batch_no bigint) -returns setof dissemination_federalaward -as $batches$ - select * from public.dissemination_federalaward where api_v1_1_0_functions.batch(public.dissemination_federalaward.id) = batch_no; -$batches$ -language sql; -GRANT EXECUTE ON FUNCTION api_v1_1_0.get_federal_award_batch(bigint) TO api_fac_gov; - - --- We should consider dropping and regenerating this index --- every night after MV generation. --- drop index batch_by_id_dfa; -create index IF NOT EXISTS batch_by_id_dfa - on public.dissemination_federalaward - using btree(api_v1_1_0_functions.batch(public.dissemination_federalaward.id)); - - -NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/api/api_v1_1_0/create_schema.sql b/backend/dissemination/sql/api/api_v1_1_0/create_schema.sql deleted file mode 100644 index 2d8811dd3a..0000000000 --- a/backend/dissemination/sql/api/api_v1_1_0/create_schema.sql +++ /dev/null @@ -1,60 +0,0 @@ -begin; - -do -$$ -begin - DROP SCHEMA IF EXISTS api_v1_1_0 CASCADE; - DROP SCHEMA IF EXISTS api_v1_1_0_functions CASCADE; - - if not exists (select schema_name from information_schema.schemata where schema_name = 'api_v1_1_0') then - create schema api_v1_1_0; - create schema api_v1_1_0_functions; - - grant usage on schema api_v1_1_0_functions to api_fac_gov; - grant usage on schema api_v1_1_0 to api_fac_gov; - grant select, usage on all sequences in schema api_v1_1_0 to api_fac_gov; - - -- Grant access to tables and views - -- NOTE 20240920: This grants SELECT to all tables - -- created in this schema *in the future*. Nothing - -- that already exists is affected. - -- https://www.postgresql.org/docs/current/sql-alterdefaultprivileges.html - alter default privileges - in schema api_v1_1_0 - grant select - -- this includes views - on tables - to api_fac_gov; - - alter default privileges - in schema api_v1_1_0_functions - grant select - -- this includes views - on tables - to api_fac_gov; - - -- Grant access to future sequences, if we have them - alter default privileges - in schema api_v1_1_0 - grant select, usage - on sequences - to api_fac_gov; - - end if; -end -$$ -; - --- https://postgrest.org/en/stable/references/api/openapi.html --- This is the title (version number) and description (text). -COMMENT ON SCHEMA api_v1_1_0 IS -$$v1.1.0 - -A RESTful API that serves data from the SF-SAC.$$; - - -commit; - -notify pgrst, - 'reload schema'; - diff --git a/backend/dissemination/sql/api/api_v1_1_0/create_views.sql b/backend/dissemination/sql/api/api_v1_1_0/create_views.sql deleted file mode 100644 index ec3fda2336..0000000000 --- a/backend/dissemination/sql/api/api_v1_1_0/create_views.sql +++ /dev/null @@ -1,308 +0,0 @@ -begin; - ---------------------------------------- --- finding_text ---------------------------------------- -create view api_v1_1_0.findings_text as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - ft.finding_ref_number, - ft.contains_chart_or_table, - ft.finding_text - from - dissemination_findingtext ft, - dissemination_general gen - where - ft.report_id = gen.report_id - and - (gen.is_public = true - or (gen.is_public = false and api_v1_1_0_functions.has_tribal_data_access())) - order by ft.id -; - ---------------------------------------- --- additional_ueis ---------------------------------------- -create view api_v1_1_0.additional_ueis as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - uei.additional_uei - from - dissemination_general gen, - dissemination_additionaluei uei - where - gen.report_id = uei.report_id - order by uei.id -; - ---------------------------------------- --- finding ---------------------------------------- -create view api_v1_1_0.findings as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - finding.award_reference, - finding.reference_number, - finding.is_material_weakness, - finding.is_modified_opinion, - finding.is_other_findings, - finding.is_other_matters, - finding.prior_finding_ref_numbers, - finding.is_questioned_costs, - finding.is_repeat_finding, - finding.is_significant_deficiency, - finding.type_requirement - from - dissemination_finding finding, - dissemination_general gen - where - finding.report_id = gen.report_id - order by finding.id -; - ---------------------------------------- --- federal award ---------------------------------------- -create view api_v1_1_0.federal_awards as - select - award.report_id, - gen.auditee_uei, - gen.audit_year, - --- - award.award_reference, - award.federal_agency_prefix, - award.federal_award_extension, - award.additional_award_identification, - award.federal_program_name, - award.amount_expended, - award.cluster_name, - award.other_cluster_name, - award.state_cluster_name, - award.cluster_total, - award.federal_program_total, - award.is_major, - award.is_loan, - award.loan_balance, - award.is_direct, - award.audit_report_type, - award.findings_count, - award.is_passthrough_award, - award.passthrough_amount - from - dissemination_federalaward award, - dissemination_general gen - where - award.report_id = gen.report_id - order by award.id -; - - ---------------------------------------- --- corrective_action_plan ---------------------------------------- -create view api_v1_1_0.corrective_action_plans as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - ct.finding_ref_number, - ct.contains_chart_or_table, - ct.planned_action - from - dissemination_CAPText ct, - dissemination_General gen - where - ct.report_id = gen.report_id - and - (gen.is_public = true - or (gen.is_public = false and api_v1_1_0_functions.has_tribal_data_access())) - order by ct.id -; - ---------------------------------------- --- notes_to_sefa ---------------------------------------- -create view api_v1_1_0.notes_to_sefa as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - note.note_title as title, - note.accounting_policies, - note.is_minimis_rate_used, - note.rate_explained, - note.content, - note.contains_chart_or_table - from - dissemination_general gen, - dissemination_note note - where - note.report_id = gen.report_id - and - (gen.is_public = true - or (gen.is_public = false and api_v1_1_0_functions.has_tribal_data_access())) - order by note.id -; - ---------------------------------------- --- passthrough ---------------------------------------- -create view api_v1_1_0.passthrough as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - pass.award_reference, - pass.passthrough_id, - pass.passthrough_name - from - dissemination_general as gen, - dissemination_passthrough as pass - where - gen.report_id = pass.report_id - order by pass.id -; - - ---------------------------------------- --- general ---------------------------------------- -create view api_v1_1_0.general as - select - -- every table starts with report_id, UEI, and year - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - gen.auditee_certify_name, - gen.auditee_certify_title, - gen.auditee_contact_name, - gen.auditee_email, - gen.auditee_name, - gen.auditee_phone, - gen.auditee_contact_title, - gen.auditee_address_line_1, - gen.auditee_city, - gen.auditee_state, - gen.auditee_ein, - gen.auditee_zip, - -- auditor - gen.auditor_certify_name, - gen.auditor_certify_title, - gen.auditor_phone, - gen.auditor_state, - gen.auditor_city, - gen.auditor_contact_title, - gen.auditor_address_line_1, - gen.auditor_zip, - gen.auditor_country, - gen.auditor_contact_name, - gen.auditor_email, - gen.auditor_firm_name, - gen.auditor_foreign_address, - gen.auditor_ein, - -- agency - gen.cognizant_agency, - gen.oversight_agency, - -- dates - gen.date_created, - gen.ready_for_certification_date, - gen.auditor_certified_date, - gen.auditee_certified_date, - gen.submitted_date, - gen.fac_accepted_date, - gen.fy_end_date, - gen.fy_start_date, - gen.audit_type, - gen.gaap_results, - gen.sp_framework_basis, - gen.is_sp_framework_required, - gen.sp_framework_opinions, - gen.is_going_concern_included, - gen.is_internal_control_deficiency_disclosed, - gen.is_internal_control_material_weakness_disclosed, - gen.is_material_noncompliance_disclosed, - gen.dollar_threshold, - gen.is_low_risk_auditee, - gen.agencies_with_prior_findings, - gen.entity_type, - gen.number_months, - gen.audit_period_covered, - gen.total_amount_expended, - gen.type_audit_code, - gen.is_public, - gen.data_source, - gen.is_aicpa_audit_guide_included, - gen.is_additional_ueis, - CASE EXISTS(SELECT ein.report_id FROM dissemination_additionalein ein WHERE ein.report_id = gen.report_id) - WHEN FALSE THEN 'No' - ELSE 'Yes' - END AS is_multiple_eins, - CASE EXISTS(SELECT aud.report_id FROM dissemination_secondaryauditor aud WHERE aud.report_id = gen.report_id) - WHEN FALSE THEN 'No' - ELSE 'Yes' - END AS is_secondary_auditors - from - dissemination_general gen - order by gen.id -; - ---------------------------------------- --- auditor (secondary auditor) ---------------------------------------- -create view api_v1_1_0.secondary_auditors as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - sa.auditor_ein, - sa.auditor_name, - sa.contact_name, - sa.contact_title, - sa.contact_email, - sa.contact_phone, - sa.address_street, - sa.address_city, - sa.address_state, - sa.address_zipcode - from - dissemination_General gen, - dissemination_SecondaryAuditor sa - where - sa.report_id = gen.report_id - order by sa.id -; - -create view api_v1_1_0.additional_eins as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - ein.additional_ein - from - dissemination_general gen, - dissemination_additionalein ein - where - gen.report_id = ein.report_id - order by ein.id -; - -commit; - - - -notify pgrst, - 'reload schema'; - diff --git a/backend/dissemination/sql/api/api_v1_1_0/db_to_db.yaml b/backend/dissemination/sql/api/api_v1_1_0/db_to_db.yaml deleted file mode 100644 index 71a7cd5157..0000000000 --- a/backend/dissemination/sql/api/api_v1_1_0/db_to_db.yaml +++ /dev/null @@ -1,316 +0,0 @@ -# This uses an env var called PG to -# set the connection string. -source: PG -target: PG -env: - SLING_ALLOW_EMPTY: true - -streams: - # The first table we create is the general table. All of the - # general table is public data, so we copy it wholesale. - public.dissemination_general: - mode: full-refresh - object: public_data.general - sql: | - -- All of the general table is public data. - SELECT - -- every table starts with report_id, UEI, and year - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.agencies_with_prior_findings, - gen.audit_period_covered, - gen.audit_type, - gen.auditee_address_line_1, - gen.auditee_certified_date, - gen.auditee_certify_name, - gen.auditee_certify_title, - gen.auditee_city, - gen.auditee_contact_name, - gen.auditee_contact_title, - gen.auditee_ein, - gen.auditee_email, - gen.auditee_name, - gen.auditee_phone, - gen.auditee_state, - gen.auditee_zip, - gen.auditor_address_line_1, - gen.auditor_certified_date, - gen.auditor_certify_name, - gen.auditor_certify_title, - gen.auditor_city, - gen.auditor_contact_name, - gen.auditor_contact_title, - gen.auditor_country, - gen.auditor_ein, - gen.auditor_email, - gen.auditor_firm_name, - gen.auditor_foreign_address, - gen.auditor_phone, - gen.auditor_state, - gen.auditor_zip, - gen.cognizant_agency, - gen.data_source, - gen.date_created, - gen.dollar_threshold, - gen.entity_type, - gen.fac_accepted_date, - gen.fy_end_date, - gen.fy_start_date, - gen.gaap_results, - gen.is_additional_ueis, - gen.is_aicpa_audit_guide_included, - gen.is_going_concern_included, - gen.is_internal_control_deficiency_disclosed, - gen.is_internal_control_material_weakness_disclosed, - gen.is_low_risk_auditee, - gen.is_material_noncompliance_disclosed, - CASE EXISTS - ( - SELECT - ein.report_id - FROM - dissemination_additionalein ein - WHERE - ein.report_id = gen.report_id - ) - WHEN FALSE THEN 'No' - ELSE 'Yes' - END AS is_multiple_eins, - gen.is_public, - CASE EXISTS - ( - SELECT - aud.report_id - FROM - dissemination_secondaryauditor aud - WHERE - aud.report_id = gen.report_id - ) - WHEN FALSE THEN 'No' - ELSE 'Yes' - END AS is_secondary_auditors, - gen.is_sp_framework_required, - gen.number_months, - gen.oversight_agency, - gen.ready_for_certification_date, - gen.sp_framework_basis, - gen.sp_framework_opinions, - gen.submitted_date, - gen.total_amount_expended, - gen.type_audit_code - FROM - public.dissemination_general gen - ORDER BY gen.id - - # All additional EINs are public. - public.additional_eins: - mode: full-refresh - object: public_data.additional_eins - sql: | - SELECT - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - ein.additional_ein - FROM - public_data.general gen, - public.dissemination_additionalein ein - WHERE - gen.report_id = ein.report_id - ORDER BY ein.id - - # All of the additional UEI info is public info. - public.dissemination_additional_ueis: - mode: full-refresh - object: public_data.additional_ueis - sql: | - SELECT - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - uei.additional_uei - FROM - public_data.general gen, - public.dissemination_additionaluei uei - WHERE - gen.report_id = uei.report_id - ORDER BY uei.id - - # Corrective action plans are NOT always public. - public.dissemination_corrective_action_plan: - mode: full-refresh - object: public_data.corrective_action_plans - sql: | - SELECT - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - ct.contains_chart_or_table, - ct.finding_ref_number, - ct.planned_action - FROM - public.dissemination_captext ct, - public_data.general gen - WHERE - ct.report_id = gen.report_id - AND - -- Only include the public corrective action plans. - gen.is_public = true - ORDER BY ct.id - - # All Federal award data is public. - public.dissemination_federalaward: - mode: full-refresh - object: public_data.federal_awards - sql: | - SELECT - dg.report_id, - dg.auditee_uei, - dg.audit_year, - dg.fac_accepted_date, - --- - award.additional_award_identification, - award.amount_expended, - award.audit_report_type, - award.award_reference, - award.cluster_name, - award.cluster_total, - award.federal_agency_prefix, - award.federal_award_extension, - award.federal_program_name, - award.federal_program_total, - award.findings_count, - award.is_direct, - award.is_loan, - award.is_major, - award.is_passthrough_award, - award.loan_balance, - award.other_cluster_name, - award.passthrough_amount, - award.state_cluster_name - FROM - public.dissemination_federalaward award, - public_data.general dg - WHERE - award.report_id = dg.report_id - ORDER BY award.id - - # The findings table is public. - public.dissemination_findings: - mode: full-refresh - object: public_data.findings - sql: | - SELECT - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - finding.award_reference, - finding.is_material_weakness, - finding.is_modified_opinion, - finding.is_other_findings, - finding.is_other_matters, - finding.is_questioned_costs, - finding.is_repeat_finding, - finding.is_significant_deficiency, - finding.prior_finding_ref_numbers, - finding.reference_number, - finding.type_requirement - FROM - public.dissemination_finding finding, - public_data.general gen - WHERE - finding.report_id = gen.report_id - ORDER BY finding.id - - # The notes to SEFA are NOT all public. - public.dissemination_notes: - mode: full-refresh - object: public_data.notes_to_sefa - sql: | - SELECT - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - note.accounting_policies, - note.contains_chart_or_table, - note.content, - note.is_minimis_rate_used, - note.note_title as title, - note.rate_explained - FROM - public_data.general gen, - public.dissemination_note note - WHERE - note.report_id = gen.report_id - AND - -- Some notes are not public. - gen.is_public = true - ORDER BY note.id - - # All passthrough information is public. - public.dissemination_passthrough: - mode: full-refresh - object: public_data.passthrough - sql: | - SELECT - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - pass.award_reference, - pass.passthrough_id, - pass.passthrough_name - FROM - public_data.general gen, - public.dissemination_passthrough pass - WHERE - gen.report_id = pass.report_id - ORDER BY pass.id - - # All secondary auditor info is public. - public.secondary_auditors: - mode: full-refresh - object: public_data.secondary_auditors - sql: | - SELECT - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - sa.address_city, - sa.address_state, - sa.address_street, - sa.address_zipcode, - sa.auditor_ein, - sa.auditor_name, - sa.contact_email, - sa.contact_name, - sa.contact_phone, - sa.contact_title - FROM - public_data.general gen, - public.dissemination_secondaryauditor sa - WHERE - sa.report_id = gen.report_id - ORDER BY sa.id - - - -# SELECT column_name -# FROM information_schema.columns -# WHERE table_schema = 'public' -# AND table_name = 'dissemination_general' -# AND column_name NOT IN ('id'); diff --git a/backend/dissemination/sql/api/api_v1_1_0/drop_schema.sql b/backend/dissemination/sql/api/api_v1_1_0/drop_schema.sql deleted file mode 100644 index e32038ee46..0000000000 --- a/backend/dissemination/sql/api/api_v1_1_0/drop_schema.sql +++ /dev/null @@ -1,11 +0,0 @@ - -begin; - -DROP SCHEMA IF EXISTS api_v1_1_0 CASCADE; --- DROP ROLE IF EXISTS authenticator; --- DROP ROLE IF EXISTS api_fac_gov; - -commit; - -notify pgrst, - 'reload schema'; diff --git a/backend/dissemination/sql/api/api_v1_1_0/drop_views.sql b/backend/dissemination/sql/api/api_v1_1_0/drop_views.sql deleted file mode 100644 index 877d5920c6..0000000000 --- a/backend/dissemination/sql/api/api_v1_1_0/drop_views.sql +++ /dev/null @@ -1,17 +0,0 @@ -begin; - drop table if exists api_v1_1_0.metadata; - drop view if exists api_v1_1_0.general; - drop view if exists api_v1_1_0.auditor; - drop view if exists api_v1_1_0.federal_awards; - drop view if exists api_v1_1_0.findings; - drop view if exists api_v1_1_0.findings_text; - drop view if exists api_v1_1_0.corrective_action_plans; - drop view if exists api_v1_1_0.additional_ueis; - drop view if exists api_v1_1_0.notes_to_sefa; - drop view if exists api_v1_1_0.passthrough; - drop view if exists api_v1_1_0.secondary_auditors; - drop view if exists api_v1_1_0.additional_eins; -commit; - -notify pgrst, - 'reload schema'; diff --git a/backend/dissemination/sql/api/api_v1_1_0/performance.sql b/backend/dissemination/sql/api/api_v1_1_0/performance.sql deleted file mode 100644 index da21b5ba15..0000000000 --- a/backend/dissemination/sql/api/api_v1_1_0/performance.sql +++ /dev/null @@ -1,36 +0,0 @@ -select count(report_id) from dissemination_general where api_v1_1_0.batch(id) = 2; - -grant select on dissemination_general to api_fac_gov; - - -alter default privileges - in schema api_v1_1_0_functions - grant select --- this includes views -on tables -to api_fac_gov; - -SET SESSION SESSION AUTHORIZATION api_fac_gov; - -SELECT SESSION_USER, CURRENT_USER; -SET SESSION SESSION AUTHORIZATION 'api_fac_gov'; -SELECT SESSION_USER, CURRENT_USER; -RESET SESSION AUTHORIZATION; -SELECT SESSION_USER, CURRENT_USER; - -SET SESSION SESSION AUTHORIZATION 'api_fac_gov'; -select count(report_id) from dissemination_general where api_v1_1_0.batch(id) = 2; - -GRANT SELECT ON public.dissemination_general to api_fac_gov; -GRANT SELECT ON public.dissemination_federalaward to api_fac_gov; - -REVOKE SELECT ON public.dissemination_federalaward from api_fac_gov; -REVOKE SELECT ON public.dissemination_general from api_fac_gov; - -SET SESSION SESSION AUTHORIZATION 'api_fac_gov'; -select api_v1_1_0.get_federal_award_batch(2); - - -------------------- - -SELECT id, report_id FROM public.dissemination_general WHERE id < 3000; diff --git a/backend/dissemination/sql/api/api_v1_1_0/public_data.general b/backend/dissemination/sql/api/api_v1_1_0/public_data.general deleted file mode 100644 index 8a8c8743ca..0000000000 --- a/backend/dissemination/sql/api/api_v1_1_0/public_data.general +++ /dev/null @@ -1,314 +0,0 @@ -id,report_id,auditee_certify_name,auditee_certify_title,auditee_contact_name,auditee_email,auditee_name,auditee_phone,auditee_contact_title,auditee_address_line_1,auditee_city,auditee_state,auditee_ein,auditee_uei,is_additional_ueis,auditee_zip,auditor_phone,auditor_state,auditor_city,auditor_contact_title,auditor_address_line_1,auditor_zip,auditor_country,auditor_contact_name,auditor_email,auditor_firm_name,auditor_foreign_address,auditor_ein,cognizant_agency,oversight_agency,date_created,ready_for_certification_date,auditor_certified_date,auditee_certified_date,submitted_date,fy_end_date,fy_start_date,audit_year,audit_type,gaap_results,sp_framework_basis,is_sp_framework_required,sp_framework_opinions,is_going_concern_included,is_internal_control_deficiency_disclosed,is_internal_control_material_weakness_disclosed,is_material_noncompliance_disclosed,is_aicpa_audit_guide_included,dollar_threshold,is_low_risk_auditee,agencies_with_prior_findings,entity_type,number_months,audit_period_covered,total_amount_expended,type_audit_code,is_public,data_source,fac_accepted_date,auditor_certify_name,auditor_certify_title -56000,2022-06-CENSUS-0000232294,"VISTA AT ENTRADA, SCHOOL OF PERFORMING ARTS AND TECHNOLOGY",FINANCE AND IT DIRECTOR,TROY BRADSHAW,TBRADSHAW@VISTAUTAH.COM,"VISTA AT ENTRADA, SCHOOL OF PERFORMING ARTS AND TECHNOLOGY",4356734110,FINANCE AND IT DIRECTOR,585 E CENTER ST,IVINS,UT,263368224,XXW1B6FYZKU7,No,84738,8012256900,UT,OREM,PARTNER,1329 SOUTH 800 EAST,84097,USA,M. PAUL WINWARD,PAULW@SQUIRE.COM,"SQUIRE & COMPANY, PC",,870343246,,84,2024-01-20,2022-11-02,2022-11-03,2022-11-03,2022-11-02,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"84, 10",local,,annual,1225211,UG,true,CENSUS,2022-11-02,M. PAUL WINWARD,PARTNER -57000,2022-06-CENSUS-0000235102,NORTHEAST EDUCATIONAL SERVICES COOPERATIVE,BUSINESS MANAGER,TIFFANY STORMO,TIFFANY.STORMO@K12.SD.US,NORTHEAST EDUCATIONAL SERVICES COOPERATIVE,6057833607,BUSINESS MANAGER,310 5TH STREET,HAYTI,SD,460360461,J3GGF5B55VP3,No,57241,6059967717,SD,MITCHELL,PARTNER,PO BOX 249,57301,USA,TERRI POST,TERRI.POST@ELOCPA.COM,ELO CPAS & ADVISORS,,460434947,,84,2024-01-20,2023-03-20,2023-04-12,2023-04-11,2023-03-20,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,Yes,84,local,,annual,2537584,UG,true,CENSUS,2023-03-20,TERRI POST,PARTNER -58000,2022-06-CENSUS-0000257833,"NATIONAL ASSOCIATION OF FAMILY DEVELOPMENT CENTERS, INC.",EXECUTIVE DIRECTOR,PAMELA KAPLAN,PAMLKAP3@GMAIL.COM,"NATIONAL ASSOCIATION OF FAMILY DEVELOPMENT CENTERS, INC.",7182587767,EXECUTIVE DIRECTOR,1114 AVENUE J,BROOKLYN,NY,112707362,ZL6RQ6TZJDH7,No,11230,7184681240,NY,QUEENS VILLAGE,OWNER,"221-10 JAMAICA AVENUE, SUITE 207",11428,USA,GRACE NKENKE,GRACECPA@AOL.COM,"GRACE NKENKE, CPA",,331112630,,93,2024-01-20,2023-03-30,2023-03-31,2023-03-31,2023-03-30,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 10, 84",non-profit,,annual,3535134,UG,true,CENSUS,2023-03-30,GRACE NKENKE,OWNER -59000,2022-06-CENSUS-0000231704,ROANOKE CITY PUBLIC SCHOOLS,DIRECTOR OF ACCOUNTING SERVICES,DONNA CALDWELL,DCALDWELL@RCPS.INFO,ROANOKE CITY PUBLIC SCHOOLS,5408532125,DIRECTOR OF ACCOUNTING SERVICES,"40 DOUGLAS AVENUE, N.W.",ROANOKE,VA,546001570,ZU3QCHKBMW13,No,24012,4349489000,VA,ROANOKE,DIRECTOR,3905 ELECTRIC ROAD,24018,USA,SUSAN CHAPMAN,SCHAPMAN@BECPAS.COM,"BROWN, EDWARDS & COMPANY LLP.",,540504608,84,,2024-01-20,2023-01-10,2022-12-16,2023-01-11,2023-01-10,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,1741814,Yes,"84, 10, 12, 93, 21",local,,annual,58060475,UG,true,CENSUS,2023-01-10,SUSAN CHAPMAN,DIRECTOR -60000,2022-06-CENSUS-0000085100,"CATHOLIC CHARITITES OF THE ARCHDIOCESE OF ATLANTA, INC.",CEO,VANESSA RUSSELL,VRUSSELL@CCATLANTA.ORG,"CATHOLIC CHARITITES OF THE ARCHDIOCESE OF ATLANTA, INC.",4049207729,CEO,"2401 LAKE PARK DRIVE, SE",SMYRNA,GA,581097003,SP3QV7HE24K4,No,30080,4042627920,GA,ATLANTA,SHAREHOLDER,"3475 PIEDMONT ROAD, SUITE 1500",30305,USA,LAURA HELLER,LHH@JONESKOLB.COM,"JONES AND KOLB, CPA'S",,581763570,,93,2024-01-20,2023-03-30,2023-03-24,2023-03-29,2023-03-30,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,Yes,"99, 14, 97, 10, 93, 19, 94, 16, 84",non-profit,,annual,2564967,UG,true,CENSUS,2023-03-30,LAURA HELLER,SHAREHOLDER -61000,2021-08-CENSUS-0000174864,AUSTIN COMMUNITY COLLEGE DISTRICT,"EVC, FINANCE AND ADMINISTRATION",NEIL VICKERS,NVICKERS@AUSTINCC.EDU,AUSTIN COMMUNITY COLLEGE DISTRICT,5122231098,"EVC, FINANCE AND ADMINISTRATION",5930 MIDDLE FISKVILLE ROAD,AUSTIN,TX,741742036,GSA_MIGRATION,No,78752,9724489205,TX,DALLAS,PARTNER,"2300 NORTH FIELD STREET, SUITE 1000",75201,USA,DALLAS DUGGER,DALLAS.DUGGER@WEAVER.COM,"WEAVER AND TIDWELL, LLP",,750786316,84,,2024-01-20,2022-01-27,2022-01-28,2022-01-28,2022-01-27,2021-08-31,2020-08-31,2021,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,2889917,No,"11, 84, 47, 10, 93, 17",higher-ed,,annual,96330573,UG,true,CENSUS,2022-01-27,DALLAS DUGGER,PARTNER -62000,2021-06-CENSUS-0000204454,CHAMBERLAIN'S YOUTH SERVICES,CHIEF FINANCIAL OFFICER,RENEE ROCHA,RROCHA@CHAMBERLAINSYOUTH.ORG,CHAMBERLAIN'S YOUTH SERVICES,8316362121,CHIEF FINANCIAL OFFICER,1850 SAN BENITO ST,HOLLISTER,CA,942357401,GSA_MIGRATION,No,95023,6264036801,CA,PASADENA,PARTNER,2698 MATARO STREET,91107,USA,CARLOS A. DAVIS,CARLOS@NPOCPAS.COM,HARRINGTON GROUP,,954557617,,93,2024-01-20,2022-04-07,2022-03-28,2022-04-08,2022-04-07,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"97, 93, 10",non-profit,,annual,1329595,UG,true,CENSUS,2022-04-07,CARLOS A. DAVIS,PARTNER -65000,2021-06-CENSUS-0000251712,"TOWN OF HULETT, WYOMING",MAYOR,TED PARSONS,HULETTMAYOR@GMAIL.COM,"TOWN OF HULETT, WYOMING",3074675771,MAYOR,POBOX 278,HULETT,WY,830211382,GSA_MIGRATION,No,82720,3073335093,WY,CASPER,OWNER,90 VALLEY DR,82604,USA,DEBRA KAY RAUGUTT,DEBRARCPA2016@CHARTER.NET,DEBRA RAUGUTT. LLC,,811317330,,20,2024-01-20,2021-12-12,2021-12-12,2021-12-13,2021-12-12,2021-06-30,2020-06-30,2021,single-audit,not_gaap,cash_basis,No,unmodified_opinion,No,No,No,No,Yes,750000,No,20,local,,annual,1071068,UG,true,CENSUS,2021-12-12,DEBRA KAY RAUGUTT,OWNER -63000,2021-06-CENSUS-0000181365,KENTUCKY HIGHER EDUCATION STUDENT LOAN CORPORATION,DIRECTOR OF INTERNAL AUDIT,RHONDA MANN,RMANN@KHESLC.COM,KENTUCKY HIGHER EDUCATION STUDENT LOAN CORPORATION,5026387435,DIRECTOR OF INTERNAL AUDIT,10180 LINN STATION ROAD,LOUISVILLE,KY,521294450,GSA_MIGRATION,No,402240266,5025851600,KY,LOUISVILLE,MANAGING PARTNER,"1600 WATERFRONT PLAZA, 325 WEST MAIN STREET",40202,USA,WILLIAM MEYER,BMEYER@STROTHMAN.COM,STROTHMAN AND CO.,,611191655,,84,2024-01-20,2021-10-14,2021-10-15,2021-10-15,2021-10-14,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,84,state,,annual,2069309,UG,true,CENSUS,2021-10-14,WILLIAM MEYER,MANAGING PARTNER -64000,2021-12-CENSUS-0000245950,TIMBER LAKE HOUSING CORPORATION,VICE PRESIDENT OF ACCOUNTING,TERRA PAWLICK-MCGIFFIN,TERRA.MCGIFFIN@IMPACTSEVEN.ORG,TIMBER LAKE HOUSING CORPORATION,7154341710,VICE PRESIDENT OF ACCOUNTING,2961 DECKER DR,RICE LAKE,WI,391984458,GSA_MIGRATION,No,54868,8506500125,FL,DESTIN,MANAGING PARTNER,543 HARBOR BLVD 201,32541,USA,CHRISTINE CARTER,CHRISTINE@CARTERCPA.NET,CARTER & COMPANY CPA LLC,,582646754,,14,2024-01-20,2022-03-13,2022-03-13,2022-03-14,2022-03-13,2021-12-31,2020-12-31,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,941026,UG,true,CENSUS,2022-03-13,CHRISTINE CARTER,MANAGING PARTNER -66000,2021-06-CENSUS-0000227760,PINNACLE ACADEMY,AUDIT MANAGER,JANET THATCHER,NHAAUDIT@NHASCHOOLS.COM,PINNACLE ACADEMY,6169291327,AUDIT MANAGER,860 EAST 22ND STREET,EUCLID,OH,201104751,GSA_MIGRATION,No,44123,7402894131,OH,PIKETON,PARTNER,528 SOUTH WEST STREET,45661,USA,BRENDEN BALESTRA,BRENDEN.BALESTRA@BHMCPAGROUP.COM,BHM CPA GROUP,,311413363,,84,2024-01-20,2022-01-31,2022-02-01,2022-02-01,2022-01-31,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"21, 10, 84",local,,annual,1281231,UG,true,CENSUS,2022-01-31,BRENDEN BALESTRA,PARTNER -67000,2021-09-CENSUS-0000234422,JUST 4 KIDS FOOD PROGRAM,EXECUTIVE DIRECTOR,YOLANDA LEE,JUST4KIDSFP@YAHOO.COM,JUST 4 KIDS FOOD PROGRAM,4053879455,EXECUTIVE DIRECTOR,PO BOX 1475,NEWCASTLE,OK,262344185,GSA_MIGRATION,No,73065,5803328548,OK,ADA,MANAGING MEMBER,PO BOX 1406,748211406,USA,G. B. SAUNDERS,GSAUNDERS@SAUNDERSCPAS.COM,"SAUNDERS & ASSOCIATES, PLLC",,208209116,,10,2024-01-20,2022-06-26,2022-06-27,2022-06-27,2022-06-26,2021-09-30,2020-09-30,2021,single-audit,not_gaap,cash_basis,No,unmodified_opinion,No,No,No,No,No,750000,No,10,non-profit,,annual,1076958,UG,true,CENSUS,2022-06-26,G. B. SAUNDERS,MANAGING MEMBER -68000,2021-12-CENSUS-0000010982,THE HEBREW HOME FOR THE AGED AT RIVERDALE,VP OF FINANCE,CARL WILLNER,CARL.WILLNER@RIVERSPRINGHEALTH.ORG,THE HEBREW HOME FOR THE AGED AT RIVERDALE,7185811313,VP OF FINANCE,5901 PALISADE AVENUE,BRONX,NY,131739971,GSA_MIGRATION,No,10471,2128674000,NY,NEW YORK,MANAGING DIRECTOR,1155 AVENUE OF THE AMERICAS,10036,USA,ERIC GOLDFARB,ERIC.GOLDFARB@FORVIS.COM,FORVIS LLP,,440160260,,97,2024-01-20,2022-09-28,2022-09-29,2022-09-29,2022-09-28,2021-12-31,2020-12-31,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"97, 93",non-profit,,annual,15841051,UG,true,CENSUS,2022-09-28,ERIC GOLDFARB,MANAGING DIRECTOR -69000,2021-01-CENSUS-0000029970,FAMILY CARE HEALTH CENTERS,CFO,KEVIN MADDOX,KMADDOX@FCHCSTL.ORG,FAMILY CARE HEALTH CENTERS,3144811615,CFO,401 HOLLY HILLS AVENUE,ST. LOUIS,MO,237076112,GSA_MIGRATION,No,63111,3142315544,MO,ST. LOUIS,MANAGING DIRECTOR,211 N. BROADWAY SUITE 600,63102,USA,BETH KNYSAK,BKNYSAK@BKD.COM,"BKD, LLP",,440160260,,93,2024-01-20,2021-08-30,2021-08-31,2021-08-31,2021-08-30,2021-01-31,2020-02-01,2021,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,Yes,"10, 93, 21",non-profit,,annual,7918734,UG,true,CENSUS,2021-08-30,BETH KNYSAK,MANAGING DIRECTOR -70000,2021-12-CENSUS-0000187458,"HOLLYBROOK HOMES, INC. (HOLLYBROOK HOMES)",MANAGING AGENT,"LYNN GRIFFIN, SR.",LYNNGRIFFINSR@180CARES.COM,"HOLLYBROOK HOMES, INC. (HOLLYBROOK HOMES)",9043891196,MANAGING AGENT,104 KING STREET,JACKSONVILLE,FL,591210087,GSA_MIGRATION,No,32204,2259263360,LA,BATON ROUGE,CPA,"5627 BANKERS AVENUE, BUILDING 2",708082610,USA,JEFF MADDOX,MAIL@MADDOXASSOCIATES.COM,"MADDOX & ASSOCIATES, APC",,721314069,,14,2024-01-20,2022-06-13,2022-06-14,2022-06-13,2022-06-13,2021-12-31,2020-12-31,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,2452646,UG,true,CENSUS,2022-06-13,JEFF MADDOX,CPA -71000,2021-06-CENSUS-0000242842,"COMMUNITY EDUCATION ASSOCIATION, INC",PRINCIPAL,SCOT HOOPER,SHOOPER@ODYSSEYCHARTERSCHOOL.NET,"COMMUNITY EDUCATION ASSOCIATION, INC",7702516111,PRINCIPAL,14 ST. JOHN CIRCLE,NEWNAN,GA,582429902,GSA_MIGRATION,No,30265,7709656455,GA,MARIETTA,MANAGING DIRECTOR,"707 WHITLOCK AVE SW, BLDG B STE 21",30064,USA,BAMBO SONAIKE,BAMBO@CPA-SERVICE.COM,"BAMBO SONAIKE CPA, LLC",,900226617,,84,2024-01-20,2021-08-30,2021-08-30,2021-08-31,2021-08-30,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,84,non-profit,,annual,1958209,UG,true,CENSUS,2021-08-30,BAMBO SONAIKE,MANAGING DIRECTOR -72000,2021-12-CENSUS-0000050324,ALLINA HEALTH SYSTEM,MANAGER RESEARCH GRANTS & FINANCE,MICHAEL FULCHER,MICHAEL.FULCHER@ALLINA.COM,ALLINA HEALTH SYSTEM,3302558060,MANAGER RESEARCH GRANTS & FINANCE,2925 CHICAGO AVE,MINNEAPOLIS,MN,363261413,GSA_MIGRATION,No,554071321,3126654266,MN,MINNEAPOLIS,PARTNER,90 SOUTH 7TH STREET SUITE 4200,55402,USA,JARED SILVER,JSSILVER@KPMG.COM,KPMG LLP,,135565207,93,,2024-01-20,2022-09-21,2022-09-22,2022-09-22,2022-09-21,2021-12-31,2020-12-31,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,3000000,Yes,"97, 93, 12",non-profit,,annual,159557744,UG,true,CENSUS,2022-09-21,JARED SILVER,PARTNER -73000,2021-06-CENSUS-0000058645,ALVERNO COLLEGE,CONTROLLER,JEAN V. NORTON,JEAN.POLCZYNSKI-NORTON@ALVERNO.EDU,ALVERNO COLLEGE,4143826188,CONTROLLER,3401 S 39TH STREET,MILWAUKEE,WI,390806263,GSA_MIGRATION,No,532343922,4147775423,WI,MILWAUKEE,PARTNER,"777 E WISCONSIN AVE, 32ND FLOOR",53202,USA,WENDI UNGER,WENDI.UNGER@BAKERTILLY.COM,"BAKER TILLY US, LLP",,390859910,,84,2024-01-20,2022-03-07,2022-03-08,2022-03-08,2022-03-07,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"21, 47, 84",higher-ed,,annual,26996479,UG,true,CENSUS,2022-03-07,WENDI UNGER,PARTNER -74000,2021-06-CENSUS-0000053634,SHAWNEE HEALTH SERVICE AND DEVELOPMENT CORPORATION,CHIEF FINANCIAL OFFICER,JEFF COOPER,JCOOPER@SHSDC.ORG,SHAWNEE HEALTH SERVICE AND DEVELOPMENT CORPORATION,6189859513,CHIEF FINANCIAL OFFICER,109 CALIFORNIA ST,CARTERVILLE,IL,370966854,GSA_MIGRATION,No,62918,6185291040,IL,MARION,PARTNER,3401 OFFICE PARK DRIVE,62959,USA,MARK DALLAS,MARKD@KEBCPA.COM,"KERBER, ECK & BRAECKEL",,430352985,,93,2024-01-20,2022-03-31,2022-04-01,2022-04-01,2022-03-31,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"21, 93",non-profit,,annual,8851122,UG,true,CENSUS,2022-03-31,MARK DALLAS,PARTNER -75000,2021-06-CENSUS-0000206536,SHOW BUS PUBLIC TRANSPORTATION NFP,DIRECTOR,LAURA DICK,LAURA.SHOWBUSNFP@GMAIL.COM,SHOW BUS PUBLIC TRANSPORTATION NFP,8159458500,DIRECTOR,510 HOSELTON DRIVE,CHENOA,IL,264530305,GSA_MIGRATION,No,61726,8152231095,IL,LASALLE,PRESIDENT,219 BUCKLIN STREET,61301,USA,MICHAEL WALKER,MIKE.WALKER@WALKERACCOUNTINGGROUP.COM,"WALKER ACCOUNTING GROUP, LTD.",,453552783,,20,2024-01-20,2022-04-27,2022-04-28,2022-04-28,2022-04-27,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,20,non-profit,,annual,2714828,UG,true,CENSUS,2022-04-27,MICHAEL WALKER,PRESIDENT -76000,2021-06-CENSUS-0000165493,NOBLE PUBLIC SCHOOLS,SUPERINTENDENT,FRANK SOLOMON,FSOLOMON@NOBLEPS.COM,NOBLE PUBLIC SCHOOLS,4058723452,SUPERINTENDENT,111 S. 4TH ST.,NOBLE,OK,736021050,GSA_MIGRATION,No,73068,9182508838,OK,BROKEN ARROW,CPA,2101 N WILLOW AVE.,74012,USA,KERRY PATTEN,KERRY.PATTEN@PATTENCPA.NET,"KERRY JOHN PATTEN, C.P.A.",,731408273,,84,2024-01-20,2022-03-10,2022-03-11,2022-03-11,2022-03-10,2021-06-30,2020-06-30,2021,single-audit,not_gaap,regulatory_basis,Yes,qualified_opinion,No,No,No,No,No,750000,No,"10, 84",local,,annual,4254210,UG,true,CENSUS,2022-03-10,KERRY PATTEN,CPA -77000,2021-06-CENSUS-0000209555,SCHOOL DISTRICT OF MISHICOT,BUSINESS MANAGER,CHRISTINE THELEN,CTHELEN@MISHICOT.K12.WI.US,SCHOOL DISTRICT OF MISHICOT,9207553159,BUSINESS MANAGER,660 WASHINGTON STREET,MISHICOT,WI,396023446,GSA_MIGRATION,No,54223,9204554305,WI,GREEN BAY,PRINCIPAL,2200 RIVERSIDE DRIVE,54305,USA,"SUSAN PABLE, CPA",SUE.PABLE@CLACONNECT.COM,CLIFTONLARSONALLEN LLP,,410746749,,84,2024-01-20,2021-12-19,2021-12-20,2021-12-20,2021-12-19,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,No,"84, 10, 93",local,,annual,1188691,UG,true,CENSUS,2021-12-19,"SUSAN PABLE, CPA",PRINCIPAL -78000,2021-06-CENSUS-0000231497,WATCHUNG HILLS REGIONAL HIGH SCHOOL DISTRICT,SCHOOL BUSINESS ADMINISTRATOR,"TIMOTHY STYS, CPA, SBA",TSTYS@WHRHS.ORG,WATCHUNG HILLS REGIONAL HIGH SCHOOL DISTRICT,9086474800,SCHOOL BUSINESS ADMINISTRATOR,108 STIRLING RD.,WARREN,NJ,226012324,GSA_MIGRATION,No,07059,2017917100,NJ,FAIR LAWN,PARTNER,17-17 ROUTE 208 NORTH,07410,USA,GARY J. VINCI,GVINCI@LVHCPA.COM,"LERCH, VINCI & HIGGINS, LLP",,223015339,,84,2024-01-20,2022-03-03,2022-03-04,2022-03-04,2022-03-03,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"84, 21",local,,annual,789736,UG,true,CENSUS,2022-03-03,GARY J. VINCI,PARTNER -79000,2021-06-CENSUS-0000246657,WESTERN MICHIGAN UNIVERSITY HOMER STRYKER M.D. SCHOOL OF MEDICINE,"DIRECTOR OF ACCOUNTING, CONTROLLER",AMY SMITCHOLS,AMY.SMITCHOLS@MED.WMICH.EDU,WESTERN MICHIGAN UNIVERSITY HOMER STRYKER M.D. SCHOOL OF MEDICINE,2693374704,"DIRECTOR OF ACCOUNTING, CONTROLLER",1000 OAKLAND DRIVE,KALAMAZOO,MI,454135256,GSA_MIGRATION,No,45202,5137444704,OH,CINCINNATI,PARTNER,"537 E PETE ROSE WAY, SUITE 300",45202,USA,STEVEN BISHOP,PM.SINGLEAUDIT@PLANTEMORAN.COM,"PLANTE & MORAN, PLLC",,381357951,,84,2024-01-20,2022-03-15,2022-03-12,2022-03-16,2022-03-15,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"16, 84, 45, 93",higher-ed,,annual,20824246,UG,true,CENSUS,2022-03-15,STEVEN BISHOP,PARTNER -80000,2021-06-CENSUS-0000166784,LANE COUNCIL OF GOVERNMENTS,SENIOR FINANCIAL ANALYST,DAVID JOYAL,DJOYAL@LCOG.ORG,LANE COUNCIL OF GOVERNMENTS,5426823384,SENIOR FINANCIAL ANALYST,"859 WILLAMETTE ST, STE 500",EUGENE,OR,936014373,GSA_MIGRATION,No,974012910,5413425161,OR,EUGENE,PARTNER,1976 GARDEN AVE,97403,USA,GATLIN HAWKINS,GHAWKINS@ISLERCPA.COM,ISLER CPA,,204723636,,93,2024-01-20,2022-02-21,2022-02-18,2022-02-22,2022-02-21,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"16, 10, 20, 66, 93, 14",local,,annual,4409905,UG,true,CENSUS,2022-02-21,GATLIN HAWKINS,PARTNER -81000,2021-06-CENSUS-0000190395,CITY OF ALAMOGORDO,FINANCE DIRECTOR,EVELYN HUFF,EHUFF@CI.ALAMOGORDO.NM.US,CITY OF ALAMOGORDO,5754394268,FINANCE DIRECTOR,1376 EAST 9TH STREET,ALAMOGORDO,NM,856000099,GSA_MIGRATION,No,88310,5058832727,NM,ALBUQUERQUE,PARTNER,2424 LOUISIANA BLVD NE SUITE 300,87110,USA,"ALAN D. BOWERS, JR.",ABOWERS@CRICPA.COM,"CARR, RIGGS & INGRAM, LLC",,721396621,,97,2024-01-20,2022-04-17,2022-04-15,2022-04-18,2022-04-17,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,No,"95, 94, 97, 20, 14, 93, 21, 16, 45",local,,annual,7793927,UG,true,CENSUS,2022-04-17,"ALAN D. BOWERS, JR.",PARTNER -82000,2021-06-CENSUS-0000151757,INDEPENDENT SCHOOL DISTRICT NO. 200,DIRECTOR OF BUSINESS,JENNIFER SEUBERT,JSEUBERT@HASTINGS.K12.MN.US,INDEPENDENT SCHOOL DISTRICT NO. 200,6514807005,DIRECTOR OF BUSINESS,1000 WEST 11TH STREET,HASTINGS,MN,416000810,GSA_MIGRATION,No,55033,9525450424,MN,MINNEAPOLIS,PRINCIPAL,"5353 WAYZATA BLVD, SUITE #410",55416,USA,AARON NIELSEN,ANIELSEN@MMKR.COM,"MALLOY, MONTAGUE, KARNOWSKI, RADOSEVICH & CO., P.A.",,411690382,,84,2024-01-20,2021-11-15,2021-11-15,2021-11-16,2021-11-15,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 10, 21",local,,annual,4645282,UG,true,CENSUS,2021-11-15,AARON NIELSEN,PRINCIPAL -83000,2021-06-CENSUS-0000135824,"TOWN OF TOLLAND, CONNECTICUT",DIRECTOR OF FINANCE AND RECORDS,LISA A HANCOCK,LHANCOCK@TOLLAND.ORG,"TOWN OF TOLLAND, CONNECTICUT",8608713652,DIRECTOR OF FINANCE AND RECORDS,21 TOLLAND GREEN,TOLLAND,CT,066002109,GSA_MIGRATION,No,06084,8605616824,CT,WEST HARTFORD,PRINICPAL,29 SOUTH MAIN STREET,06107,USA,VANESSA ROSSITTO,VANESSA.ROSSITTO@CLACONNECT.COM,CLIFTONLARSONALLEN LLP,,410746749,,84,2024-01-20,2021-12-28,2021-12-29,2021-12-29,2021-12-28,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"21, 97, 90, 93, 10, 84",local,,annual,1488200,UG,true,CENSUS,2021-12-28,VANESSA ROSSITTO,PRINICPAL -84000,2021-06-CENSUS-0000120093,VISIONS UNLIMITED,FISCAL MANAGER,JOSEPH MATLEN,JMATLEN@VUINC.ORG,VISIONS UNLIMITED,9163940800,FISCAL MANAGER,6833 STOCKTON BLVD,SACRAMENTO,CA,942500214,GSA_MIGRATION,No,95823,9165648727,CA,SACRAMENTO,MANAGING PARTNER,"550 HOWE AVENUE, SUITE 210",95825,USA,INGRID SHEIPLINE,ISHEIPLINE@RICHARDSONCPAS.COM,"RICHARDSON & COMPANY, LLP",,465577902,,93,2024-01-20,2021-09-28,2021-09-29,2021-09-29,2021-09-28,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,93,non-profit,,annual,2080537,UG,true,CENSUS,2021-09-28,INGRID SHEIPLINE,MANAGING PARTNER -85000,2021-06-CENSUS-0000160236,CHAUTAUQUA LAKE CENTRAL SCHOOL DISTRICT,SCHOOL BUSINESS EXECUTIVE,JACKSON GRAHAM,JGRAHAM@CLAKE.ORG,CHAUTAUQUA LAKE CENTRAL SCHOOL DISTRICT,7167535874,SCHOOL BUSINESS EXECUTIVE,100 NORTH ERIE STREET,MAYVILLE,NY,161502023,GSA_MIGRATION,No,14757,7163721620,NY,OLEAN,SHAREHOLDER,"130 SOUTH UNION STREET, SUITE 200",14760,USA,"DAVID V. DITANNA, CPA",DVD@BWBCPA.COM,"BUFFAMANTE WHIPPLE BUTTAFARO, P.C.",,161117932,,84,2024-01-20,2021-10-19,2021-10-20,2021-10-19,2021-10-19,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,Yes,Yes,750000,Yes,"84, 10",local,,annual,1261461,UG,true,CENSUS,2021-10-19,"DAVID V. DITANNA, CPA",SHAREHOLDER -86000,2021-12-CENSUS-0000203566,HOUSING AUTHORITY OF THE CITY OF WISCONSIN RAPIDS,EXECUTIVE DIRECTOR,MARY VANG,MARY@WISRAPHA.ORG,HOUSING AUTHORITY OF THE CITY OF WISCONSIN RAPIDS,7154237288,EXECUTIVE DIRECTOR,2521 10TH STREET S,WISCONSIN RAPIDS,WI,391149728,GSA_MIGRATION,No,54494,6083266456,WI,PRAIRIE DU CHIEN,CPA,516 S MARQUETTE RD,53821,USA,BRAD BERTHIAUME,REBECCASTROSCHEIN@CPACOLLINS.COM,"COLLINS & ASSOCIATES, S.C.",,391769340,,14,2024-01-20,2022-09-27,2022-09-28,2022-09-28,2022-09-27,2021-12-31,2020-12-31,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,local,,annual,1418248,UG,true,CENSUS,2022-09-27,BRAD BERTHIAUME,CPA -87000,2021-06-CENSUS-0000187622,GLENS FALLS CITY SCHOOL DISTRICT,ASSISTANT SUPERINTENDENT FOR BUSINESS,ROBERT D YUSKO JR,RYUSKO@GFSD.ORG,GLENS FALLS CITY SCHOOL DISTRICT,5187921451,ASSISTANT SUPERINTENDENT FOR BUSINESS,15 QUADE STREET,GLENS FALLS,NY,146001855,GSA_MIGRATION,No,12801,5188516650,NY,CLAVERACK,CPA,P.O. BOX 538,12513,USA,SCOTT PREUSSER,SCOTT@RGPREUSSER-CPA.COM,"RAYMOND G. PREUSSER, CPA, P.C.",,141821605,,84,2024-01-20,2022-03-02,2022-03-03,2022-03-03,2022-03-02,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 10",local,,annual,2450731,UG,true,CENSUS,2022-03-02,SCOTT PREUSSER,CPA -88000,2021-06-CENSUS-0000218262,"ST. BONIFACE GARDENS, INC. (HUD PROJECT NO. 066-EE074)",CHIEF FINANCIAL OFFICER,DAVID D'AMICO,DDAMICO@CHSFLA.COM,"ST. BONIFACE GARDENS, INC. (HUD PROJECT NO. 066-EE074)",9544841515,CHIEF FINANCIAL OFFICER,11410N. KENDALL DRIVE,MIAMI,FL,650984174,GSA_MIGRATION,No,33176,3054463177,FL,CORAL GABLES,PDEARMAS@VDTCPA.COM,"255 ALHAMBRA CIRCLE, 560",33134,USA,PEDRO M. DE ARMAS,PDEARMAS@VDTCPA.COM,"VERDEJA, DE ARMAS & TRUJILLO, LLP.",,204989621,,14,2024-01-20,2021-10-18,2021-10-19,2021-10-15,2021-10-18,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,7686821,UG,true,CENSUS,2021-10-18,PEDRO M. DE ARMAS,PDEARMAS@VDTCPA.COM -89000,2021-06-CENSUS-0000004757,"CHILDREN'S SERVICES OF ROXBURY, INC.",CHIEF FINANCIAL OFFICER,EDWARD D'AMATO,EDAMATO@CSROX.ORG,"CHILDREN'S SERVICES OF ROXBURY, INC.",6179899647,CHIEF FINANCIAL OFFICER,520 DUDLEY STREET,ROXBURY,MA,043082352,GSA_MIGRATION,No,02119,9787648966,MA,BOSTON,PARTNER,"4238 WASHINGTON ST, STE 307",02131,USA,RANDALL DAVIS,RDAVIS@RSDCPAS.COM,DAVISKELLY LLP,,463169148,,93,2024-01-20,2022-02-24,2022-02-25,2022-02-25,2022-02-24,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,93,non-profit,,annual,3703284,UG,true,CENSUS,2022-02-24,RANDALL DAVIS,PARTNER -90000,2021-12-CENSUS-0000252961,"NATIONAL COWBOY AND WESTERN HERITAGE MUSEUM, INC.",CHIEF FINANCIAL OFFICER,MICHAEL MYERS,MMYERS@NATIONALCOWBOYMUSEUM.ORG,"NATIONAL COWBOY AND WESTERN HERITAGE MUSEUM, INC.",4054782250,CHIEF FINANCIAL OFFICER,1700 NORTHEAST 63RD STREET,OKLAHOMA CITY,OK,300341029,GSA_MIGRATION,No,73111,4058482020,OK,OKLAHOMA CITY,ASSURANCE PARTNER,"1225 N BROADWAY AVENUE, SUITE 200",73103,USA,BRITTNEY WYCOFF,BWYCOFF@HOGANTAYLOR.COM,HOGANTAYLOR LLP,,731413977,,59,2024-01-20,2022-04-12,2022-04-12,2022-04-13,2022-04-12,2021-12-31,2020-12-31,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"59, 45",non-profit,,annual,1370968,UG,true,CENSUS,2022-04-12,BRITTNEY WYCOFF,ASSURANCE PARTNER -91000,2021-06-CENSUS-0000138427,SUGAR-SALEM JOINT SCHOOL DISTRICT NO 322,BUSINESS MANAGER,BECKY BATES,BBATES@SUGARSALEM.COM,SUGAR-SALEM JOINT SCHOOL DISTRICT NO 322,2083568802,BUSINESS MANAGER,P.O. BOX 150,SUGAR CITY,ID,826000836,GSA_MIGRATION,No,83448,2085299276,ID,IDAHO FALLS,CPA,725 S. WOODRUFF AVE.,83401,USA,SCOTT BOND,SBOND@RUDDCO.COM,"RUDD & COMPANY, PLLC",,820467399,,84,2024-01-20,2021-11-19,2021-11-20,2021-11-19,2021-11-19,2021-06-30,2020-06-30,2021,single-audit,qualified_opinion,,,,No,No,No,No,No,750000,No,"84, 93, 10, 45, 21",local,,annual,2541333,UG,true,CENSUS,2021-11-19,SCOTT BOND,CPA -92000,2022-06-CENSUS-0000093828,NASHVILLE CARES,CEO,AMNA OSMAN,AOSMAN@NASHVILLECARES.ORG,NASHVILLE CARES,6152594866,CEO,633 THOMPSON LANE,NASHVILLE,TN,621274532,QCDWWBKW26D8,No,37204,6152427351,TN,NASHVILLE,MEMBER,555 GREAT CIRCLE ROAD,37228,USA,KENNETH YOUNSTEAD,KYOUNGSTEAD@KRAFTCPAS.COM,KRAFTCPAS PLLC,,620713250,,93,2024-01-20,2023-03-30,2023-03-30,2023-03-30,2023-03-30,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,Yes,"14, 93",non-profit,,annual,5915672,UG,true,CENSUS,2023-03-30,KENNETH YOUNSTEAD,MEMBER -93000,2021-06-CENSUS-0000141779,BETHALTO COMMUNITY UNIT SCHOOL DISTRICT NO. 8,BOOKKEEPER,HEATHER PINTER,HPINTER@BETHALTO.ORG,BETHALTO COMMUNITY UNIT SCHOOL DISTRICT NO. 8,6183777200,BOOKKEEPER,101 SCHOOL ST,BETHALTO,IL,376006341,GSA_MIGRATION,No,62010,6184654999,IL,ALTON,PRINCIPAL,1904 STATE ST,62002,USA,DENNIS ROSE,DROSECPA@DRA-CPA.COM,"DENNIS ROSE & ASSOCIATES, P.C.",,371375191,,84,2024-01-20,2022-01-24,2022-01-25,2022-01-25,2022-01-24,2021-06-30,2020-06-30,2021,single-audit,not_gaap,regulatory_basis,Yes,qualified_opinion,No,No,No,No,No,750000,No,"93, 84, 10",local,,annual,2757038,UG,true,CENSUS,2022-01-24,DENNIS ROSE,PRINCIPAL -94000,2021-03-CENSUS-0000194595,HOUSING AUTHORITY OF THE TOWN OF YORK,EXECUTIVE DIRECTOR,RENEE PRINGLE,YORK1802@BELLSOUTH.NET,HOUSING AUTHORITY OF THE TOWN OF YORK,2053925071,EXECUTIVE DIRECTOR,P.O. BOX 9,YORK,AL,636005425,GSA_MIGRATION,No,36925,2059820992,AL,BIRMINGHAM,MEMBER,"200 CHASE PARK SOUTH, SUITE 220",35244,USA,ROY W. HENDERSON JR.,RHENDERSON@HENDERSONCPA.COM,"HENDERSON & PILLETERI, LLC",,263957106,,14,2024-01-20,2021-12-27,2021-12-28,2021-12-28,2021-12-27,2021-03-31,2020-03-31,2021,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,No,14,local,,annual,887057,UG,true,CENSUS,2021-12-27,ROY W. HENDERSON JR.,MEMBER -95000,2021-09-CENSUS-0000258327,NUVANCE HEALTH AND SUBSIDIARIES,"AVP, ACCOUNTING AND STRATEGIC PROJECTS",DEVIN MURPHY,DEVIN.MURPHY@NUVANCEHEALTH.ORG,NUVANCE HEALTH AND SUBSIDIARIES,4019323469,"AVP, ACCOUNTING AND STRATEGIC PROJECTS",100 RESERVE ROAD,DANBURY,CT,834214573,GSA_MIGRATION,No,06810,2127739592,NY,NEW YORK,PARTNER,ONE MANHATTAN WEST,100018604,USA,JOSEPH MORELLO,JOE.MORELLO@EY.COM,ERNST & YOUNG LLP,,346565596,93,,2024-01-20,2023-08-03,2023-08-04,2023-08-04,2023-08-03,2021-09-30,2020-09-30,2021,single-audit,unmodified_opinion,,,,No,Yes,Yes,No,No,3000000,No,"93, 16, 32, 21",non-profit,,annual,137318932,UG,true,CENSUS,2023-08-03,JOSEPH MORELLO,PARTNER -96000,2021-06-CENSUS-0000233469,WEST MARSHALL COMMUNITY SCHOOL DISTRICT,BUSINESS MANAGER,STEPHANIE EDLER,SEDLER@WMCSD.ORG,WEST MARSHALL COMMUNITY SCHOOL DISTRICT,6414832660,BUSINESS MANAGER,"P.O. BOX 670, 601 3RD ST NW",STATE CENTER,IA,420883206,GSA_MIGRATION,No,502471011,6416722541,IA,OSKALOOSA,CERTIFIED PUBLIC ACCOUNTANT,1201 HIGH AVE W,525771937,USA,MICHAEL STANLEY,MSTANLEY@HUNTANDASSOCIATESCPA.COM,"HUNT & ASSOCIATES, PC",,421395096,,10,2024-01-20,2022-02-13,2022-02-11,2022-02-14,2022-02-13,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"93, 84, 10",local,,annual,888006,UG,true,CENSUS,2022-02-13,MICHAEL STANLEY,CERTIFIED PUBLIC ACCOUNTANT -97000,2021-04-CENSUS-0000250304,CITY OF LAHARPE,MAYOR,KATHERINE HASTEN-REED,HASTENWARD3@GMAIL.COM,CITY OF LAHARPE,2176593918,MAYOR,207 E. MAIN ST. P.O. BOX 471,LAHARPE,IL,376000988,GSA_MIGRATION,No,61450,2172247500,IL,QUINCY,PARTNER,901 HAMPSHIRE ST.,62301,USA,ROBERT GRAY,RGRAY@ABNGPC.COM,"ARNOLD, BEHRENS, NESBIT, GRAY, P.C.",,371144423,,10,2024-01-20,2022-04-24,2022-04-25,2022-04-21,2022-04-24,2021-04-30,2020-04-30,2021,single-audit,"adverse_opinion, not_gaap",cash_basis,No,unmodified_opinion,No,No,No,No,Yes,750000,No,"21, 10",local,,annual,1001124,UG,true,CENSUS,2022-04-24,ROBERT GRAY,PARTNER -98000,2021-06-CENSUS-0000134337,LIVE OAK SCHOOL DISTRICT,CHIEF BUSINESS OFFICIAL,ALISON WARNER,AWARNER@LOSD.CA,LIVE OAK SCHOOL DISTRICT,8314756333,CHIEF BUSINESS OFFICIAL,984-I BOSTWICK LANE,SANTA CRUZ,CA,946002632,GSA_MIGRATION,No,95062,6502236104,CA,MENLO PARK,PARTNER,"4040 CAMPBELL AVE., STE. 200",940251053,USA,JOYCE PETERS,JPETERS@EIDEBAILLY.COM,EIDE BAILLY LLP,,450250958,,10,2024-01-20,2022-02-28,2022-02-23,2022-03-01,2022-02-28,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"21, 93, 84, 10",local,,annual,4512861,UG,true,CENSUS,2022-02-28,JOYCE PETERS,PARTNER -99000,2021-06-CENSUS-0000150361,WEST SHORE COMMUNITY COLLEGE,CHIEF FINANCIAL OFFICER,CONNY BAX,CBAX@WESTSHORE.EDU,WEST SHORE COMMUNITY COLLEGE,2318456211,CHIEF FINANCIAL OFFICER,300 NORTH STILES ROAD,SCOTTSVILLE,MI,381852447,GSA_MIGRATION,No,49454,6169754100,MI,GRAND RAPIDS,PRINCIPAL,2330 EAST PARIS AVENUE SE,49546,USA,"MICHELLE FOWLER, CPA",MICHELLE.FOWLER@REHMANN.COM,REHMANN ROBSON LLC,,383635706,,84,2024-01-20,2021-11-09,2021-11-10,2021-11-10,2021-11-09,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"21, 84",higher-ed,,annual,3138146,UG,true,CENSUS,2021-11-09,"MICHELLE FOWLER, CPA",PRINCIPAL -100000,2021-12-CENSUS-0000255288,"FONDOS UNIDOS DE PUERTO RICO, INC.",VP FINANCE AND ADMINISTRATION,HEIDI E. CORTES,H.CORTES@FONDOSUNIDOS.ORG,"FONDOS UNIDOS DE PUERTO RICO, INC.",7877288500,VP FINANCE AND ADMINISTRATION,"CALLE MARGINAL LOS ANGELES, ESQ. SAGRADO",SANTURCE,PR,660269222,GSA_MIGRATION,No,00909,7879934360,PR,SAN JUAN,ASSURANCE & QUALITY CONTROL PARTNER,1250 PONCE DE LEON AVE.,009073912,USA,FERNANDO E. ORTIZ RAMOS,FORTIZ@GTCPAPR.COM,"GONZALEZ TORRES & CO., CPA, PSC",,660528656,,21,2024-01-20,2022-10-04,2022-10-05,2022-10-05,2022-10-04,2021-12-31,2020-12-31,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,21,non-profit,,annual,19689286,UG,true,CENSUS,2022-10-04,FERNANDO E. ORTIZ RAMOS,ASSURANCE & QUALITY CONTROL PARTNER -101000,2021-12-CENSUS-0000168797,BUCKS COUNTY HOUSING AUTHORITY,DIRECTOR OF FINANCE,DONNA MURRAY,DMURRAYBCHA@HOTMAIL.COM,BUCKS COUNTY HOUSING AUTHORITY,2153489469,DIRECTOR OF FINANCE,PO BOX 1329,DOYLESTOWN,PA,231659922,GSA_MIGRATION,No,189010967,7326237003,NJ,TOMS RIVER,PARTNER,"1144 HOOPER AVE, SUITE 203",08753,USA,RICH LARSEN,RICH.LARSEN@NOVOCO.COM,"NOVOGRADAC & COMPANY, LLP",,943108253,,14,2024-01-20,2022-09-27,2022-09-28,2022-09-28,2022-09-27,2021-12-31,2020-12-31,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,906416,Yes,14,local,,annual,30213862,UG,true,CENSUS,2022-09-27,RICH LARSEN,PARTNER -102000,2021-03-CENSUS-0000193568,YWCA OF GREATER BATON ROUGE,EXECUTIVE DIRECTOR,DIANNA PAYTON,PAYTOND@YWCA-BR.ORG,YWCA OF GREATER BATON ROUGE,2253830681,EXECUTIVE DIRECTOR,11404 LAKE SHERWOOD AVE N. STE B,BATON ROUGE,LA,720650993,GSA_MIGRATION,No,70806,5045618820,LA,NEW ORLEANS,MANAGING PARTNER,1100 POYDRAS STREET SUITE 1225,70163,USA,LUTHER SPEIGHT,LUTHER@SPEIGHTCPA.COM,"LUTHER SPEIGHT & CO., LLC",,300156143,,93,2024-01-20,2022-07-25,2022-05-19,2022-07-26,2022-07-25,2021-03-31,2020-03-31,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"14, 93",non-profit,,annual,3236878,UG,true,CENSUS,2022-07-25,LUTHER SPEIGHT,MANAGING PARTNER -103000,2021-03-CENSUS-0000114051,BLUE MOUNTAIN ACTION COUNCIL,CFO,RICK CLARIDGE,RICKC@BMACWW.ORG,BLUE MOUNTAIN ACTION COUNCIL,5095294980,CFO,8 E CHERRY ST,WALLA WALLA,WA,910793597,GSA_MIGRATION,No,99362,5097472158,WA,SPOKANE,CPA,"422 W. RIVERSIDE AVE., #1420",992010395,USA,DARCY SKJOTHAUG,DSKJOTHAUG@SCHOEDEL.COM,SCHOEDEL & SCHOEDEL CPAS PLLC,,910614823,,21,2024-01-20,2021-10-13,2021-10-13,2021-10-14,2021-10-13,2021-03-31,2020-03-31,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"17, 21, 84, 97, 81, 93, 64, 14, 10",non-profit,,annual,8740117,UG,true,CENSUS,2021-10-13,DARCY SKJOTHAUG,CPA -104000,2021-08-CENSUS-0000173942,LACKLAND INDEPENDENT SCHOOL DISTRICT,CHIEF FINANCIAL OFFICER,CEDRIC MENCHION,MENCHION.C@LACKLANDISD.NET,LACKLAND INDEPENDENT SCHOOL DISTRICT,2103575005,CHIEF FINANCIAL OFFICER,"2460 KENLY AVENUE, BUILDING 8265",SAN ANTONIO,TX,746013766,GSA_MIGRATION,No,78236,8302786276,TX,UVALDE,PARTNER,400 E. NOPAL ST.,78801,USA,"DUSTY R. ROUTH, CPA",DUSTYROUTH@COLEMANHORTONCPA.COM,"COLEMAN, HORTON AND COMPANY, LLP",,742668537,,84,2024-01-20,2022-05-30,2022-05-05,2022-05-31,2022-05-30,2021-08-31,2020-08-31,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 12, 21, 10",local,,annual,10655747,UG,true,CENSUS,2022-05-30,"DUSTY R. ROUTH, CPA",PARTNER -105000,2021-06-CENSUS-0000214106,"AHEPA 310-VIII, INC.",VICE PRESIDENT OF FINANCE,TRACY STORTS,TSTORTS@AHEPAHOUSING.ORG,"AHEPA 310-VIII, INC.",3178453410,VICE PRESIDENT OF FINANCE,12680 PADGETT SWITCH ROAD,IRVINGTON,AL,631262819,GSA_MIGRATION,No,36544,9373992000,OH,SPRINGFIELD,PRINCIPAL,14 EAST MAIN STREET SUITE 500,45502,USA,ROBERT KITCHEN,RKITCHEN@CSHCO.COM,CLARK SCHAEFER HACKETT,,310800053,,14,2024-01-20,2021-11-09,2021-11-09,2021-11-09,2021-11-09,2021-06-30,2020-06-30,2021,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,3519281,UG,true,CENSUS,2021-11-09,ROBERT KITCHEN,PRINCIPAL -106000,2020-06-CENSUS-0000118338,FELTON INSTITUTE-FKA FAMILY SERVICE,CFOO,MARVIN DAVIS,MDAVIS@FELTON.ORG,FELTON INSTITUTE-FKA FAMILY SERVICE,4154747310,CFOO,1005 ATLANTIC AVENUE,ALAMEDA,CA,941156530,GSA_MIGRATION,No,94501,6264036801,CA,PASADENA,PARTNER,"234 E. COLORADO BLVD., SUITE M150",91101,USA,TONETTA L. CONNER,TONETTA@NPOCPAS.COM,HARRINGTON GROUP,,954557617,,94,2024-01-21,2021-03-31,2021-03-31,2021-04-01,2021-03-31,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"94, 10, 93, 84, 17",non-profit,,annual,6587216,UG,true,CENSUS,2021-03-31,TONETTA L. CONNER,PARTNER -107000,2020-06-CENSUS-0000161038,GRAHAM COUNTY,FINANCE OFFICER,REBECCA GARLAND,BECKY.GARLAND@GRAHAMCOUNTY.ORG,GRAHAM COUNTY,8284797961,FINANCE OFFICER,12 NORTH MAIN STREET,ROBBINSVILLE,NC,566000302,GSA_MIGRATION,No,28771,8288378188,NC,MURPHY,CPA PARTNER,31 PEACHTREE STREET,28906,USA,HOLLY M. TURNER,HOLLY@MYTURNERCPA.COM,TURNER & COMPANY CPAS P.A.,,561225077,,93,2024-01-20,2021-06-15,2021-06-10,2021-06-15,2021-06-15,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"23, 97, 10, 93, 14, 20, 21",local,,annual,2113788,UG,true,CENSUS,2021-06-15,HOLLY M. TURNER,CPA PARTNER -108000,2020-06-CENSUS-0000194416,LA CA?ADA UNIFIED SCHOOL DISTRICT,ASSOCIATE SUPERINTENDENT OF BUSINESS,MARK EVANS,MEVANS@LCUSD.NET,LA CA?ADA UNIFIED SCHOOL DISTRICT,8189528380,ASSOCIATE SUPERINTENDENT OF BUSINESS,4490 CORNSHION AVE,LA CANADA,CA,956001790,GSA_MIGRATION,No,91011,9097552816,CA,RANCHO CUCAMONGA,PARTNER,10681 FOOTHILL BLVD STE 300,91730,USA,BRIAN RUFF,BRUFF@EIDEBAILLY.COM,EIDE BAILLY LLP,,450250958,,84,2024-01-21,2021-01-04,2021-01-05,2021-01-04,2021-01-04,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 10",local,,annual,1057396,UG,true,CENSUS,2021-01-04,BRIAN RUFF,PARTNER -109000,2020-06-CENSUS-0000118417,PENINSULA FAMILY SERVICE,CHIEF FINANCIAL OFFICER,INESSA SHISHMANYAN,ISHISHMANYAN@PENINSULAFAMILYSERVICE.ORG,PENINSULA FAMILY SERVICE,6504034300,CHIEF FINANCIAL OFFICER,24 SECOND AVENUE,SAN MATEO,CA,941186169,GSA_MIGRATION,No,944013828,4155683291,CA,SAN FRANCISCO,PARTNER,"44 MONTGOMERY STREET, SUITE 900",94104,USA,GRANT LAM,GRANT.LAM@ARMANINOLLP.COM,ARMANINO LLP,,946214841,,93,2024-01-21,2020-12-21,2020-12-21,2020-12-21,2020-12-21,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"93, 20, 17, 10",non-profit,,annual,3508377,UG,true,CENSUS,2020-12-21,GRANT LAM,PARTNER -110000,2020-06-CENSUS-0000135940,PUTNAM HOUSING AUTHORITY,EXECUTIVE DIRECTOR,KATHY CARTER,PUTNAMHOUSING@GMAIL.COM,PUTNAM HOUSING AUTHORITY,8609636829,EXECUTIVE DIRECTOR,123 LACONIA AVE,PUTNAM,CT,066009290,GSA_MIGRATION,No,06260,8043552808,VA,RICHMOND,PARTNER,21 S SHEPPARD ST,23221,USA,JAKE DOOLEY,JAKE@DVCPAS.COM,DOOLEY & VICARS L.L.P,,541950231,,14,2024-01-21,2021-04-07,2021-04-08,2021-04-08,2021-04-07,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,local,,annual,901313,UG,true,CENSUS,2021-04-07,JAKE DOOLEY,PARTNER -111000,2020-09-CENSUS-0000128011,GREEN MANOR,DIRECTOR OF FINANCE,MICHELE REDDY,MICHELEREDDY@FGA-NET.COM,GREEN MANOR,3232583512,DIRECTOR OF FINANCE,PO BOX 7070,PASADENA,CA,956205420,GSA_MIGRATION,No,91109,6192947200,CA,SAN DIEGO,PARTNER,"2810 CAMINO DEL RIO SOUTH, SUITE 200",92108,USA,MICHAEL J. ZIZZI,LEAFCOLEDATA@LEAF-COLE.COM,"LEAF & COLE, LLP",,952076568,,14,2024-01-21,2021-01-20,2021-01-21,2021-01-21,2021-01-20,2020-09-30,2019-10-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,5323675,UG,true,CENSUS,2021-01-20,MICHAEL J. ZIZZI,PARTNER -112000,2020-03-CENSUS-0000059346,"SIXTEENTH STREET COMMUNITY HEALTH CENTERS, INC.",VICE PRESIDENT OF FINANCE,SHERRI HUFF,SHERRI.HUFF@SSCHC.ORG,"SIXTEENTH STREET COMMUNITY HEALTH CENTERS, INC.",4148975156,VICE PRESIDENT OF FINANCE,"1337 SOUTH CESAR E. CHAVEZ DR., 2ND FL",MILWAUKEE,WI,391180475,GSA_MIGRATION,No,53204,4147775500,WI,MILWAUKEE,PARTNER,"777 E. WISCONSIN AVE, 32ND FLOOR",53202,USA,"KRISTA K. PANKOP, CPA",KRISTA.PANKOP@BAKERTILLY.COM,"BAKER TILLY US, LLP",,390859910,,93,2024-01-21,2020-08-06,2020-08-06,2020-08-06,2020-08-06,2020-03-31,2019-04-01,2020,single-audit,unmodified_opinion,,,,No,No,Yes,No,No,750000,Yes,"93, 10, 94",non-profit,,annual,6777350,UG,true,CENSUS,2020-08-06,"KRISTA K. PANKOP, CPA",PARTNER -113000,2020-06-CENSUS-0000250260,"NEW ORLEANS MILITARY AND MARITIME ACADEMY, INC.",FINANCE DIRECTOR,DENNIS LAFONT,DLAFONT@NOMMA.NET,"NEW ORLEANS MILITARY AND MARITIME ACADEMY, INC.",5048674610,FINANCE DIRECTOR,425 O'BANNON STREET,NEW ORLEANS,LA,270868305,GSA_MIGRATION,No,70114,5048375434,LA,METAIRIE,AUDIT MANAGER,"110 VETERANS MEMORIAL BLVD., SUITE 170",70005,USA,MIKE CAPAROTTA,MCAPAROTTA@HIENZMACALUSO.COM,"HIENZ & MACALUSO, LLC",,721473527,,84,2024-01-21,2021-03-15,2021-03-16,2021-03-16,2021-03-15,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"84, 10",non-profit,,annual,938918,UG,true,CENSUS,2021-03-15,MIKE CAPAROTTA,AUDIT MANAGER -114000,2020-06-CENSUS-0000247782,"NORTHERN MIDDLE TENNESSEE LOCAL WORKFORCE DEVELOPMENT BOARD, INC.",WIOA FISCAL DIRECTOR,GINGER FUSSELL,GFUSSELL@WORKFORCEESSENTIALS.COM,"NORTHERN MIDDLE TENNESSEE LOCAL WORKFORCE DEVELOPMENT BOARD, INC.",9319053559,WIOA FISCAL DIRECTOR,523 MADISON STREET STE A,CLARKSVILLE,TN,621812741,GSA_MIGRATION,No,37040,9315527474,TN,CLARKSVILLE,PARTNER,324 FRANKLIN STREET,37040,USA,JEFF PROCTOR,JPROCTOR@TCCPAS.COM,"THURMAN CAMPBELL GROUP, PLC",,263683574,,17,2024-01-21,2020-12-29,2020-12-30,2020-12-21,2020-12-29,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,17,non-profit,,annual,11901286,UG,true,CENSUS,2020-12-29,JEFF PROCTOR,PARTNER -115000,2020-06-CENSUS-0000181461,OHKAY OWINGEH BOARD OF EDUCATION,CFO,DOMINIC AGUINO,DOMINIC.AGUINO@OHKAY.ORG,OHKAY OWINGEH BOARD OF EDUCATION,5058520408,CFO,PO BOX 1119,OHKAY OWINGEH,NM,850442632,GSA_MIGRATION,No,875661119,5093443200,WA,LIBERTY LAKE,PARTNER,2501 N FAIRWAY ROAD,990198573,USA,SHANE COX,SCOX@STAUFFER-CPA.COM,STAUFFER & ASSOCIATES PLLC,,811196633,,15,2024-01-21,2021-06-01,2021-06-01,2021-06-02,2021-06-01,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"84, 15, 10",tribal,,annual,1522989,UG,true,CENSUS,2021-06-01,SHANE COX,PARTNER -116000,2020-12-CENSUS-0000238710,MISSISSIPPI COUNTY FEDERAL AWARDS,TREASURER,PEGGY MEATTE,TREASURER@MISSISSIPPICOUNTY.AR.ORG,MISSISSIPPI COUNTY FEDERAL AWARDS,8707622152,TREASURER,200 W WALNUT,BLYTHEVILLE,AR,716014331,GSA_MIGRATION,No,72315,8702387971,AR,WYNNE,CPA,P.O. BOX 1045,72396,USA,HODA WARD,HWARDCPA@HOTMAIL.COM,"MEYER & WARD, CPA",,710810734,,16,2024-01-21,2023-02-06,2023-01-16,2023-02-07,2023-02-06,2020-12-31,2020-01-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,No,"93, 21, 16",local,,annual,963087,UG,true,CENSUS,2023-02-06,HODA WARD,CPA -117000,2020-06-CENSUS-0000211765,CONTINUUM SUPPORTIVE HOUSING OF WEST HARTFORD HUD PROJ NO 017-HD016,CHIEF FINANCIAL OFFICER,"ANGELA WESTWOOD, CPA",AWESTWOOD@CONTINUUMCT.ORG,CONTINUUM SUPPORTIVE HOUSING OF WEST HARTFORD HUD PROJ NO 017-HD016,2035622264,CHIEF FINANCIAL OFFICER,109 LEGION AVENUE,NEW HAVEN,CT,061474439,GSA_MIGRATION,No,06519,8603475689,CT,MIDDLETOWN,PARTNER,505 MAIN STREET,06457,USA,"TERESA OPALACZ, CPA",TERESA@GDSCPAS.COM,"GUILMARTIN, DIPIRO & SOKOLOWSKI, LLC",,060971998,,14,2024-01-21,2021-01-21,2021-01-20,2021-01-22,2021-01-21,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,Yes,14,non-profit,,annual,1329953,UG,true,CENSUS,2021-01-21,"TERESA OPALACZ, CPA",PARTNER -118000,2020-06-CENSUS-0000235901,"OUR HOUSE, INC.",DIRECTOR OF OPERATIONS,MATT RUNGE,MATT.RUNGE@OURHOUSESHELTER.ORG,"OUR HOUSE, INC.",5013747383,DIRECTOR OF OPERATIONS,302 E. ROOSEVELT ROAD,LITTLE ROCK,AR,710653846,GSA_MIGRATION,No,72206,5012211000,AR,LITTLE ROCK,PRINCIPAL,11025 ANDERSON DR. SUITE 300,72212,USA,SARAH GENTRY,SGENTRY@HC-CPA.COM,HCJ CPAS & ADVISORS PLLC,,710650689,,14,2024-01-21,2021-01-19,2021-01-19,2021-01-20,2021-01-19,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"14, 97, 10, 17, 94, 93",non-profit,,annual,1372508,UG,true,CENSUS,2021-01-19,SARAH GENTRY,PRINCIPAL -119000,2020-09-CENSUS-0000228697,CITY OF LAKE WALES,FINANCE DIRECTOR,DOROTHY ABBOTT,DABBOTT@LAKEWALESFL.GOV,CITY OF LAKE WALES,8636784182,FINANCE DIRECTOR,201 W. CENTRAL AVE,LAKE WALES,FL,596000357,GSA_MIGRATION,No,33853,9412817212,FL,WINTER PARK,PARTNER,1031 W. MORSE BLVD,32789,USA,LORRI KIDDER,LKIDDER@CRICPA.COM,"CARR, RIGGS & INGRAM, LLC",,721396621,,20,2024-01-21,2021-05-02,2021-04-28,2021-05-03,2021-05-02,2020-09-30,2019-10-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"14, 45, 16, 20, 66, 21",local,,annual,3143823,UG,true,CENSUS,2021-05-02,LORRI KIDDER,PARTNER -120000,2020-06-CENSUS-0000199968,LA VERNA TERRACE HOUSING CORP DBA PROVENA LA VERNA TERRACE 073-EH057,HOUSING ACCOUNTING SUPPORT SUPERVISOR,ELIZABETH KING,NCRAUDITTEAM@NATIONALCHURCHRESIDENCES.ORG,LA VERNA TERRACE HOUSING CORP DBA PROVENA LA VERNA TERRACE 073-EH057,6142732179,HOUSING ACCOUNTING SUPPORT SUPERVISOR,2335 NORTH BANK DRIVE,COLUMBUS,OH,363438977,GSA_MIGRATION,No,43220,3178485700,IN,CARMEL,MEMBER,501 CONGRESSIONAL BLVD.,46032,USA,HEATHER PERRY,HPERRY@DOZ.NET,"DAUBY O'CONNOR & ZALESKI, LLC",,351750664,,14,2024-01-21,2020-09-24,2020-09-25,2020-09-25,2020-09-24,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,764033,UG,true,CENSUS,2020-09-24,HEATHER PERRY,MEMBER -121000,2020-06-CENSUS-0000151682,INDEPENDENT SCHOOL DISTRICT NO. 12,EXECUTIVE DIRECTOR OF BUSINESS SERVICES,PATRICK CHAFFEY,PCHAFFEY@ISD12.ORG,INDEPENDENT SCHOOL DISTRICT NO. 12,7637926001,EXECUTIVE DIRECTOR OF BUSINESS SERVICES,4707 NORTH ROAD,CIRCLE PINES,MN,416008811,GSA_MIGRATION,No,55014,9525450424,MN,MINNEAPOLIS,PRINCIPAL,"5353 WAYZATA BLVD, STE 410",55416,USA,AARON NIELSEN,ANIELSEN@MMKR.COM,"MALLOY, MONTAGUE, KARNOWSKI, RADOSEVICH & CO., P.A.",,411690382,,84,2024-01-21,2020-11-22,2020-11-21,2020-11-23,2020-11-22,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 84",local,,annual,3601530,UG,true,CENSUS,2020-11-22,AARON NIELSEN,PRINCIPAL -122000,2020-06-CENSUS-0000215794,SCIOTOVILLE COMMUNITY SCHOOL,TREASURER,KAREN BALLENGEE,KAREN.BALLENGEE@TARTAN.K12.OH.US,SCIOTOVILLE COMMUNITY SCHOOL,7407766777,TREASURER,224 MARSHALL STREET,SCIOTOVILLE,OH,311758579,GSA_MIGRATION,No,456625549,7402894131,OH,PIKETON,AUDIT DIRECTOR,528 SOUTH WEST STREET,456618042,USA,BRENDEN BALESTRA,BRENDEN.BALESTRA@BHMCPAGROUP.COM,"BHM CPA GROUP, INC.",,311413363,,84,2024-01-21,2020-12-12,2020-12-11,2020-12-13,2020-12-12,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 10",local,,annual,796416,UG,true,CENSUS,2020-12-12,BRENDEN BALESTRA,AUDIT DIRECTOR -123000,2020-06-CENSUS-0000169916,CAREER TECHNOLOGY CENTER OF LACKAWANNA COUNTY,CHIEF FINANCIAL OFFICER,JAMES MCHALE,JMCHALE@CTCLC.EDU,CAREER TECHNOLOGY CENTER OF LACKAWANNA COUNTY,5703468471,CHIEF FINANCIAL OFFICER,3201 ROCKWELL AVENUE,SCRANTON,PA,231709530,GSA_MIGRATION,No,18508,5708762300,PA,OLYPHANT,PARTNER,299 MAIN STREET - 2ND FLOOR,18447,USA,SEAN GRASSI,SGRASSI@RRCO.NET,ROBERT ROSSI AND COMPANY,,232385520,,84,2024-01-21,2021-03-03,2021-03-04,2021-03-03,2021-03-03,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"64, 84, 10, 17",local,,annual,1806528,UG,true,CENSUS,2021-03-03,SEAN GRASSI,PARTNER -124000,2019-06-CENSUS-0000248695,"MOUNTAIN POINTE APARTMENTS, LP DBA MOUNTAIN POINTE APTS 123-11269",CONTROLLER,QING CHEN,QING.CHEN@CPLC.ORG,"MOUNTAIN POINTE APARTMENTS, LP DBA MOUNTAIN POINTE APTS 123-11269",6022576788,CONTROLLER,1112 EAST BUCKEYE ROAD,PHOENIX,AZ,860971578,GSA_MIGRATION,No,85034,3178485700,IN,CARMEL,PRINCIPAL,501 CONGRESSIONAL BLVD.,46032,USA,KRISTEN KELLEHER,KKELLEHER@DOZ.NET,"DAUBY O'CONNOR & ZALESKI, LLC",,351750664,,14,2024-01-21,2019-10-02,2019-10-03,2019-10-03,2019-10-02,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,14,non-profit,,other,3360600,UG,true,CENSUS,2019-10-02,KRISTEN KELLEHER,PRINCIPAL -125000,2020-06-CENSUS-0000223716,REMINGTON COLLEGE,PRESIDENT,PAM BELL,PAM.BELL@REMINGTONCOLLEGE.EDU,REMINGTON COLLEGE,4075625501,PRESIDENT,7131 BUSINESS PARK LANE,LAKE MARY,FL,273339369,GSA_MIGRATION,No,32746,9122381001,GA,SAVANNAH,PARTNER,118 PARK OF COMMERCE DRIVE SUITE 200,31405,USA,DONYA J. FOWLER,DONYA.FOWLER@TJSDD.COM,TJS DEEMER DANA LLP,,582663273,84,,2024-01-21,2021-03-07,2021-03-08,2021-03-04,2021-03-07,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,84,higher-ed,,annual,74576483,UG,true,CENSUS,2021-03-07,DONYA J. FOWLER,PARTNER -126000,2020-03-CENSUS-0000194592,ONEIDA HOUSING AUTHORITY,EXECUTIVE DIRECTOR,ROBERT WALTERS,ONEIDAHOUSING@CNYMAIL.COM,ONEIDA HOUSING AUTHORITY,3153638450,EXECUTIVE DIRECTOR,226 FARRIER AVENUE,ONEIDA,NY,161036082,GSA_MIGRATION,No,134211667,5188277499,NY,GILBOA,CPA,126 MAYBIE ROAD,12076,USA,PATRICK BUEL,PBUEL@BUELCPA.COM,"BUEL CPA, P.C.",,200348081,,14,2024-01-21,2021-06-10,2021-06-11,2021-06-10,2021-06-10,2020-03-31,2019-04-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,local,,annual,1589036,UG,true,CENSUS,2021-06-10,PATRICK BUEL,CPA -127000,2020-11-CENSUS-0000246830,GRAMERCY PARK COOPERATIVE OF ST. PAUL,MANAGEMENT AGENT,CONNIE BEDNAREK,CBEDNAR2@FAIRVIEW.ORG,GRAMERCY PARK COOPERATIVE OF ST. PAUL,6128743460,MANAGEMENT AGENT,5688 BRENT AVENUE,INVER GROVE HEIGHTS,MN,411819879,GSA_MIGRATION,No,55076,9523451573,MN,EDINA,CPA,"7900 WEST 78TH STREET, SUITE 450",55439,USA,PAULA M MEIDL,PAULA@CASEYMENDEN.COM,"CASEY, MENDEN, FAUST & NELSON, PA",,411535741,,14,2024-01-21,2021-03-30,2021-03-30,2021-03-30,2021-03-30,2020-11-30,2019-12-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,6658025,UG,true,CENSUS,2021-03-30,PAULA M MEIDL,CPA -128000,2020-12-CENSUS-0000177899,"JUNEAU COUNTY, WISCONSIN",CPA - FINANCE DIRECTOR,LORI CHIPMAN,LCHIPMAN@CO.JUNEAU.WI.US,"JUNEAU COUNTY, WISCONSIN",6088479310,CPA - FINANCE DIRECTOR,220 EAST STATE STREET - ROOM 203,MAUSTON,WI,396005706,GSA_MIGRATION,No,53948,8883088281,WI,VIROQUA,AUDIT PARTNER,1315 BAD AXE COURT; PO BOX 271,546650271,USA,CARRIE LEONARD,CLEONARD@JOHNSONBLOCK.COM,"JOHNSON BLOCK & COMPANY, INC.",,391628949,,16,2024-01-21,2021-10-18,2021-10-15,2021-10-19,2021-10-18,2020-12-31,2020-01-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,No,"14, 93, 90, 16, 84, 21, 20, 10, 97",local,,annual,9382362,UG,true,CENSUS,2021-10-18,CARRIE LEONARD,AUDIT PARTNER -129000,2020-12-CENSUS-0000207657,ELLIS HOSPITAL (D/B/A ELLIS MEDICINE,CONTROLLER,ERIC BERGIN,BERGINE@ELLISMEDICINE.ORG,ELLIS HOSPITAL (D/B/A ELLIS MEDICINE,5186128645,CONTROLLER,1101 NOTT STREET,SCHENECTADY,NY,141338428,GSA_MIGRATION,No,12308,3154463600,NY,SYRCAUSE,PARTNER,5784 WIDEWATERS PARKWAY,13214,USA,SHANNON FLOOD,SFLOOD@FCC-CPA.COM,FUST CHARLES CHAMBERS LLP,,161226221,14,,2024-01-21,2021-10-13,2021-10-14,2021-10-13,2021-10-13,2020-12-31,2020-01-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"14, 10, 93, 16, 32, 84",non-profit,,annual,71511530,UG,true,CENSUS,2021-10-13,SHANNON FLOOD,PARTNER -130000,2020-12-CENSUS-0000217586,NIAGARA LUTHERAN DEVELOPMENT DBA GREENFIELD HEALTH AND REHABILITATION,PRESIDENT/CEO,CHRISTOPHER E KOENIG,PRESIDENT@NIAGARALUTHERAN.ORG,NIAGARA LUTHERAN DEVELOPMENT DBA GREENFIELD HEALTH AND REHABILITATION,7166840202,PRESIDENT/CEO,5949 BROADWAY STREET,LANCASTER,NY,161502150,GSA_MIGRATION,No,140869523,7164176877,NY,WHEATFIELD,PRINCIPAL,800 LIBERTY BUILDING,14120,USA,JOHN PETERSON,JOHN.PETERSON@FREEDMAXICK.COM,FREED MAXICK CPAS PC,,454051133,,14,2024-01-21,2021-04-22,2021-04-22,2021-04-19,2021-04-22,2020-12-31,2020-01-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"14, 93",non-profit,,annual,9142968,UG,true,CENSUS,2021-04-22,JOHN PETERSON,PRINCIPAL -131000,2020-06-CENSUS-0000204706,WESTMORELAND-FAYETTE WORKFORCE INVESTMENT BOARD,CHIEF FINANCIAL OFFICER,THOMAS WHETSEL,TWHETSEL@WESTFAYWIB.ORG,WESTMORELAND-FAYETTE WORKFORCE INVESTMENT BOARD,7247552145,CHIEF FINANCIAL OFFICER,145 PAVILION LANE,YOUNGWOOD,PA,251860049,GSA_MIGRATION,No,15697,7243123409,PA,NEW BRIGHTON,AUDIT MANAGER,1000 3RD AVENUE,15066,USA,KIMBERLY TURNLEY,K.TURNLEY@COMCAST.NET,MARK C TURNLEY CPA,,251570881,,17,2024-01-21,2021-03-29,2021-03-29,2021-03-30,2021-03-29,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 17, 93",non-profit,,annual,5679001,UG,true,CENSUS,2021-03-29,KIMBERLY TURNLEY,AUDIT MANAGER -132000,2020-06-CENSUS-0000161071,IREDELL-STATESVILLE BOARD OF EDUCATION,FINANCE DIRECTOR,MELISSA WIKE,MELISSA_WIKE@ISS.K12.NC.US,IREDELL-STATESVILLE BOARD OF EDUCATION,7048728931,FINANCE DIRECTOR,549 RACE STREET,STATESVILLE,NC,561744267,GSA_MIGRATION,No,18677,7048711477,NC,STATESVILLE,PARTNER,113 N CENTER STREET,28677,USA,JOHN ROSENTAL,JROSENTAL@CLCPLLC.COM,"COFFEY, LOVINS, AND COMPANY PLLC",,561929162,,84,2024-01-21,2020-11-02,2020-10-30,2020-11-02,2020-11-02,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 21, 84, 12",local,,annual,18625259,UG,true,CENSUS,2020-11-02,JOHN ROSENTAL,PARTNER -133000,2020-12-CENSUS-0000249241,"GOLDEN TERRACE VILLAGE IV, INC.",MANAGER,SHAYLA CHAPPELL,GOLDENTERRACE@SBCGLOBAL.NET,"GOLDEN TERRACE VILLAGE IV, INC.",3255731416,MANAGER,1607 52ND STREET,SNYDER,TX,752753824,GSA_MIGRATION,No,79549,4327583226,TX,SEMINOLE,AUDIT PARTNER,P.O. BOX 309,79360,USA,DEREK RAY,DEREK@DAVIS-RAY.COM,"DAVIS, RAY & CO., PC",,752057245,,14,2024-01-21,2021-03-29,2021-03-26,2021-03-30,2021-03-29,2020-12-31,2020-01-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,1634682,UG,true,CENSUS,2021-03-29,DEREK RAY,AUDIT PARTNER -134000,2020-06-CENSUS-0000214191,LAC COURTE OREILLES OJIBWE SCHOOL,ACCOUNTING DIRECTOR,DAWN DOYLE,DAWN.DOYLE@LCOOSK12.ORG,LAC COURTE OREILLES OJIBWE SCHOOL,7156348924,ACCOUNTING DIRECTOR,8575 N TREPANIA RD,HAYWARD,WI,391763080,GSA_MIGRATION,No,54843,9897321156,MI,GAYLORD,SENIOR AUDIT PARTNER,215 S COURT AVE,49735,USA,KURT TUCKER,KURT@MWP.US.COM,"MIDWEST PROFESSIONALS, PLLC",,383571340,,15,2024-01-21,2021-09-28,2021-09-29,2021-09-29,2021-09-28,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,Yes,Yes,No,Yes,750000,No,"15, 10, 93, 84",tribal,,annual,9102660,UG,true,CENSUS,2021-09-28,KURT TUCKER,SENIOR AUDIT PARTNER -135000,2020-06-CENSUS-0000133949,ROUND VALLEY UNIFIED SCHOOL DISTRICT,BUSINESS MANAGER,JESSICA WILLIAMSON,JWILLIAMSON@RVUSD.US,ROUND VALLEY UNIFIED SCHOOL DISTRICT,7079836171,BUSINESS MANAGER,76280 HIGH SCHOOL STREET,COVELO,CA,946002711,GSA_MIGRATION,No,95428,7079836171,CA,SAN DIEGO,PARTNER,348 OLIVE STREET,92103,USA,MICHAEL ASH,MASH@CHRISTYWHITE.COM,CHRISTY WHITE INC,,272956198,,84,2024-01-21,2021-03-31,2021-04-01,2021-03-31,2021-03-31,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 84",local,,annual,2618781,UG,true,CENSUS,2021-03-31,MICHAEL ASH,PARTNER -136000,2020-12-CENSUS-0000241605,HOUSING AUTHORITY OF THE BOROUGH OF GLASSBORO,EXECUTIVE DIRECTOR,KIMBERLY GOBER,KGOBER@HAGC.ORG,HOUSING AUTHORITY OF THE BOROUGH OF GLASSBORO,8568454959,EXECUTIVE DIRECTOR,181 DELSEA MANOR DRIVE,GLASSBORO,NJ,216008418,GSA_MIGRATION,No,08028,8564356200,NJ,WOODBURY,PARTNER,"6 NORTH BROAD STREET, SUITE 201",08096,USA,NINA SORELLE,NSORELLE@BOWMAN.CPA,"BOWMAN & COMPANY, LLP",,210658561,,14,2024-01-21,2021-07-27,2021-07-21,2021-07-28,2021-07-27,2020-12-31,2020-01-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 14",local,,annual,2490531,UG,true,CENSUS,2021-07-27,NINA SORELLE,PARTNER -137000,2020-06-CENSUS-0000192522,CITY OF MILES CITY,CITY CLERK,MARY ROWE,CITYCLERK@MILESCITY-MT.ORG,CITY OF MILES CITY,4068748609,CITY CLERK,PO BOX 910,MILES CITY,MT,816001292,GSA_MIGRATION,No,593010910,4066980022,MT,BILLINGS,SHAREHOLDER,2810 CENTRAL AVE SUITE B,59102,USA,CURTIS D WYSS,CURT@OLNESSCPA.COM,"OLNESS & ASSOCIATES, PC",,810485423,,20,2024-01-21,2021-02-10,2021-02-11,2021-02-11,2021-02-10,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,Yes,Yes,No,No,750000,No,"21, 93, 20, 97, 94",local,,annual,1393438,UG,true,CENSUS,2021-02-10,CURTIS D WYSS,SHAREHOLDER -138000,2020-06-CENSUS-0000157881,PENNS GROVE CARNEYS POINT REGIONAL SCHOOL DISTRICT,BUSINESS ADMINISTRATOR,BRIAN FERGUSON,BFERGUSON@PGCPSCHOOLS.ORG,PENNS GROVE CARNEYS POINT REGIONAL SCHOOL DISTRICT,8563583094,BUSINESS ADMINISTRATOR,100 IONA AVENUE,PENNS GROVE,NJ,216000282,GSA_MIGRATION,No,08069,8566293111,NJ,WILLIAMSTOWN,PARTNER,991 SOUTH BLACK HORSE PIKE,08094,USA,RAYMOND COLAVITA,ANTHONY@COLAVITA.NET,"NIGHTLINGER, COLAVITA, & VOLPA",,222617821,,93,2024-01-21,2021-01-24,2021-01-25,2021-01-25,2021-01-24,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,No,Yes,Yes,No,750000,No,"84, 10, 93",local,,annual,3700498,UG,true,CENSUS,2021-01-24,RAYMOND COLAVITA,PARTNER -139000,2020-08-CENSUS-0000174376,PHARR-SAN JUAN-ALAMO I.S.D.,CHIEF FINANCIAL OFFICER,JANET ROBLES,JANET.ROBLES@PSJAISD.US,PHARR-SAN JUAN-ALAMO I.S.D.,9563542019,CHIEF FINANCIAL OFFICER,601 E KELLY,PHARR,TX,746001876,GSA_MIGRATION,No,78577,9566863701,TX,MCALLEN,PARTNER,4100 N 23RD ST,78504,USA,AARON RIOS,ARIOS@CRICPA.COM,"CARR RIGGS & INGRAM, LLC",,721396621,84,,2024-01-21,2021-03-09,2021-03-10,2021-03-09,2021-03-09,2020-08-31,2019-09-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,2099022,Yes,"84, 93, 12, 17, 10",local,,annual,69967392,UG,true,CENSUS,2021-03-09,AARON RIOS,PARTNER -140000,2020-12-CENSUS-0000241399,ELICA HEALTH CENTERS,CONTROLLER,JAMES MOHRHERR,JMOHRHERR@ELICAHEALTH.ORG,ELICA HEALTH CENTERS,9165698484,CONTROLLER,"1860 HOWE AVENUE, SUITE 455",SACRAMENTO,CA,371424390,GSA_MIGRATION,No,95825,9169290540,CA,SACRAMENTO,PARTNER,"1760 CREEKSIDE OAKS DRIVE, SUITE 160",95833,USA,BRADLEY BARTELLS,BJB@MUNCPAS.COM,"MANN, URRUTIA, NELSON CPAS & ASSOCIATES, LLP",,200276349,,93,2024-01-21,2021-07-28,2021-07-29,2021-07-28,2021-07-28,2020-12-31,2020-01-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,93,non-profit,,annual,4347431,UG,true,CENSUS,2021-07-28,BRADLEY BARTELLS,PARTNER -141000,2020-12-CENSUS-0000193149,"SENIOR CITIZENS' Y-HOUSE OF BEAUMONT, INC.",MANAGEMENT AGENT,"JOSHUA W. ALLEN, SR.",RASHANDA@JALLENMGMT.COM,"SENIOR CITIZENS' Y-HOUSE OF BEAUMONT, INC.",4098338947,MANAGEMENT AGENT,1390 BROADWAY,BEAUMONT,TX,760512267,GSA_MIGRATION,No,77701,4098383755,TX,BEAUMONT,TWHIDDON@FMWCPAS.COM,1150 NORTH ELEVENTH ST.,77702,USA,TERRY S. WHIDDON,TWHIDDON@FMWCPAS.COM,"FMW, PC",,760154479,,14,2024-01-21,2021-04-05,2021-04-05,2021-04-06,2021-04-05,2020-12-31,2020-01-01,2020,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,14,non-profit,,annual,2505978,UG,true,CENSUS,2021-04-05,TERRY S. WHIDDON,TWHIDDON@FMWCPAS.COM -142000,2019-06-CENSUS-0000160541,HOLLAND PATENT CENTRAL SCHOOL DISTRICT,SUPERINTENDENT,JASON EVANGELIST,JEVANGELIST@HPSCHOOLS.ORG,HOLLAND PATENT CENTRAL SCHOOL DISTRICT,3158657200,SUPERINTENDENT,9601 MAIN STREET,HOLLAND PATENT,NY,156002239,GSA_MIGRATION,No,13354,3153369220,NY,ROME,PARTNER,200 EAST GARDEN STREET,13440,USA,ROY J. CLARK,CCLARK@DARCANGELO-CNY.COM,"D'ARCANGELO & CO., LLP",,132550103,,84,2024-01-21,2019-11-18,2019-11-15,2019-11-18,2019-11-18,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 84",local,,annual,1065188,UG,true,CENSUS,2019-11-18,ROY J. CLARK,PARTNER -143000,2019-06-CENSUS-0000004952,"THE BOSTON HEALTHCARE FOR THE HOMELESS PROGRAM, INC.",CHIEF FINANCIAL OFFICER,AGNES LEUNG,ALEUNG@BHCHP.ORG,"THE BOSTON HEALTHCARE FOR THE HOMELESS PROGRAM, INC.",8576541100,CHIEF FINANCIAL OFFICER,780 ALBANY STREET,BOSTON,MA,043160480,GSA_MIGRATION,No,01581,5083669789,MA,WESTBOROUGH,VICE PRESIDENT,50 WASHINGTON STREET,01581,USA,SORIE M. KABA,SKABA@AAFCPA.COM,"ALEXANDER, ARONSON, FINNING & CO., P.C.",,042571780,,93,2024-01-21,2019-11-19,2019-12-16,2019-12-17,2019-11-19,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 94, 93",non-profit,,annual,5777532,UG,true,CENSUS,2019-11-19,SORIE M. KABA,VICE PRESIDENT -144000,2020-06-CENSUS-0000249289,FORT TOTTEN PUBLIC SCHOOL DISTRICT #30,BUSINESS MANAGER,MEGAN A CALLAHAN,MEGAN.A.CALLAHAN@K12.ND.US,FORT TOTTEN PUBLIC SCHOOL DISTRICT #30,7017661400,BUSINESS MANAGER,HWY 57 S,FORT TOTTEN,ND,456000113,GSA_MIGRATION,No,58301,7012525422,ND,JAMESTOWN,CPA,111 9TH ST SW,58401,USA,DUANE R DUNN,D.DUNN@SCHAUERCPA.COM,"SCHAUER & ASSOCIATES, P.C.",,450350118,,84,2024-01-21,2021-03-31,2021-03-30,2021-03-31,2021-03-31,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,Yes,84,local,,annual,2418519,UG,true,CENSUS,2021-03-31,DUANE R DUNN,CPA -145000,2020-09-CENSUS-0000142159,SOUTH CENTRAL ILLINOIS REGIONAL PLANNING & DEVELOPMENT COMMISSION,EXECUTIVE DIRECTOR,JAMES PATRICK,JPATRICK@SCIRPDC.COM,SOUTH CENTRAL ILLINOIS REGIONAL PLANNING & DEVELOPMENT COMMISSION,6185484234,EXECUTIVE DIRECTOR,120 DELMAR AVE.,SALEM,IL,370974030,GSA_MIGRATION,No,62882,6182673213,IL,ODIN,PRESIDENT,5859 US HWY 51,62870,USA,SUSAN J LYONS,SLYONS1007@GMAIL.COM,SUSAN J LYONS CPA PC,,364149296,,10,2024-01-21,2021-01-28,2021-01-29,2021-01-28,2021-01-28,2020-09-30,2019-10-01,2020,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,No,"20, 11, 10, 59",local,,annual,2131397,UG,true,CENSUS,2021-01-28,SUSAN J LYONS,PRESIDENT -146000,2019-12-CENSUS-0000181260,ST MATTHEW HOUSING DEVELOPMENT INC T/A ST MATTHEW MANOR,MANAGING AGENT,CHARLES MENSCH,CMENSCH@COMMUNITYREALTYMANAGEMENT.COM,ST MATTHEW HOUSING DEVELOPMENT INC T/A ST MATTHEW MANOR,6096468861,MANAGING AGENT,230-242 NORTH 57TH STREET,PHILADELPHIA,PA,222813779,GSA_MIGRATION,No,19139,6096414000,NJ,NORTHFIELD,PARTNER,332 TILTON ROAD,08225,USA,JEFFREY WILSON,JAWILSON@CAPALDIREYNOLDS.COM,"CAPALDI REYNOLDS & PELOSI, P.A.",,222097516,,14,2024-01-21,2020-04-23,2020-04-24,2020-04-24,2020-04-23,2019-12-31,2018-12-31,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,5511201,UG,true,CENSUS,2020-04-23,JEFFREY WILSON,PARTNER -147000,2019-06-CENSUS-0000222692,"PORT CITY VOA HOUSING, INC. 062-HD050",SENIOR ACCOUNTING MANAGER,CINDY FULDORD,CFULFORD@VOASE.ORG,"PORT CITY VOA HOUSING, INC. 062-HD050",2513003500,SENIOR ACCOUNTING MANAGER,1204 HILLCREST ROAD,MOBILE,AL,753114572,GSA_MIGRATION,No,36695,2513431200,AL,MOBILE,PARTNER,3800 AIRPORT BLVD. STE 101,36608,USA,ZACHARY WOLF,ZACK@SMITHDUKES.COM,"SMITH, DUKES & BUCKALEW, LLP",,630191630,,14,2024-01-21,2019-10-18,2019-10-18,2019-10-19,2019-10-18,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,1009643,UG,true,CENSUS,2019-10-18,ZACHARY WOLF,PARTNER -148000,2019-12-CENSUS-0000213472,"MEEKER SENIOR HOUSING, INC.",MANAGEMENT AGENT,"LYNCO, INC.",DEBRA@LYNCOPROPERTIES.COM,"MEEKER SENIOR HOUSING, INC.",9184888485,MANAGEMENT AGENT,209 W GREEN ST,MEEKER,OK,731396847,GSA_MIGRATION,No,74855,9182612758,OK,TULSA,CPA,6851 S CANTON,74136,USA,JOHN FLUSCHE,CPA@OKLAHOMA.NET,"JOHN FLUSCHE, CPA",,731513650,,14,2024-01-21,2020-04-27,2020-04-28,2020-04-28,2020-04-27,2019-12-31,2018-12-31,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,1173058,UG,true,CENSUS,2020-04-27,JOHN FLUSCHE,CPA -149000,2019-12-CENSUS-0000249807,LOUP VALLEYS RURAL PUBLIC POWER DISTRICT,OFFICE MANAGER,JOSH ROSBERG,JOSH@LOUPVALLEYSPOWER.COM,LOUP VALLEYS RURAL PUBLIC POWER DISTRICT,3087283633,OFFICE MANAGER,606 S STREET,ORD,NE,476000977,GSA_MIGRATION,No,68862,4023915336,NE,OMAHA,CPA,2711 SOUTH 88TH STREET,68124,USA,LAURA RIGGS,FREEMNRIGG@AOL.COM,FREEMAN & RIGGS,,470784935,,97,2024-01-21,2020-10-08,2020-10-09,2020-10-08,2020-10-08,2019-12-31,2018-12-31,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,97,local,,annual,9332446,UG,true,CENSUS,2020-10-08,LAURA RIGGS,CPA -150000,2019-06-CENSUS-0000158282,BERNALILLO PUBLIC SCHOOL DISTRICT,FINANCE DIRECTOR,VICKIE GRACIA,VGARCIA@BPS.K12.NM.US,BERNALILLO PUBLIC SCHOOL DISTRICT,5054045689,FINANCE DIRECTOR,224 N. CAMINO DEL PUEBLO,BERNALLIO,NM,856000791,GSA_MIGRATION,No,87004,5059984468,NM,ALBUQUERQUE,AUDIT SENIOR,1030 18TH STREET NW,87104,USA,ESTHER ALEJO,ESTHER.ALEJO@RICCICPA.COM,RICCI & COMPANY LLC CPA,,205949532,,84,2024-01-21,2020-03-29,2020-03-26,2020-03-30,2020-03-29,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,Yes,No,No,Yes,750000,Yes,"10, 93, 84",state,,annual,13041709,UG,true,CENSUS,2020-03-29,ESTHER ALEJO,AUDIT SENIOR -151000,2019-06-CENSUS-0000133772,RICHLAND UNION SCHOOL DISTRICT,CHIEF BUSINESS OFFICER,MARTIN RODRIGUEZ,MRODRIGUEZ@RSDSHAFTER.ORG,RICHLAND UNION SCHOOL DISTRICT,6617468610,CHIEF BUSINESS OFFICER,331 SHAFTER AVE,SHAFTER,CA,956002546,GSA_MIGRATION,No,93263,5104525051,CA,SAN DIEGO,MANAGING PARTNER,"16870 W BERNARDO DR #250, , CA 92127",92127,USA,SANWAR HARSHWAL,SANWAR@HARSHWAL.COM,"HARSHWAL& COMPANY, LLP",,270741376,,10,2024-01-21,2020-03-29,2020-03-30,2020-03-30,2020-03-29,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 10, 84",local,,annual,5559395,UG,true,CENSUS,2020-03-29,SANWAR HARSHWAL,MANAGING PARTNER -152000,2019-06-CENSUS-0000189571,CITY OF PITTSBURG,FINANCIAL REPORTING MANAGER,LAURA MENDEZ,LMENDEZ@CI.PITTSBURG.CA.US,CITY OF PITTSBURG,9252524872,FINANCIAL REPORTING MANAGER,65 CIVIC AVENUE,PITTSBURG,CA,946000395,GSA_MIGRATION,No,94565,9259300902,CA,PLEASANT HILL,PARTNER,"3478 BUSKIRK AVENUE, SUITE 215",94523,USA,VIKKI RODRIGUEZ,VIKR@MAZEASSOCIATES.COM,MAZE & ASSOCIATES ACCOUNTANCY,,942590179,,14,2024-01-21,2020-02-16,2020-02-17,2020-02-14,2020-02-16,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"17, 16, 66, 14, 20",local,,annual,19729027,UG,true,CENSUS,2020-02-16,VIKKI RODRIGUEZ,PARTNER -153000,2019-12-CENSUS-0000244861,LORETTO APTS AT O'BRIEN ROAD HDFC INC (O'BRIEN RD SR APPTS 2) 014EE287,V.P. OF FINANCE (MANAGING AGENT),DAVID R. FILIPSKI,DFILIPSKI@CHRISTOPHER-COMMUNITY.ORG,LORETTO APTS AT O'BRIEN ROAD HDFC INC (O'BRIEN RD SR APPTS 2) 014EE287,3154241821,V.P. OF FINANCE (MANAGING AGENT),7174 O'BRIEN ROAD,SYRACUSE,NY,462450456,GSA_MIGRATION,No,13209,3154719171,NY,SYRACUSE,MEMBER,443 N FRANKLIN STREET,13204,USA,DAVID J. DE SILVA,DJD@DBBLLC.COM,"DERMODY, BURKE & BROWN, CPAS, LLC",,010723685,,14,2024-01-21,2020-03-15,2020-03-14,2020-03-16,2020-03-15,2019-12-31,2018-12-31,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,3422426,UG,true,CENSUS,2020-03-15,DAVID J. DE SILVA,MEMBER -154000,2019-06-CENSUS-0000160714,GREATER SOUTHERN TIER BOCES,ASSISTANT SUPERINTENDENT FOR FINANCE,T. GREGORY DALE,GDALE@GSTBOCES.ORG,GREATER SOUTHERN TIER BOCES,6077393581,ASSISTANT SUPERINTENDENT FOR FINANCE,459 PHILO ROAD,ELMIRA,NY,760814144,GSA_MIGRATION,No,14903,6072724444,NY,ITHACA,PARTNER,"401 EAST STATE STREET, SUITE 500",14850,USA,LESLIE SPURGIN,DCF@INSEROCPA.COM,"INSERO & CO. CPAS, LLP",,475324570,,84,2024-01-21,2019-10-21,2019-10-22,2019-10-22,2019-10-21,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 10",local,,annual,2065783,UG,true,CENSUS,2019-10-21,LESLIE SPURGIN,PARTNER -155000,2019-06-CENSUS-0000227008,"POSITIVELY THIRD STREET HOUSING DEVELOPMENT FUND CO.,INC. 012-EE287",VICE PRESIDENT OF FINANCE,MICHAEL KLIDAS,MKLIDAS@JASA.ORG,"POSITIVELY THIRD STREET HOUSING DEVELOPMENT FUND CO.,INC. 012-EE287",2122735257,VICE PRESIDENT OF FINANCE,"247 WEST 37TH STREET, 9TH FLOOR",NEW YORK,NY,300054211,GSA_MIGRATION,No,10018,3019615546,NY,NEW YORK,PARTNER,1301 AVENUE OF THE AMERICAS,10019,USA,WINELL BELFONTE,WINELL.BELFONTE@COHNREZNICK.COM,COHNREZNICK LLP,,221478099,,14,2024-01-21,2019-11-19,2019-12-23,2019-12-23,2019-11-19,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,8815693,UG,true,CENSUS,2019-11-19,WINELL BELFONTE,PARTNER -156000,2019-09-CENSUS-0000221617,"ADAMS COUNTY HEALTH CENTER, INC.",CFO,MARY ANN DOMECQ,MARYANND@ACHCID.ORG,"ADAMS COUNTY HEALTH CENTER, INC.",2082534242,CFO,PO BOX 428,COUNCIL,ID,208341138,GSA_MIGRATION,No,83612,5092420874,WA,SPOKANE VALLEY,OWNER,12015 EAST MAIN AVENUE,99206,USA,SHAUN JOHNSON,SJOHNSON@DZACPA.COM,"DINGUS, ZARECOR AND ASSOCIATES PLLC",,200079326,,93,2024-01-21,2020-01-12,2020-01-10,2020-01-13,2020-01-12,2019-09-30,2018-09-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,93,non-profit,,annual,1333589,UG,true,CENSUS,2020-01-12,SHAUN JOHNSON,OWNER -157000,2019-09-CENSUS-0000056721,MICHIGAN PEER REVIEW ORGANIZATION,SR. V-P AND CHIEF ADMINISTRATIVE OFFICER,FRANK FOX,FFOX@MPRO.ORG,MICHIGAN PEER REVIEW ORGANIZATION,2484657300,SR. V-P AND CHIEF ADMINISTRATIVE OFFICER,"22670 HAGGERTY ROAD, SUITE 100",FARMINGTON HILLS,MI,382536610,GSA_MIGRATION,No,483352611,3139652655,MI,DETROIT,PRINCIPAL,"1200 BUHL BUILDING, 535 GRISWOLD",482263689,USA,"MICHAEL R. NICHOLAS, C.P.A.",MNICHOLAS@GJC-CPA.COM,GEORGE JOHNSON & COMPANY,,382029668,,93,2024-01-21,2020-02-09,2020-02-10,2020-02-10,2020-02-09,2019-09-30,2018-09-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,93,non-profit,,annual,3521000,UG,true,CENSUS,2020-02-09,"MICHAEL R. NICHOLAS, C.P.A.",PRINCIPAL -158000,2019-12-CENSUS-0000227812,ALLIANCE GOOD SAMARITAN HSG DBA WILD FLOWER TERRACE 103-EE035,VICE PRESIDENT,ERIC VANDEN HULL,SBEAMAN@GOOD-SAM.COM,ALLIANCE GOOD SAMARITAN HSG DBA WILD FLOWER TERRACE 103-EE035,6053230921,VICE PRESIDENT,4800 W 57TH ST,SIOUX FALLS,SD,204714573,GSA_MIGRATION,No,57108,6053391999,SD,SIOUX FALLS,PARTNER,"200 E 10TH ST, STE 500",57104,USA,ANGIE HILLESTAD,AHILLESTAD@EIDEBAILLY.COM,EIDE BAILLY LLP,,450250958,,14,2024-01-21,2020-04-22,2020-04-23,2020-04-23,2020-04-22,2019-12-31,2018-12-31,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,2206024,UG,true,CENSUS,2020-04-22,ANGIE HILLESTAD,PARTNER -159000,2019-09-CENSUS-0000053662,"EAST CENTRAL ILLINOIS AREA AGENCY ON AGING, INC.",DEPUTY DIRECTOR,SUSAN REDMAN,SREDMAN@ECIAAA.ORG,"EAST CENTRAL ILLINOIS AREA AGENCY ON AGING, INC.",3098292065,DEPUTY DIRECTOR,1003 MAPLE HILL ROAD,BLOOMINGTON,IL,370982325,GSA_MIGRATION,No,617059327,2177933363,IL,SPRINGFIELD,DIRECTOR,"3201 WEST WHITE OAKS DRIVE, SUITE 102",62704,USA,"TERI TAYLOR, CPA",TERI.TAYLOR@SIKICH.COM,SIKICH LLP,,363168081,,93,2024-01-21,2020-04-02,2020-04-02,2020-04-02,2020-04-02,2019-09-30,2018-09-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,93,non-profit,,annual,3692515,UG,true,CENSUS,2020-04-02,"TERI TAYLOR, CPA",DIRECTOR -160000,2019-12-CENSUS-0000202475,CITY OF ALBERT LEA,FINANCE DIRECTOR,KRISTI BRUTLAG,KBRUTLAG@CI.ALBERTLEA.MN.US,CITY OF ALBERT LEA,5073774305,FINANCE DIRECTOR,221 E. CLARK ST,ALBERT LEA,MN,416004922,GSA_MIGRATION,No,56007,9525636873,MN,BLOOMINGTON,PARTNER,3800 AMERICAN BLVD W,55431,USA,MATT MAYER,MATT.MAYER@BERGANKDV.COM,"BERGANKDV, LTD",,411431613,,66,2024-01-21,2020-06-17,2020-06-18,2020-06-18,2020-06-17,2019-12-31,2018-12-31,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"15, 20, 66, 14, 16",local,,annual,1591450,UG,true,CENSUS,2020-06-17,MATT MAYER,PARTNER -161000,2019-12-CENSUS-0000139132,"CITY OF GALESBURG, ILLINOIS",DIRECTOR OF FINANCE AND INFORMATION SYST,GLORIA OSBORN,GOSBORN@CI.GALESBURG.IL.US,"CITY OF GALESBURG, ILLINOIS",3093453677,DIRECTOR OF FINANCE AND INFORMATION SYST,55 WEST TOMPKINS STREET,GALESBURG,IL,376001160,GSA_MIGRATION,No,61401,6306456205,IL,OAK BROOK,PARTNER,"1301 W .22ND STREET, STE 400",60523,USA,JASON COYLE,JASON.COYLE@BAKERTILLY.COM,BAKER TILLY VIRCHOW KRAUSE LLP,,390859910,,16,2024-01-21,2020-08-13,2020-08-14,2020-08-14,2020-08-13,2019-12-31,2018-12-31,2019,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,No,"14, 97, 16, 20, 66, 10",local,,annual,2544113,UG,true,CENSUS,2020-08-13,JASON COYLE,PARTNER -162000,2019-06-CENSUS-0000165855,MCLOUD PUBLIC SCHOOLS,SUPERINTENDENT,STEVE STANLEY,SSTANLEY@MCLOUDSCHOOLS.US,MCLOUD PUBLIC SCHOOLS,4059643314,SUPERINTENDENT,PO BOX 240,MCLOUD,OK,730939118,GSA_MIGRATION,No,74851,9183664440,OK,BIXBY,PARTNER,116 W BRECKENRIDGE,74008,USA,JACK JENKINS,JHJCPA@JACKHJENKINSCPA.COM,"JENKINS & KEMPER, CPAS, P.C.",,731514558,,84,2024-01-21,2020-03-30,2020-03-30,2020-03-31,2020-03-30,2019-06-30,2018-06-30,2019,single-audit,not_gaap,regulatory_basis,Yes,qualified_opinion,No,No,No,No,No,750000,No,"10, 84",state,,annual,1945179,UG,true,CENSUS,2020-03-30,JACK JENKINS,PARTNER -163000,2019-06-CENSUS-0000131980,CITY OF SANGER,ADMINISTRATIVE SERVICE DIRECTOR,BRET HARMON,BHARMON@CI.SANGER.CA.US,CITY OF SANGER,5598766300,ADMINISTRATIVE SERVICE DIRECTOR,1700 7TH STREET,SANGER,CA,946000425,GSA_MIGRATION,No,93657,9169290540,CA,SACRAMENTO,PARTNER,"1760 CREEKSIDE OAKS DRIVE, SUITE 160",95833,USA,JUSTIN WILLIAMS,JJW@MUNCPAS.COM,"MANN, URRUTIA, NELSON CPAS & ASSOCIATES",,200276349,,20,2024-01-21,2020-03-04,2020-07-22,2020-07-24,2020-03-04,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,No,"16, 20, 93, 15, 14",local,,annual,1206157,UG,true,CENSUS,2020-03-04,JUSTIN WILLIAMS,PARTNER -164000,2019-06-CENSUS-0000134527,EXETER UNIFIED SCHOOL DISTRICT,BUSINESS MANAGER,KRISTEN KIRK,KKIRK@EXETER.K12.CA.US,EXETER UNIFIED SCHOOL DISTRICT,5595929421,BUSINESS MANAGER,215 N. CRESPI AVE,EXETER,CA,462351710,GSA_MIGRATION,No,93221,5595584112,CA,FRESNO,PARTNER,"6051 N. FRESNO STREET, SUITE 101",93710,USA,BILL WILLIAMS,BCWILLIAMS@EIDEBAILLY.COM,EIDE BAILLY LLP,,450250958,,84,2024-01-21,2019-11-19,2019-12-23,2019-12-30,2019-11-19,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 10",local,,annual,2992667,UG,true,CENSUS,2019-11-19,BILL WILLIAMS,PARTNER -165000,2019-12-CENSUS-0000248475,NATIVE VILLAGE OF TAZLINA,TRIBAL ADMINISTRATOR,MARCIE SIMEON,TAZLINA.TRIBAL@GMAIL.COM,NATIVE VILLAGE OF TAZLINA,9078224375,TRIBAL ADMINISTRATOR,P.O. BOX 87,GLENALLEN,AK,920068390,GSA_MIGRATION,No,99588,9072742992,AK,ANCHORAGE,PRINCIPAL,"3000 C ST. N., SUITE 201",99503,USA,JOSEPH V. BERGENE,FEDCENSUS@ALTROGCO.COM,"ALTMAN, ROGERS & CO.",,920143182,,12,2024-01-21,2020-07-23,2020-07-24,2020-07-24,2020-07-23,2019-12-31,2018-12-31,2019,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,No,"66, 20, 39, 15, 12",tribal,,annual,933053,UG,true,CENSUS,2020-07-23,JOSEPH V. BERGENE,PRINCIPAL -166000,2019-06-CENSUS-0000099289,SOUTHEASTERN LOUISIANA AREA HEALTH EDUCATION CENTER,CFO,CHERYL TANNER,CHERYL.TANNER@SELAHEC.ORG,SOUTHEASTERN LOUISIANA AREA HEALTH EDUCATION CENTER,9853451119,CFO,1302 JW DAVIS DRIVE,HAMMOND,LA,721155014,GSA_MIGRATION,No,70403,5048355522,LA,METAIRIE,DIRECTOR,"111 VETERANS MEMORIAL BLVD, SUITE 600",70005,USA,"LANCE MORAN, CPA",LMORAN@LAPORTE.COM,"LAPORTE, APAC",,721088864,,84,2024-01-21,2020-02-01,2020-01-30,2020-02-02,2020-02-01,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 84",non-profit,,annual,3489073,UG,true,CENSUS,2020-02-01,"LANCE MORAN, CPA",DIRECTOR -167000,2019-11-CENSUS-0000138661,"MCDONOUGH COUNTY, ILLINOIS",COUNTY CLERK,GRETCHEN DEJAYNES,GDEJAYNES@MCDONOUGHCOUNTYCLERK.ORG,"MCDONOUGH COUNTY, ILLINOIS",3098332474,COUNTY CLERK,ONE COURTHOUSE SQUARE #7,MACOMB,IL,376001537,GSA_MIGRATION,No,61455,2177933363,IL,SPRINGFIELD,DIRECTOR,"3201 WEST WHITE OAKS DRIVE, SUITE 102",62704,USA,"MEGAN COCHRAN, CPA",MEGAN.COCHRAN@SIKICH.COM,SIKICH LLP,,363168081,,16,2024-01-21,2020-10-26,2020-10-22,2020-10-27,2020-10-26,2019-11-30,2018-11-30,2019,single-audit,unmodified_opinion,,,,No,Yes,No,No,Yes,750000,No,"93, 97, 20, 16, 10, 90, 66",local,,annual,856084,UG,true,CENSUS,2020-10-26,"MEGAN COCHRAN, CPA",DIRECTOR -168000,2019-09-CENSUS-0000192518,MARTIN HOUSING AUTHORITY,EXECUTIVE DIRECTOR,PAM BRATCHER,PAM@MARTINHOUSING.ORG,MARTIN HOUSING AUTHORITY,7315873186,EXECUTIVE DIRECTOR,134 EAST HEIGHTS DRIVE,MARTIN,TN,620719631,GSA_MIGRATION,No,38237,2059820992,AL,BIRMINGHAM,MEMBER,"200 CHASE PARK SOUTH, SUITE 220",35244,USA,"ROY W. HENDERSON, JR.",RHENDERSON@HENDERSONCPA.COM,"HENDERSON & PILLETERI, LLC",,263957106,,14,2024-01-21,2020-03-30,2020-03-31,2020-03-31,2020-03-30,2019-09-30,2018-09-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 97, 10, 14",local,,annual,1759625,UG,true,CENSUS,2020-03-30,"ROY W. HENDERSON, JR.",MEMBER -169000,2019-12-CENSUS-0000248376,"WILDBERRY MANOR PROJECT OF COMMUNITY HOUSING, INC.",MANAGER,BRENDA NABHOLZ,BNABHOLZ@HOTMAIL.COM,"WILDBERRY MANOR PROJECT OF COMMUNITY HOUSING, INC.",3193347110,MANAGER,302 1ST ST E,INDEPENDENCE,IA,420993731,GSA_MIGRATION,No,50644,5152258400,IA,WEST DES MOINES,PARTNER,"1601 22ND STREET, SUITE 400",50266,USA,MANDI HOLCOMB,MHOLCOMB@DENMAN-CPA.COM,"DENMAN & COMPANY, LLP",,420794029,,10,2024-01-21,2021-05-09,2020-08-04,2021-05-10,2021-05-09,2019-12-31,2018-12-31,2019,single-audit,unmodified_opinion,,,,No,Yes,Yes,No,No,750000,No,10,non-profit,,annual,984678,UG,true,CENSUS,2021-05-09,MANDI HOLCOMB,PARTNER -170000,2019-06-CENSUS-0000200348,"VICTIM OUTREACH INTERVENTION CENTER, INC.",EXECUTIVE DIRECTOR,LINDA STRACHAN,LINDAS@VOICEFORVICTIMS.COM,"VICTIM OUTREACH INTERVENTION CENTER, INC.",7242838700,EXECUTIVE DIRECTOR,"111 SOUTH CLIFF STREET, SUITE 1-A",BUTLER,PA,251364362,GSA_MIGRATION,No,16001,4126359088,PA,PITTSBURGH,CPA,"8150 PERRY HIGHWAY, SUITE 105",15237,USA,JAMES R. HUNE,JHUNE@CREESESMITH.COM,"CREESE, SMITH & CO. LLC",,251801538,,14,2024-01-21,2019-11-19,2019-11-19,2019-11-18,2019-11-19,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 16, 14",non-profit,,annual,1288547,UG,true,CENSUS,2019-11-19,JAMES R. HUNE,CPA -171000,2019-06-CENSUS-0000195886,LAKE VILLA COMMUNITY CONSOLIDATED SCHOOL DISTRICT NO 41,BUSINESS MANAGER,ANNA KASPRZYK,ANNA.KASPRZYK@DISTRICT41.ORG,LAKE VILLA COMMUNITY CONSOLIDATED SCHOOL DISTRICT NO 41,8472458010,BUSINESS MANAGER,131 MCKINLEY AVENUE,LAKE VILLA,IL,366004850,GSA_MIGRATION,No,60046,8153441300,IL,MCHENRY,PARTNER,"5400 WEST ELM STREET, SUITE 203",60050,USA,CHERYDEN JUERGENSEN,CPAS@EDERCASELLA.COM,"EDER, CASELLA & CO",,363614997,,84,2024-01-21,2019-10-07,2019-10-08,2019-10-07,2019-10-07,2019-06-30,2018-06-30,2019,single-audit,"adverse_opinion, not_gaap",regulatory_basis,No,unmodified_opinion,No,No,No,No,No,750000,No,"93, 10, 84",local,,annual,1183418,UG,true,CENSUS,2019-10-07,CHERYDEN JUERGENSEN,PARTNER -172000,2019-06-CENSUS-0000211402,"PRO-YOUTH, INC.",CHIEF FINANCIAL OFFICER,DOLORES TAYLOR,DTAYLOR@PYHEART.ORG,"PRO-YOUTH, INC.",5597989135,CHIEF FINANCIAL OFFICER,505 N. COURT STREET,VISALIA,CA,770337714,GSA_MIGRATION,No,93291,6618347411,CA,BAKERSFIELD,PARTNER,300 NEW STINE RD,93308,USA,NANCY BELTON,NANCY@DPVB.COM,DANIELLS PHILLIPS VAUGHAN AND BOCK,,952972229,,84,2024-01-21,2020-03-19,2020-03-19,2020-03-20,2020-03-19,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 10",non-profit,,annual,5362587,UG,true,CENSUS,2020-03-19,NANCY BELTON,PARTNER -173000,2019-06-CENSUS-0000170093,CONEMAUGH TOWNSHIP AREA SCHOOL DISTRICT,BUSINESS MANAGER,REGINA REMBOLD,REGINA.REMBOLD@CTASD.ORG,CONEMAUGH TOWNSHIP AREA SCHOOL DISTRICT,8144797431,BUSINESS MANAGER,300 WEST CAMPUS AVENUE,DAVIDSVILLE,PA,256004241,GSA_MIGRATION,No,159280407,8142694912,PA,JOHNSTOWN,CERTIFIED PUBLIC ACCOUNTANT,334 BUDFIELD STREET SUITE 180,159043214,USA,CAROLYN MUSSELMAN,CMUSSELMAN@KOTZANCPA.COM,"KOTZAN CPA & ASSOCIATES, P.C.",,262055351,,84,2024-01-21,2020-01-07,2020-01-08,2020-01-08,2020-01-07,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 10, 84",local,,annual,763390,UG,true,CENSUS,2020-01-07,CAROLYN MUSSELMAN,CERTIFIED PUBLIC ACCOUNTANT -174000,2019-12-CENSUS-0000043477,"LABOR COUNCIL SENIOR HOUSING, INC. 042-11164",MANAGING AGENT,STEVEN BOTNICK,SBOTNICK@BOTNICKREALTY.COM,"LABOR COUNCIL SENIOR HOUSING, INC. 042-11164",3308674836,MANAGING AGENT,1653 MERRIMAN ROAD,AKRON,OK,341399884,GSA_MIGRATION,No,44313,3308677350,OH,AKRON,SENIOR MANAGER,1540 WEST MARKET STREET,44313,USA,CHRISTINE KLAIBER,CKLAIBER@APPLEGROWTH.COM,APPLE GROWTH PARTNERS,,341082617,,14,2024-01-21,2020-05-18,2020-05-19,2020-05-19,2020-05-18,2019-12-31,2018-12-31,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,2349626,UG,true,CENSUS,2020-05-18,CHRISTINE KLAIBER,SENIOR MANAGER -175000,2019-06-CENSUS-0000111765,"SER - JOBS FOR PROGRESS OF SOUTHERN ARIZONA, INC.",EXECUTIVE DIRECTOR,ERNEST URIAS,SER7@QWESTOFFICE.NET,"SER - JOBS FOR PROGRESS OF SOUTHERN ARIZONA, INC.",5206248629,EXECUTIVE DIRECTOR,40 WEST 28TH ST.,TUCSON,AZ,860251307,GSA_MIGRATION,No,85713,6022643077,AZ,SCOTTSDALE,PARTNER,9019 E BAHIA DRIVE SUITE 100,85260,USA,KEVIN CAMBERG,KCAMBERG@F-CPC.COM,"FESTER & CHAPMAN, PLLC",,821455657,,17,2024-01-21,2020-03-15,2020-03-16,2020-03-10,2020-03-15,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"17, 93, 14",non-profit,,annual,1680310,UG,true,CENSUS,2020-03-15,KEVIN CAMBERG,PARTNER -176000,2019-06-CENSUS-0000201775,"CITY OF SOUTH BURLINGTON, VERMONT",FINANCE OFFICER,SUE DOREY,SDOREY@SBURL.COM,"CITY OF SOUTH BURLINGTON, VERMONT",8028464104,FINANCE OFFICER,575 DORSET STREET,SOUTH BURLINGTON,VT,036002712,GSA_MIGRATION,No,05403,2079294606,ME,BUXTON,QUALITY CONTROL SPECIALIST,3 OLD ORCHARD ROAD,04093,USA,CATHY MARKAVICH,CMARKAVICH@RHRSMITH.COM,RHR SMITH & COMPANY,,043383155,,20,2024-01-21,2020-04-14,2021-01-18,2021-01-18,2020-04-14,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"16, 14, 20",local,,annual,4273532,UG,true,CENSUS,2020-04-14,CATHY MARKAVICH,QUALITY CONTROL SPECIALIST -177000,2019-12-CENSUS-0000228105,AIRPORT AUTHORITY FOR AIPORT DISTRICT #1 OF CALCASIEU PARISH,EXECUTIVE DIRECTOR,HEATH ALLEN,HALLEN@FLYLAKECHARLES.COM,AIRPORT AUTHORITY FOR AIPORT DISTRICT #1 OF CALCASIEU PARISH,3374776051,EXECUTIVE DIRECTOR,P O DRAWER 5820,LAKE CHARLES,LA,720551224,GSA_MIGRATION,No,70606,3375134915,LA,LAKE CHARLES,CPA,"2720 RUE DE JARDIN, SUITE 300",70605,USA,STEVEN M DEROUEN,STEVE@SDEROUENCPA.COM,STEVEN M DEROUEN & ASSOCIATES,,800488055,,20,2024-01-21,2020-07-26,2020-07-27,2020-07-27,2020-07-26,2019-12-31,2018-12-31,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"97, 20",local,,annual,2880293,UG,true,CENSUS,2020-07-26,STEVEN M DEROUEN,CPA -178000,2020-06-CENSUS-0000152122,HEADWATERS REGIONAL DEVELOPMENT COMMISSION,FINANCE DIRECTOR,NICOLE MCKINNON,NMCKINNON@HRDC.ORG,HEADWATERS REGIONAL DEVELOPMENT COMMISSION,2184444732,FINANCE DIRECTOR,1320 NIELSON AVE SE,BEMIDJI,MN,410983661,GSA_MIGRATION,No,56601,2187516300,MN,BEMIDJI,PARTNER,513 BELTRAMI AVENUE,56601,USA,JOSH SANDEN,JSANDEN@MILLERMCDONALD.COM,"MILLER MCDONALD, INC.",,411281737,,11,2024-01-22,2022-04-24,2022-04-25,2022-04-13,2022-04-24,2020-06-30,2019-07-01,2020,single-audit,unmodified_opinion,,,,No,Yes,No,No,Yes,750000,No,"20, 10, 11",local,,annual,3119109,UG,true,CENSUS,2022-04-24,JOSH SANDEN,PARTNER -179000,2019-06-CENSUS-0000145513,UNIFIED SCHOOL DISTRICT NO. 428,SUPERINTENDENT,KHRIS THEXTON,KHRIS.THEXTON@USD428.NET,UNIFIED SCHOOL DISTRICT NO. 428,6207931500,SUPERINTENDENT,201 S PATTON,GREAT BEND,KS,480721637,GSA_MIGRATION,No,67530,6207922428,KS,GREAT BEND,CPA-PARTNER,2006 BROADWAY SUITE 2A,67530,USA,VICKIE DREILING,VDREILING@ABBB.COM,"ADAMS, BROWN, BERAN & BALL, CHTD.",,481040139,,84,2024-01-22,2020-01-26,2020-01-27,2020-01-27,2020-01-26,2019-06-30,2018-06-30,2019,single-audit,not_gaap,regulatory_basis,No,unmodified_opinion,No,No,Yes,No,No,750000,No,"10, 93, 84",local,,annual,4126613,UG,true,CENSUS,2020-01-26,VICKIE DREILING,CPA-PARTNER -180000,2017-09-CENSUS-0000056213,"TURNING POINT, INC.",CFO,JULIA BINGHAM,JBINGHAM@TURNINGPOINTMACOMB.ORG,"TURNING POINT, INC.",5864634430,CFO,158 S. MAIN STREET,MOUNT CLEMENS,MI,382292020,GSA_MIGRATION,No,480432309,2483688868,MI,SOUTHFIELD,PARTNER,"2000 TOWN CENTER, SUITE 900",480751103,USA,LAURIE L. HORVATH,LAURIE.HORVATH@BAKERTILLY.COM,"BAKER TILLY VIRCHOW KRAUSE, LLP",,380859910,,16,2024-01-27,2018-04-03,2018-04-04,2018-03-29,2018-04-03,2017-09-30,2016-09-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 97, 14, 16, 10",non-profit,,annual,1410903,UG,true,CENSUS,2018-04-03,LAURIE L. HORVATH,PARTNER -181000,2019-09-CENSUS-0000216900,COMMUNITY APARTMENTS CORPORATION OF FORSYTH COUNTY #3,DIRECTOR OF HOUSING SERVICES,LISA PARDUE,GSA_MIGRATION,COMMUNITY APARTMENTS CORPORATION OF FORSYTH COUNTY #3,3362734404,DIRECTOR OF HOUSING SERVICES,353 E. SIX FORKS ROAD SUITE 300,RALEIGH,NC,562277204,GSA_MIGRATION,No,27609,3368544277,NC,GREENSBORO,SHAREHOLDER,445 DOLLEY MADISON ROAD SUITE 202,27410,USA,BRETT KOCEJA,BRETT@AKACPA.COM,"APPLE, KOCEJA & ASSOCIATES, PA",,561793406,,14,2024-01-23,2020-03-22,2020-03-22,2020-03-17,2020-03-22,2019-09-30,2018-09-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,771829,UG,true,CENSUS,2020-03-22,BRETT KOCEJA,SHAREHOLDER -182000,2019-06-CENSUS-0000244276,"THE ARC OF CAPE MAY COUNTY, INC AND AFFILIATE",DIRECTOR OF FINANCE,BARBARA MARCHIANO,BMARCHIANO@ARCOFCAPEMAY.ORG,"THE ARC OF CAPE MAY COUNTY, INC AND AFFILIATE",6098617100,DIRECTOR OF FINANCE,"822 ROUTE 47, P.O. BOX 255",SOUTH DENNIS,NJ,222271013,GSA_MIGRATION,No,08245,6096414000,NJ,NORTHFIELD,PARTNER,332 TILTON ROAD,08225,USA,DONNA BUZBY,DHBUZBY@CAPALDIREYNOLDS.COM,"CAPALDI REYNOLDS & PELOSI, CPA PA",,222097516,,14,2024-01-23,2020-02-25,2020-02-26,2020-01-15,2020-02-25,2019-06-30,2018-06-30,2019,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,942033,UG,true,CENSUS,2020-02-25,DONNA BUZBY,PARTNER -183000,2018-06-CENSUS-0000031006,"NEW BRIDGE FOUNDATION, INC.",DIRECTOR OF ADMINISTRATION,NATHALIE GIREAUD FERKO,NATHALIE@NEWBRIDGEFOUNDATION.ORG,"NEW BRIDGE FOUNDATION, INC.",8007852400,DIRECTOR OF ADMINISTRATION,2323 HEARST AVENUE,BERKELEY,CA,237131209,GSA_MIGRATION,No,94709,5104525051,CA,OAKLAND,MANAGING PARTNER,7677 OAKPORT STREET,94621,USA,SANWAR HARSHWAL,SANWAR@HARSHWAL.COM,"HARSHWAL & COMPANY, LLP",,270741376,,93,2024-01-26,2019-03-24,2019-03-24,2019-03-22,2019-03-24,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,93,non-profit,,annual,1791503,UG,true,CENSUS,2019-03-24,SANWAR HARSHWAL,MANAGING PARTNER -184000,2018-03-CENSUS-0000224763,MAGNOLIA HEIGHTS HOUSING CORPORATION 084-HD051 M016Q041005,CHIEF FINANCIAL OFFICER,NATHAN MORDICA,NMMORDICA@FREEMANHEALTH.COM,MAGNOLIA HEIGHTS HOUSING CORPORATION 084-HD051 M016Q041005,4173477606,CHIEF FINANCIAL OFFICER,1105 EAST 32ND STREET,JOPLIN,MO,470950622,GSA_MIGRATION,No,64804,4178658701,MO,SPRINGFIELD,PARTNER,"910 E. SAINT LOUIS STREET, P.O. BOX 1190",658011190,USA,KRISTEN BRIGHT,KBRIGHT@BKD.COM,"BKD, LLP",,440160260,,14,2024-01-26,2018-07-02,2018-07-03,2018-06-28,2018-07-02,2018-03-31,2017-03-31,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,2422017,UG,true,CENSUS,2018-07-02,KRISTEN BRIGHT,PARTNER -185000,2018-06-CENSUS-0000091952,"BLUE GRASS COMMUNITY ACTION PARTNERSHIP, INC.",EXECUTIVE DIRECTOR,TROY ROBERTSQ,TROY.ROBERTS@BGCAP.ORG,"BLUE GRASS COMMUNITY ACTION PARTNERSHIP, INC.",5026954290,EXECUTIVE DIRECTOR,111 PROFESSIONAL COURT,FRANKFORT,KY,610659583,GSA_MIGRATION,No,40601,8592544427,KY,LEXINGTON,PARTNER,190 MARKET STREET,40507,USA,ANDREW DIERUF,ADIERUF@BANDDCPA.COM,"BESTEN & DIERUF, PLLC",,453048812,,93,2024-01-26,2018-12-11,2018-12-11,2018-12-12,2018-12-11,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 81, 97, 17, 94, 20, 93",non-profit,,annual,9674474,UG,true,CENSUS,2018-12-11,ANDREW DIERUF,PARTNER -186000,2018-06-CENSUS-0000191547,STARR KING SCHOOL FOR THE MINISTRY,VP OF FINANCE AND ADMINISTRATION,R. KELLEY GIBBS,KGIBBS@SKSM.EDU,STARR KING SCHOOL FOR THE MINISTRY,5105494702,VP OF FINANCE AND ADMINISTRATION,2441 LE CONTE AVENUE,BERKELEY,CA,941196217,GSA_MIGRATION,No,94709,4158196718,CA,SAN FRANCISCO,OWNER,PO BOX 641773,941641773,USA,KENNETH MIERZWINSKI,KEN@KPMCPAS.COM,KPM ACCOUNTING & MANAGEMENT SOLUTIONS,,453220824,,84,2024-01-26,2019-03-24,2019-03-24,2019-03-25,2019-03-24,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,84,non-profit,,annual,1010464,UG,true,CENSUS,2019-03-24,KENNETH MIERZWINSKI,OWNER -188000,2018-06-CENSUS-0000247990,"VOICES FOR CHILDREN FOUNDATION, INC.",CONTROLLER,ALAIN GARCIA,AGARCIA@BEAVOICE.ORG,"VOICES FOR CHILDREN FOUNDATION, INC.",7864693863,CONTROLLER,"601 NW 1ST CT, 10TH FL",MIAMI,FL,592746076,GSA_MIGRATION,No,33136,3058585800,FL,MIAMI,CERTIFIED PUBLIC ACCOUNTANT,"3225 AVIATION AVENUE, SUITE 500",33133,USA,DARWISH KAIYAL,GSA_MIGRATION,PINCHASIK YELEN MUSKAT STEIN,,274256127,,16,2024-01-26,2019-02-27,2019-02-28,2019-02-28,2019-02-27,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,16,non-profit,,annual,949658,UG,true,CENSUS,2019-02-27,DARWISH KAIYAL,CERTIFIED PUBLIC ACCOUNTANT -189000,2018-12-CENSUS-0000246173,"SAINT JAMES HEALTH, INC.",CFO,MARCUS FREEMAN,MFREEMAN@SJHEALTHNEWARK.COM,"SAINT JAMES HEALTH, INC.",9737898111,CFO,228 LAFAYETTE STREET,NEWARK,NJ,465221991,GSA_MIGRATION,No,07105,9734726250,NJ,CLIFTON,PARTNER,855 VALLEY ROAD,07013,USA,JOSH CHANANIE,JCHANANIE@SAXLLP.COM,SAX LLP,,818950760,,93,2024-01-26,2019-07-31,2019-08-01,2019-07-25,2019-07-31,2018-12-31,2017-12-31,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,93,non-profit,,annual,1079809,UG,true,CENSUS,2019-07-31,JOSH CHANANIE,PARTNER -190000,2018-12-CENSUS-0000192752,"ROCK VALLEY COMMUNITY PROGRAMS, INC.",EXECUTIVE DIRECTOR,ANGEL EGGERS,AEGGERS@RVCP.ORG,"ROCK VALLEY COMMUNITY PROGRAMS, INC.",6087414500,EXECUTIVE DIRECTOR,203 W. SUNNY LANE ROAD,JANESVILLE,WI,391438843,GSA_MIGRATION,No,535120932,8153162375,IL,ROCKFORD,PARTNER,6815 WEAVER ROAD SUITE SUITE 300,611148060,USA,CURTIS D. KLECKLER,CKLECKLER@BENNINGGROUP.COM,"BENNING GROUP, LLC",,300003878,,64,2024-01-26,2019-07-25,2019-07-25,2019-07-26,2019-07-25,2018-12-31,2017-12-31,2018,single-audit,unmodified_opinion,,,,Yes,No,No,No,Yes,750000,No,"93, 16, 64",non-profit,,annual,4451969,UG,true,CENSUS,2019-07-25,CURTIS D. KLECKLER,PARTNER -191000,2018-12-CENSUS-0000187800,TEXARKANA AIRPORT AUTHORITY,DIRECTOR,MARK MELLINGER,MARK.MELLINGER@TXKAIRPORT.COM,TEXARKANA AIRPORT AUTHORITY,8707742171,DIRECTOR,201 AIRPORT DRIVE,TEXARKANA,AR,716045823,GSA_MIGRATION,No,71854,9037935646,TX,TEXARKANA,SHAREHOLDER,1810 GALLERIA OAKS,75503,USA,"KELLY BIRDWELL, CPA",KELLY@WILHEN.COM,"WILF & HENDERSON, P.C.",,751726749,,20,2024-01-26,2019-07-28,2019-07-29,2019-07-29,2019-07-28,2018-12-31,2017-12-31,2018,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,No,20,local,,annual,759045,UG,true,CENSUS,2019-07-28,"KELLY BIRDWELL, CPA",SHAREHOLDER -192000,2018-06-CENSUS-0000211066,CLEVELAND HEIGHTS-UNIVERSITY HEIGHTS CITY SCHOOL DISTRICT,TREASURER,A. SCOTT GAINER,S_GAINER@CHUH.ORG,CLEVELAND HEIGHTS-UNIVERSITY HEIGHTS CITY SCHOOL DISTRICT,2163717171,TREASURER,2155 MIRAMAR BOULEVARD,UNIVERSITY HEIGHTS,OH,346000687,GSA_MIGRATION,No,44118,2167873665,OH,COLUMBUS,SENIOR AUDIT MANAGER,88 EAST BROAD STREET,43215,USA,MATTHEW GOLDMAN,MAGOLDMAN@OHIOAUDITOR.GOV,"DAVE YOST, AUDITOR OF STATE",,311334820,,84,2024-01-26,2019-01-27,2019-01-28,2019-01-26,2019-01-27,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,Yes,No,750000,Yes,"10, 84",local,,annual,8195043,UG,true,CENSUS,2019-01-27,MATTHEW GOLDMAN,SENIOR AUDIT MANAGER -193000,2017-10-CENSUS-0000182894,OBLATES RESIDENCES 042-11193-PM,VICE PRESIDENT,KATHLEEN CROWLEY,KCROWLEY@VMC.ORG,OBLATES RESIDENCES 042-11193-PM,4192422300,VICE PRESIDENT,1931 SCOTTWOOD AVENUE SUITE 700,TOLEDO,OH,341320478,GSA_MIGRATION,No,43620,3178485700,IN,CARMEL,PRINCIPAL,"501 CONGRESSIONAL BLVD, SUITE 300",46032,USA,KRISTEN KELLEHER,KKELLEHER@DOZ.NET,"DAUBY O'CONNOR & ZALESKI, LLC",,351750664,,14,2024-01-27,2018-01-23,2018-01-24,2018-01-24,2018-01-23,2017-10-31,2016-10-31,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,14,non-profit,,annual,3889656,UG,true,CENSUS,2018-01-23,KRISTEN KELLEHER,PRINCIPAL -194000,2018-06-CENSUS-0000187947,"GRAND PRAIRIE SERVICES, INC.",PRESIDENT & CEO,SHARRONNE WARD,SWARD@GPSBH.ORG,"GRAND PRAIRIE SERVICES, INC.",7084441012,PRESIDENT & CEO,17746 OAK PARK AVENUE,TINLEY PARK,IL,362362364,GSA_MIGRATION,No,60477,6304064490,IL,BATAVIA,PRESIDENT,201 HOUSTON STREET SUITE 301,60510,USA,PAUL WIELAND,PAUL@WIELANDCPAS.NET,"WIELAND & COMPANY, INC.",,364025026,,93,2024-01-26,2019-03-20,2019-03-21,2019-03-21,2019-03-20,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,93,non-profit,,annual,1213940,UG,true,CENSUS,2019-03-20,PAUL WIELAND,PRESIDENT -195000,2018-06-CENSUS-0000189752,"CHEROKEE FAMILY VIOLENCE CENTER, INC.",EXECUTIVE DIRECTOR,MEG ROGERS,MEG@CFVC.ORG,"CHEROKEE FAMILY VIOLENCE CENTER, INC.",7705477346,EXECUTIVE DIRECTOR,100 HEARTHSTONE LANDING DRIVE,CANTON,GA,581650925,GSA_MIGRATION,No,30114,7067782154,GA,CLARKESVILLE,AUDIT PARTNER,854 WASHINGTON STREET SUITE 200,30523,USA,JOELY MIXON,JOELY@MIXONCPA.COM,"MIXON, MIXON, BROWN & TENCH CPAS",,823104745,,16,2024-01-26,2018-12-25,2018-12-26,2018-12-26,2018-12-25,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"16, 93, 97, 14",non-profit,,annual,1076770,UG,true,CENSUS,2018-12-25,JOELY MIXON,AUDIT PARTNER -196000,2018-06-CENSUS-0000145974,POWELL COUNTY FISCAL COURT,COUNTY TREASURER,CONNIE CRABTREE,RCRABTREE33@HOTMAIL.COM,POWELL COUNTY FISCAL COURT,6066632834,COUNTY TREASURER,525 WASHINGTON STREET,STANTON,KY,616016937,GSA_MIGRATION,No,40380,5025645841,KY,FRANKFORT,ASSISTANT AUDITOR OF PUBLIC ACCOUNTS,209 ST. CLAIR STREET,40601,USA,FARRAH PETTER,FARRAH.PETTER@KY.GOV,"MIKE HARMON, KY AUDITOR OF PUBLIC ACCOUNTS",,610600439,,97,2024-01-26,2019-10-17,2019-10-18,2019-10-14,2019-10-17,2018-06-30,2017-06-30,2018,single-audit,not_gaap,regulatory_basis,No,unmodified_opinion,No,Yes,No,Yes,No,750000,No,"14, 20, 97",local,,annual,851352,UG,true,CENSUS,2019-10-17,FARRAH PETTER,ASSISTANT AUDITOR OF PUBLIC ACCOUNTS -197000,2018-06-CENSUS-0000083404,LIMESTONE COLLEGE,VP FOR FINANCIAL AFFAIRS,WAYDE DAWSON,LDAWSON@LIMESTONE.EDU,LIMESTONE COLLEGE,8644884522,VP FOR FINANCIAL AFFAIRS,1115 COLLEGE DRIVE,GAFFNEY,SC,570314402,GSA_MIGRATION,No,29340,8642405166,SC,GREENVILLE,DIRECTOR,"110 EAST COURT STREET, SUITE 500",29601,USA,ROSELLE BONNOITT,RBONNOITT@CBH.COM,CHERRY BEKAERT LLP,,560574444,,84,2024-01-26,2018-10-10,2018-10-11,2018-10-01,2018-10-10,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,84,non-profit,,annual,26788199,UG,true,CENSUS,2018-10-10,ROSELLE BONNOITT,DIRECTOR -198000,2018-09-CENSUS-0000172888,"CITY OF EULESS, TEXAS",DIRECTOR OF FINANCE,JANINA JEWELL,JJEWELL@EULESSTX.GOV,"CITY OF EULESS, TEXAS",8176851444,DIRECTOR OF FINANCE,201 N. ECTOR DRIVE,EULESS,TX,756004644,GSA_MIGRATION,No,76039,9724901970,TX,DALLAS,PARTNER,"2300 NORTH FIELD STREET, SUITE 1000",75201,USA,SARA DEMPSEY,SARA.DEMPSEY@WEAVER.COM,"WEAVER AND TIDWELL, L.L.P.",,750786316,,16,2024-01-26,2019-03-11,2019-03-12,2019-03-12,2019-03-11,2018-09-30,2017-09-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"16, 97, 66, 45, 20",local,,annual,801977,UG,true,CENSUS,2019-03-11,SARA DEMPSEY,PARTNER -199000,2018-06-CENSUS-0000166560,DOUGLAS EDUCATION SERVICE DISTRICT,FINANCE DIRECTOR,BARBARA TAYLOR,BARBARA.TAYLOR@DOUGLASESD.K12.OR.US,DOUGLAS EDUCATION SERVICE DISTRICT,5414404752,FINANCE DIRECTOR,1871 NE STEPHENS ST,ROSEBURG,OR,936000405,GSA_MIGRATION,No,974701433,5416724886,OR,ROSEBURG,CERTIFIED PUBLIC ACCOUNTANT,2500 W. HARVARD AVE,974712507,USA,JEFFREY R. COOLEY,JEFF@NEUNERDAVIDSON.COM,NEUNER DAVIDSON & CO,,930764156,,84,2024-01-26,2018-12-27,2018-12-26,2018-12-21,2018-12-27,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 93",local,,annual,2521742,UG,true,CENSUS,2018-12-27,JEFFREY R. COOLEY,CERTIFIED PUBLIC ACCOUNTANT -200000,2018-06-CENSUS-0000226783,MARLBOROUGH COMMUNITY DEVELOPMENT AUTHORITY,EXECUTIVE DIRECTOR,DOUGLAS BUSHMAN,DBUSHMAN@MARLBOROUGH-MA.GOV,MARLBOROUGH COMMUNITY DEVELOPMENT AUTHORITY,5086240980,EXECUTIVE DIRECTOR,"255 MAIN STREET, SUITE 212",MARLBOROUGH,MA,046001399,GSA_MIGRATION,No,01752,6173766226,MA,BRAINTREE,DIRECTOR,"100 GRANDVIEW ROAD, SUITE 320",02184,USA,MICHAEL GUYDER,MGUYDER@GUYDER-HURLEY.COM,"GUYDER HURLEY, PC",,043520245,,14,2024-01-26,2019-01-06,2019-01-04,2019-01-07,2019-01-06,2018-06-30,2017-06-30,2018,program-specific,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,local,,annual,1580926,UG,true,CENSUS,2019-01-06,MICHAEL GUYDER,DIRECTOR -201000,2018-06-CENSUS-0000233189,RICHMOND HEIGHTS LOCAL SCHOOL DISTRICT,TREASURER,COOPER MARTIN,MARTIN.COOPER@RICHMONDHEIGHTSSCHOOLS.ORG,RICHMOND HEIGHTS LOCAL SCHOOL DISTRICT,2166920089,TREASURER,447 RICHMOND ROAD,RICHMOND HEIGHTS,OH,346002300,GSA_MIGRATION,No,44143,2167873665,OH,COLUMBUS,SENIOR AUDIT MANAGER,88 EAST BROAD STREET,43215,USA,WILLIAM WARD,WJWARD@OHIOAUDITOR.GOV,"KEITH FABER, AUDITOR OF STATE",,311334820,,84,2024-01-26,2019-03-25,2019-03-26,2019-03-26,2019-03-25,2018-06-30,2017-06-30,2018,single-audit,"unmodified_opinion, qualified_opinion",,,,No,Yes,No,Yes,No,750000,Yes,"10, 84",local,,annual,1058244,UG,true,CENSUS,2019-03-25,WILLIAM WARD,SENIOR AUDIT MANAGER -202000,2017-06-CENSUS-0000183467,UNION COUNTY SCHOOL DISTRICT NO. 1,BUSINESS MANAGER,CHRIS PANIKE,CHRIS.PANIKE@LAGRANDESD.ORG,UNION COUNTY SCHOOL DISTRICT NO. 1,5416633202,BUSINESS MANAGER,1305 N WILLOW STREET,LA GRANDE,OR,936000972,GSA_MIGRATION,No,97850,5419634191,OR,LA GRANDE,OWNER/MEMBER,PO BOX 1024,97850,USA,"CHELSEA HERRON, CPA",CHERRON@LAGRANDECPA.COM,"LEWIS, POE, MOELLER, GUNDERSON & ROBERTS, LLC",,931245066,,84,2024-01-27,2017-12-19,2017-12-20,2017-12-19,2017-12-19,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,Yes,"84, 10",local,,annual,2080149,UG,true,CENSUS,2017-12-19,"CHELSEA HERRON, CPA",OWNER/MEMBER -203000,2018-06-CENSUS-0000232362,ARLINGTON STREET PEOPLE'S ASSISTANCE NETWORK AND AFFILIATE,EXECUTIVE DIRECTOR,KATHLEEN SIBERT,KSIBERT@A-SPAN.ORG,ARLINGTON STREET PEOPLE'S ASSISTANCE NETWORK AND AFFILIATE,7032287807,EXECUTIVE DIRECTOR,2020 14TH STREET NORTH,ARLINGTON,VA,541615993,GSA_MIGRATION,No,22201,7038930026,VA,VIENNA,PRINCIPAL,"8300 BOONE BLVD, SUITE 600",22182,USA,JORGE DIAZ,JDIAZ@ROGERSPLLC.COM,ROGERS & COMPANY PLLC,,585676261,,14,2024-01-26,2018-12-02,2018-12-03,2018-12-03,2018-12-02,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,1428871,UG,true,CENSUS,2018-12-02,JORGE DIAZ,PRINCIPAL -204000,2018-06-CENSUS-0000185583,"GRANVILLE PLAZA, INC. 053-EE001",ASSISTANT VICE PRESIDENT,JOHN NICHOLS,TVERNON@NCHSM.ORG,"GRANVILLE PLAZA, INC. 053-EE001",3367653906,ASSISTANT VICE PRESIDENT,750 BETHESDA ROAD,WINSTON-SALEM,NC,582004370,GSA_MIGRATION,No,27103,3362944494,NC,GREENSBORO,PARTNER,"1501 HIGHWOODS BLVD., SUITE 300",27410,USA,ERICA VERNON,EVERNON@BRCCPA.COM,"BERNARD ROBINSON & COMPANY, L.L.P.",,560571159,,14,2024-01-26,2019-01-02,2019-01-02,2019-01-03,2019-01-02,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,2368407,UG,true,CENSUS,2019-01-02,ERICA VERNON,PARTNER -205000,2018-06-CENSUS-0000146445,BOONE COUNTY BOARD OF EDUCATION,FINANCE DIRECTOR,LINDA SCHILD,LINDA.SCHILD@BOONE.KYSCHOOLS.US,BOONE COUNTY BOARD OF EDUCATION,8592822938,FINANCE DIRECTOR,8330 U.S. HIGHWAY 42,FLORENCE,KY,616001252,GSA_MIGRATION,No,41042,8593446400,KY,CRESTVIEW HILLS,DIRECTOR,2617 LEGENDS WAY,41017,USA,"GEORGE S. SPARKS, JR.",GSPARKS@BARNESDENNIG.COM,BARNES DENNIG,,311119890,,84,2024-01-26,2018-12-11,2018-12-12,2018-12-12,2018-12-11,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 10, 93",state,,annual,12054169,UG,true,CENSUS,2018-12-11,"GEORGE S. SPARKS, JR.",DIRECTOR -206000,2018-09-CENSUS-0000070523,"NORTHWEST KANSAS AREA AGENCY ON AGING, INC.",EXECUTIVE DIRECTOR,MICHELLE MORGAN,MMORGAN@NWKAAA.ORG,"NORTHWEST KANSAS AREA AGENCY ON AGING, INC.",7856288204,EXECUTIVE DIRECTOR,510 WEST 29TH STREET,HAYS,KS,480874448,GSA_MIGRATION,No,67601,7856283046,KS,HAYS,PARTNER,PO BOX 1186,67601,USA,MEAGAN M WELLBROCK,MWELLBROCK@ABBB.COM,"ADAMS, BROWN, BERAN & BALL CHTD.",,481040139,,93,2024-01-26,2019-01-02,2019-01-03,2019-01-03,2019-01-02,2018-09-30,2017-09-30,2018,single-audit,not_gaap,other_basis,No,unmodified_opinion,No,Yes,No,No,No,750000,No,"93, 10",non-profit,,annual,1383333,UG,true,CENSUS,2019-01-02,MEAGAN M WELLBROCK,PARTNER -207000,2018-09-CENSUS-0000220164,"MONTGOMERY COUNTY, MISSISSIPPI",CHANCERY CLERK,RYAN WOOD,RYANWOOD@MONTGOMERYCOUNTYMS.COM,"MONTGOMERY COUNTY, MISSISSIPPI",6622832233,CHANCERY CLERK,P.O. BOX 71,WINONA,MS,646000851,GSA_MIGRATION,No,38967,6015762800,MS,JACKSON,"DIRECTOR, COUNTY AUDIT SECTION",P.O. BOX 956,39205,USA,JOE MCKNIGHT,JOE.MCKNIGHT@OSA.MS.GOV,OFFICE OF THE STATE AUDITOR,,646000742,,23,2024-01-27,2020-04-20,2020-04-21,2020-04-21,2020-04-20,2018-09-30,2017-09-30,2018,single-audit,"unmodified_opinion, adverse_opinion",,,,No,Yes,No,No,No,750000,No,"23, 20, 97, 16, 14",local,,annual,1140725,UG,true,CENSUS,2020-04-20,JOE MCKNIGHT,"DIRECTOR, COUNTY AUDIT SECTION" -208000,2018-06-CENSUS-0000134269,SAN MATEO UNION HIGH SCHOOL DISTRICT,DIRECTOR OF BUDGET AND FISCAL SERVICES,VALERIE MILLER,VMILLER@SMUHSD.ORG,SAN MATEO UNION HIGH SCHOOL DISTRICT,6505582223,DIRECTOR OF BUDGET AND FISCAL SERVICES,650 NORTH DELAWARE STREET,SAN MATEO,CA,942700919,GSA_MIGRATION,No,94401,9169939494,CA,SACRAMENTO,PARTNER,701 HOWE AVE STE E3,95825,USA,DAVID BECKER,DBECKER@JPMCPA.COM,JAMES MARTA & COMPANY LLP,,271682261,,84,2024-01-27,2019-01-02,2018-12-17,2019-01-03,2019-01-02,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 84",local,,annual,4123164,UG,true,CENSUS,2019-01-02,DAVID BECKER,PARTNER -209000,2018-12-CENSUS-0000242324,HAWAII RURAL WATER ASSOCIATION,EXECUTIVE DIRECTOR,MAHANA GOMES,MAHANA@HAWAIIRWA.ORG,HAWAII RURAL WATER ASSOCIATION,8083158925,EXECUTIVE DIRECTOR,"65-1158 MAMALAHOA HWY, STE 2D",KAMUELA,HI,270711752,GSA_MIGRATION,No,96743,8089615532,HI,HILO,PRINCIPAL,PO BOX 6691,96720,USA,ANN FUKUHARA,ANN.FUKUHARA@HAWAIIANTEL.NET,"ANN FUKUHARA CPA MBA, AN ACCOUNTING CORPORATION",,990350770,,10,2024-01-27,2019-05-28,2019-05-27,2019-05-28,2019-05-28,2018-12-31,2017-12-31,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"66, 10",non-profit,,annual,905543,UG,true,CENSUS,2019-05-28,ANN FUKUHARA,PRINCIPAL -210000,2018-06-CENSUS-0000135408,LAKE COUNTY SCHOOL DISTRICT R-1,ACCOUNTING MANAGER,RENA SANCHEZ,RSANCHEZ@LAKECOUNTYSCHOOLS.NET,LAKE COUNTY SCHOOL DISTRICT R-1,7194866812,ACCOUNTING MANAGER,328 WEST 5TH STREET,LEADVILLE,CO,846011994,GSA_MIGRATION,No,804613661,9702453000,CO,GRAND JUNCTION,PRINCIPAL,2499 HWY 6&50,815051106,USA,LISA HEMANN,LISAH@CSDCPA.COM,"CHADWICK, STEINKIRCHNER, DAVIS & CO., P.C.",,840865725,,93,2024-01-27,2019-02-03,2019-02-04,2019-02-04,2019-02-03,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,No,"10, 93, 84",local,,annual,2024613,UG,true,CENSUS,2019-02-03,LISA HEMANN,PRINCIPAL -211000,2018-12-CENSUS-0000209247,ATLANTIC INSTITUTE OF ORIENTAL MEDICINE,FINANCIAL OFFICER,CELIA MUNOZ,CELIAMUNOZ@ATOM.EDU,ATLANTIC INSTITUTE OF ORIENTAL MEDICINE,9547639840,FINANCIAL OFFICER,100 EAST BORWARD BLVD #100,FORT LAUDERDALE,FL,650512087,GSA_MIGRATION,No,33301,3057613132,FL,MIAMI,PARTNER,10250 SW 56 STREET # B201,33165,USA,DOMIMGO ALVAREZ,DOMINGO@ALVAREZANDMENDOSA.COM,ALVAREZ & MENDOZA PA,,203162550,,84,2024-01-27,2019-06-20,2019-06-20,2019-06-20,2019-06-20,2018-12-31,2017-12-31,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,84,non-profit,,annual,3012801,UG,true,CENSUS,2019-06-20,DOMIMGO ALVAREZ,PARTNER -212000,2018-12-CENSUS-0000245306,"NEW YORK GENOME CENTER, INC.","SENIOR VICE PRESIDENT, FINANCE AND ADMIN",ANA BASTIANI-POSNER,ABASTIANI@NYGENOME.ORG,"NEW YORK GENOME CENTER, INC.",6469777343,"SENIOR VICE PRESIDENT, FINANCE AND ADMIN",101 AVENUE OF THE AMERICAS,NEW YORK,NY,800631734,GSA_MIGRATION,No,10013,6314256053,NY,MELVILLE,PARTNER,1305 WALT WHITMAN,11747,USA,EDWARD LEE,ENLEE@KPMG.COM,KPMG LLP,,135565207,,93,2024-01-27,2019-09-26,2019-09-27,2019-09-27,2019-09-26,2018-12-31,2017-12-31,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"47, 93, 12",non-profit,,annual,17318302,UG,true,CENSUS,2019-09-26,EDWARD LEE,PARTNER -213000,2018-06-CENSUS-0000133843,EAST WHITTIER CITY SCHOOL DISTRICT,"ASSISTANT SUPERINTENDENT, BUSINESS SERVI",RUBEN HERNANDEZ,RHERNANDEZ@EWCSD.ORG,EAST WHITTIER CITY SCHOOL DISTRICT,5629075900,"ASSISTANT SUPERINTENDENT, BUSINESS SERVI",14535 E WHITTIER BLVD,WHITTIER,CA,956001046,GSA_MIGRATION,No,90605,6192708222,CA,SAN DIEGO,PARTNER,348 OLIVE STREET,92103,USA,CHRISTY WHITE,CWHITE@CHRISTYWHITE.COM,CHRISTY WHITE ASSOCIATES,,272956198,,10,2024-01-27,2019-01-24,2019-01-25,2019-01-04,2019-01-24,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 10, 93",local,,annual,6741671,UG,true,CENSUS,2019-01-24,CHRISTY WHITE,PARTNER -214000,2018-12-CENSUS-0000157275,CAMDEN COUNTY MUNICIPAL UTILITIES AUTHORITY,CHIEF FINANCIAL OFFICER,WAYNE PLANAMENTO,WAYNE@CCMUA.ORG,CAMDEN COUNTY MUNICIPAL UTILITIES AUTHORITY,8565413700,CHIEF FINANCIAL OFFICER,1645 FERRY AVENUE,CAMDEN,NJ,222003702,GSA_MIGRATION,No,08104,8564356200,NJ,VOORHEES,PARTNER,601 WHITE HORSE ROAD,08043,USA,CAROL A. MCALLISTER,CMCALLISTER@BOWMANLLP.COM,BOWMAN & COMPANY LLP,,210658561,,66,2024-01-27,2019-11-19,2019-11-18,2019-12-06,2019-11-19,2018-12-31,2017-12-31,2018,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,66,local,,annual,27337045,UG,true,CENSUS,2019-11-19,CAROL A. MCALLISTER,PARTNER -215000,2017-08-CENSUS-0000174370,HIDALGO INDEPENDENT SCHOOL DISTRICT,CFO,NANCY FLORES,NASANCHEZ@HIDALGO-ISD.ORG,HIDALGO INDEPENDENT SCHOOL DISTRICT,9568434413,CFO,324 E FLORA ST,HIDALGO,TX,746001085,GSA_MIGRATION,No,78557,9565447778,TX,BROWNSVILLE,ENGAGEMENT PARTNER,765 E 7TH STREET,78520,USA,CARLOS CASCOS,CCASCOS@PBHCPA.COM,PATILLO BROWN HILL LLP,,742751720,,12,2024-01-27,2018-02-08,2018-02-09,2018-02-09,2018-02-08,2017-08-31,2016-08-31,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 10, 12, 84",local,,annual,6055809,UG,true,CENSUS,2018-02-08,CARLOS CASCOS,ENGAGEMENT PARTNER -216000,2017-12-CENSUS-0000128718,"MUTUAL HOUSING ASSOCIATION OF HAWAII, INC.",EXECUTIVE DIRECTOR,DAVID NAKAMURA,DNAKAMURA@MUTUAL-HOUSING.ORG,"MUTUAL HOUSING ASSOCIATION OF HAWAII, INC.",8085500804,EXECUTIVE DIRECTOR,"900 FORT STREET, SUITE 1690",HONOLULU,HI,990308739,GSA_MIGRATION,No,96813,8085242255,HI,HONOLULU,AUDIT SENIOR MANAGER,"1001 BISHOP STREET, ASB TOWER, SUITE 1700",96813,USA,JOHN BAUTISTA,JBAUTISTA@NKCPA.COM,"N&K CPAS, INC.",,990169131,,14,2024-01-27,2018-06-19,2018-06-20,2018-06-20,2018-06-19,2017-12-31,2016-12-31,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"21, 14",non-profit,,annual,5308000,UG,true,CENSUS,2018-06-19,JOHN BAUTISTA,AUDIT SENIOR MANAGER -217000,2017-06-CENSUS-0000238317,FRIENDS OUTSIDE IN LOS ANGELES COUNTY,BOOKKEEPER,MARILU CLARIDGE,MCLARIDGE@FRIENDSOUTSIDELA.ORG,FRIENDS OUTSIDE IN LOS ANGELES COUNTY,6267957607,BOOKKEEPER,"261 E. COLORADO BLVD., SUITE 217",PASADENA,CA,953557032,GSA_MIGRATION,No,91101,3102884213,CA,LOS ANGELES,PRINCIPAL,"1925 CENTURY PARK EAST, 16TH FLOOR",90067,USA,TAYIIKA M. DENNIS,TAYIIKA.DENNIS@CLACONNECT.COM,CLIFFTONLARSONALLEN LLP,,410746749,,93,2024-01-27,2018-03-25,2018-03-24,2018-03-26,2018-03-25,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"17, 93",non-profit,,annual,1759579,UG,true,CENSUS,2018-03-25,TAYIIKA M. DENNIS,PRINCIPAL -218000,2017-12-CENSUS-0000200376,CREATIVE ALTERNATIVES SENIOR CITIZEN OPPORTUNITES 136-EH103-NP-L8-WAH,CONTROLLER,CADE SCHOLL,CSCHOLL@MERCYHOUSING.ORG,CREATIVE ALTERNATIVES SENIOR CITIZEN OPPORTUNITES 136-EH103-NP-L8-WAH,3038303300,CONTROLLER,"2512 RIVER PLAZA DRIVE, SUITE 200",SACRAMENTO,CA,770030850,GSA_MIGRATION,No,95833,9168045367,CA,EL DORADO HILLS,SOLE PROPRIETOR,3811 TILDEN DRIVE,95762,USA,THOMAS TOMASZEWSKI,TOMCPA@DIRECTCON.NET,THOMAS TOMASZEWSKI CPA,,680466584,,14,2024-01-27,2018-04-24,2018-04-18,2018-04-23,2018-04-24,2017-12-31,2016-12-31,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,state,,annual,853035,UG,true,CENSUS,2018-04-24,THOMAS TOMASZEWSKI,SOLE PROPRIETOR -219000,2017-06-CENSUS-0000161106,MONTGOMERY COUNTY BOARD OF EDUCATION,CHIEF FINANCIAL OFFICER,MITCH TAYLOR,MITCH.TAYLOR@MONTGOMERY.K12.NC.US,MONTGOMERY COUNTY BOARD OF EDUCATION,9105766511,CHIEF FINANCIAL OFFICER,441 PAGE STREET,TROY,NC,566001076,GSA_MIGRATION,No,27371,9109971418,NC,ROCKINGHAM,MEMBER,116 GREYLYN ROAD,28379,USA,"DALE SMITH, CPA",DSMITH@ASW-CPA.COM,ANDERSON SMITH & WIKE PLLC,,271473864,,84,2024-01-27,2017-11-12,2017-11-13,2017-11-13,2017-11-12,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 93, 84, 12",state,,annual,6456704,UG,true,CENSUS,2017-11-12,"DALE SMITH, CPA",MEMBER -220000,2017-09-CENSUS-0000194114,CHARLOTTE COUNTY AIRPORT AUTHORITY,DIRECTOR OF FINANCE,SANDY CAULEY,SCAULEY@FLYPGD.COM,CHARLOTTE COUNTY AIRPORT AUTHORITY,9416391101,DIRECTOR OF FINANCE,28000 A-1 AIRPORT ROAD,PUNTA GORDA,FL,591082365,GSA_MIGRATION,No,33982,2392222090,FL,FORT MYERS,PARTNER,"12621 WORLD PLAZA LANE, BLDG 55",33907,USA,JEFFREY M. TUSCAN,JTUSCAN@TUSCANCPA.COM,"TUSCAN & COMPANY, PA",,260254161,,20,2024-01-27,2018-04-25,2018-04-26,2018-04-25,2018-04-25,2017-09-30,2016-09-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"97, 20",local,,annual,1963905,UG,true,CENSUS,2018-04-25,JEFFREY M. TUSCAN,PARTNER -221000,2017-12-CENSUS-0000243537,SC THRIVE,CHIEF FINANCIAL OFFICER,ALLIE BOYKIN,ABOYKIN@SCTHRIVE.ORG,SC THRIVE,8037279965,CHIEF FINANCIAL OFFICER,2211 ALPINE ROAD EXTENSION,COLUMBIA,SC,901011409,GSA_MIGRATION,No,29223,8037990555,SC,COLUMBIA,SHAREHOLDER,1704 LAUREL STREET,29201,USA,CHRISTINA KELLY,CHRISTINA@HOBBSCPA.COM,"THE HOBBS GROUP, P.A.",,570957419,,10,2024-01-27,2018-04-29,2018-04-20,2018-04-30,2018-04-29,2017-12-31,2016-12-31,2017,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,Yes,"93, 10",non-profit,,annual,1333188,UG,true,CENSUS,2018-04-29,CHRISTINA KELLY,SHAREHOLDER -222000,2017-03-CENSUS-0000111191,"HOMEWISE, INC.",CONTROLLER,ROBIN COLE,RCOLE@HOMEWISE.ORG,"HOMEWISE, INC.",5059557014,CONTROLLER,1301 SILER RD. BLDG. D,SANTA FE,NM,850346325,GSA_MIGRATION,No,87505,5053380800,NM,ALBUQUERQUE,AUDIT MANAGER,"6200 UPTOWN BLVD NE, SUITE 400",87110,USA,LEE BALDWIN,LEE.BALDWIN@RICCICPA.COM,"RICCI & COMPANY, LLC",,205949532,,21,2024-01-27,2017-08-10,2017-08-11,2017-08-11,2017-08-10,2017-03-31,2016-03-31,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"14, 21",non-profit,,annual,2065432,UG,true,CENSUS,2017-08-10,LEE BALDWIN,AUDIT MANAGER -223000,2017-06-CENSUS-0000222752,GREENFIELD MANOR 122-HD144-WDD-NP,EXECUTIVE DIRECTOR,NORMA DELGADO,NORMAD@HOMEOPENINGDOORS.ORG,GREENFIELD MANOR 122-HD144-WDD-NP,3109884000,EXECUTIVE DIRECTOR,"5601 W SLAUSON AVE., SUITE 180",CULVER CITY,CA,731628868,GSA_MIGRATION,No,90230,8188485585,CA,VALENCIA,CPA,"27200 TOURNEY ROAD, SUITE 475",91355,USA,THOMAS ENGMAN,TOM@KKAJCPA.COM,"KKAJ, LLP",,463462919,,14,2024-01-27,2018-02-20,2018-02-21,2018-02-21,2018-02-20,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,911062,UG,true,CENSUS,2018-02-20,THOMAS ENGMAN,CPA -224000,2017-06-CENSUS-0000156554,MIDDLESEX COUNTY VOCATIONAL AND TECHNICAL HIGH SCHOOLS,BUSINESS ADMINISTRATOR,KARL J. KNEHR,KNEHRK@MCVTS.NET,MIDDLESEX COUNTY VOCATIONAL AND TECHNICAL HIGH SCHOOLS,7322573300,BUSINESS ADMINISTRATOR,112 RUES LANE,EAST BRUNSWICK,NJ,226002455,GSA_MIGRATION,No,08816,9089676855,NJ,CRANFORD,PARTNER,20 COMMERCE DRIVE SUITE 301,07016,USA,DAVID GANNON,DGANNON@PKFOD.COM,PKF O'CONNOR DAVIES LLP,,271728945,,84,2024-01-27,2017-12-21,2017-12-11,2017-12-22,2017-12-21,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 93, 84",local,,annual,2585883,UG,true,CENSUS,2017-12-21,DAVID GANNON,PARTNER -225000,2017-06-CENSUS-0000150272,SPARTA AREA SCHOOLS,FINANCE DIRECTOR,DEBORAH CARNES,DEB.CARNES@SPARTASCHOOLS.ORG,SPARTA AREA SCHOOLS,6168878253,FINANCE DIRECTOR,465 SOUTH UNION STREET,SPARTA,MI,386002095,GSA_MIGRATION,No,49345,6169493200,MI,GRAND RAPIDS,SHAREHOLDER,2910 LUCERNE DRIVE SE,495467175,USA,"ADRIANE SCHRAUBEN, CPA, MBA",ASCHRAUBEN@HUNGERFORDNICHOLS.COM,HUNGERFORD NICHOLS,,382184825,,84,2024-01-27,2017-11-05,2017-11-06,2017-11-06,2017-11-05,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 10, 84",local,,annual,2144687,UG,true,CENSUS,2017-11-05,"ADRIANE SCHRAUBEN, CPA, MBA",SHAREHOLDER -226000,2017-09-CENSUS-0000241244,CITRUS RESEARCH BOARD,MANAGER OF FINANCE & ACCOUNTING,KELLY SALINAS,KELLY@CITRUSRESEARCH.ORG,CITRUS RESEARCH BOARD,5597380246,MANAGER OF FINANCE & ACCOUNTING,PO BOX 230,VISALIA,CA,953777492,GSA_MIGRATION,No,93279,5592256891,CA,FRESNO,PARTNER,1180 E. SHAW AVE. SUITE 110,93710,USA,SCOTT FAETH,SCOTT@BCFZ-CPA.COM,"BORCHARDT, CORONA, FAETH & ZAKARIAN",,770314847,,10,2024-01-27,2018-05-01,2018-05-02,2018-05-02,2018-05-01,2017-09-30,2016-09-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,10,local,,annual,1086375,UG,true,CENSUS,2018-05-01,SCOTT FAETH,PARTNER -227000,2017-12-CENSUS-0000245491,THE GOOD SAMARITAN HEALTH CENTER OF COBB,DIRECTOR OF STEWARDSHIP,KACIE MCDONNELL,KACIE.MCDONNELL@GOODSAMHC.ORG,THE GOOD SAMARITAN HEALTH CENTER OF COBB,4047832533,DIRECTOR OF STEWARDSHIP,1605 ROBERTA DRIVE,MARIETTA,GA,320045238,GSA_MIGRATION,No,30008,6782905100,GA,MARIETTA,CPA,POST AND ASSOCIATES 3475 DALLAS HWY #125,30064,USA,JIM CASTLE,JIM@POSTCPAS.COM,DAN POST/POST AND ASSOCIATES,,813068280,,93,2024-01-27,2018-09-24,2018-09-24,2018-09-25,2018-09-24,2017-12-31,2016-12-31,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 14",non-profit,,annual,1052368,UG,true,CENSUS,2018-09-24,JIM CASTLE,CPA -228000,2017-06-CENSUS-0000068545,"HIT, INC. (HOUSING INDUSTRY TRAINING, INC.)",EXECUTIVE DIRECTOR,MIKE REMBOLDT,MREMBOLDT@HITINC.ORG,"HIT, INC. (HOUSING INDUSTRY TRAINING, INC.)",7016330379,EXECUTIVE DIRECTOR,1007 18TH STREET NW,MANDAN,ND,450353818,GSA_MIGRATION,No,585549369,7012558488,ND,BISMARCK,AUDIT PARTNER,1730 BURNT BOAT DR,585021914,USA,KATIE WILLIAMS,KWILLIAMS@EIDEBAILLY.COM,EIDE BAILLY LLP,,450250958,,93,2024-01-27,2017-10-16,2017-10-16,2017-10-16,2017-10-16,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 10",non-profit,,annual,1582738,UG,true,CENSUS,2017-10-16,KATIE WILLIAMS,AUDIT PARTNER -229000,2016-06-CENSUS-0000169847,"SCHOOL DISTRICT OF THE CITY OF ERIE, PENNSYLVANIA",CHIEF FINANCIAL OFFICER,BRIAN POLITO,BPOLITO@ERIESD.ORG,"SCHOOL DISTRICT OF THE CITY OF ERIE, PENNSYLVANIA",8148746137,CHIEF FINANCIAL OFFICER,148 WEST 21ST STREET,ERIE,PA,256001265,GSA_MIGRATION,No,16502,4123677102,PA,PITTSBURGH,PARTNER,"3800 MCKNIGHT EAST DRIVE, SUITE 3805",15237,USA,CORY JOHNSON,CJOHNSON@ZALLC.ORG,ZELENKOFSKE AXELROD LLC,,233022352,,84,2024-01-29,2017-03-16,2017-03-20,2017-03-20,2017-03-16,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,Yes,Yes,No,No,No,872151,Yes,"84, 93, 10",local,,annual,29200682,UG,true,CENSUS,2017-03-16,CORY JOHNSON,PARTNER -230000,2017-06-CENSUS-0000235032,EDISON TERRACE SOUTH 171-HD022,DEPARTMENT MANAGER,KIM CONWAY,KCONWAY@G-B.COM,EDISON TERRACE SOUTH 171-HD022,5094596102,DEPARTMENT MANAGER,818 W RIVERSIDE AVE#300,SPOKANE,WA,264362933,GSA_MIGRATION,No,99201,5097472158,WA,SPOKANE,CPA,422 W RIVERSIDE AVE #1420,99201,USA,DARCY SKJOTHAUG,DSKJOTHAUG@SCHOEDEL.COM,SCHOEDEL & SCHOEDEL CPAS PLLC,,910614823,,14,2024-01-27,2017-10-22,2017-10-16,2017-10-23,2017-10-22,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,1702419,UG,true,CENSUS,2017-10-22,DARCY SKJOTHAUG,CPA -231000,2017-12-CENSUS-0000233543,DOUGLAS POINTE III ASSOCIATES LLC DBA DOUGLAS POINTE III APTS 07311392,CONTROLLER,ROB ELSBY,RELSBY@FLCO.COM,DOUGLAS POINTE III ASSOCIATES LLC DBA DOUGLAS POINTE III APTS 07311392,3178169300,CONTROLLER,"ONE INDIANA SQUARE, SUITE 3000",INDIANAPOLIS,IN,351946082,GSA_MIGRATION,No,46204,3178485700,IN,CARMEL,PRINCIPAL,"501 CONGRESSIONAL BLVD, SUITE 300",46032,USA,RYAN STRUTZ,RSTRUTZ@DOZ.NET,"DAUBY O'CONNOR & ZALESKI, LLC",,351750664,,14,2024-01-27,2018-04-10,2018-04-11,2018-04-09,2018-04-10,2017-12-31,2016-12-31,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,2605252,UG,true,CENSUS,2018-04-10,RYAN STRUTZ,PRINCIPAL -232000,2017-09-CENSUS-0000188538,"CITY OF ROLLA, MISSOURI",FINANCE DIRECTOR,STEFFANIE ROGERS,SROGERS@ROLLACITY.ORG,"CITY OF ROLLA, MISSOURI",5734266980,FINANCE DIRECTOR,901 NORTH ELM STREET,ROLLA,MO,436003049,GSA_MIGRATION,No,65402,6365329525,MO,CHESTERFIELD,PARTNER,"15450 SOUTH OUTER ROAD FORTY, SUITE 135",630172066,USA,"TAMBER M. ALSOP, CPA",TAMMY@HBCLP.COM,"HOCHSCHILD, BLOOM & COMPANY LLP",,430673920,,16,2024-01-27,2018-10-24,2018-10-25,2018-10-25,2018-10-24,2017-09-30,2016-09-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"97, 20, 16",local,,annual,1130787,UG,true,CENSUS,2018-10-24,"TAMBER M. ALSOP, CPA",PARTNER -233000,2017-05-CENSUS-0000220905,"PENSDALE HOUSING, INC. 034-EE100-WAH",CONTROLLER,CHARMAINE KENT,CHARMAINE.KENT@PRESBYSINSPIREDLIFE.ORG,"PENSDALE HOUSING, INC. 034-EE100-WAH",6102601119,CONTROLLER,2000 JOSHUA ROAD,LAFAYETTE HILL,PA,311719838,GSA_MIGRATION,No,194442430,7177617910,PA,CAMP HILL,PARTNER,415 FALLOWFIELD ROAD,17011,USA,GARY DUBAS,GDUBAS@MACPAS.COM,"MCKONLY & ASBURY, LLP",,231909723,,14,2024-01-27,2017-08-29,2017-08-30,2017-08-30,2017-08-29,2017-05-31,2016-05-31,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,7436685,UG,true,CENSUS,2017-08-29,GARY DUBAS,PARTNER -234000,2017-12-CENSUS-0000230805,"GEORGIA MENTAL HEALTH CONSUMER NETWORK, INC.",FINANCIAL MANAGER,LYNN THOGERSEN,LYNN@GMHCN.ORG,"GEORGIA MENTAL HEALTH CONSUMER NETWORK, INC.",4046879487,FINANCIAL MANAGER,246 SYCAMORE ST. SUITE 260,DECATUR,GA,581981093,GSA_MIGRATION,No,300303434,7068864343,GA,TOCCOA,CPA,3438 HIGHWAY 17 ALT,30577,USA,DAVID BENFIELD,DAVID@DAVIDBENFIELDCPA.COM,"DAVID BENFIELD, CPA, PC",,582363670,,93,2024-01-27,2018-07-23,2018-07-24,2018-07-24,2018-07-23,2017-12-31,2016-12-31,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,93,non-profit,,annual,4425348,UG,true,CENSUS,2018-07-23,DAVID BENFIELD,CPA -235000,2017-06-CENSUS-0000144750,IOWA CENTRAL COMMUNITY COLLEGE,"VICE PRESIDENT, BUSINESS AFFAIRS",ANGIE MARTIN,MARTIN_A@IOWACENTRAL.EDU,IOWA CENTRAL COMMUNITY COLLEGE,5155741064,"VICE PRESIDENT, BUSINESS AFFAIRS",1 TRITON CIRCLE,FORT DODGE,IA,410906391,GSA_MIGRATION,No,50501,5155767136,IA,FORT DODGE,PARTNER,822 CENTRAL AVE,50501,USA,MURRAY STANLEY,MURRAY@SCHNURRLLP.COM,"SCHNURR & COMPANY, LLP",,421282836,,84,2024-01-27,2018-03-28,2018-03-19,2018-03-28,2018-03-28,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,Yes,"10, 84, 17",local,,annual,28805867,UG,true,CENSUS,2018-03-28,MURRAY STANLEY,PARTNER -236000,2017-12-CENSUS-0000234455,COLE STREET DEVELOPMENT CORPORATION,DIRECTOR OF FINANCE,MARK WRIGHT,MWRIGHT@MARIETTAHOUSINGAUTHORITY.ORG,COLE STREET DEVELOPMENT CORPORATION,7704193200,DIRECTOR OF FINANCE,95 COLE STREET,MARIETTA,GA,264536445,GSA_MIGRATION,No,30061,7708798411,GA,LAWRENCEVILLE,PRINCIPAL,"1255 LAKES PARKWAY, STE 375",30043,USA,DALE R. RECTOR,DRECTOR@CPA-RRL.COM,"RECTOR, REEDER & LOFTON, P.C.",,263670494,,14,2024-01-27,2018-07-11,2018-07-12,2018-05-17,2018-07-11,2017-12-31,2016-12-31,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,14,non-profit,,annual,3320075,UG,true,CENSUS,2018-07-11,DALE R. RECTOR,PRINCIPAL -237000,2017-06-CENSUS-0000247375,LAWS LOUDOUN CITIZENS FOR SOCIAL JUSTICE,EXECUTIVE DIRECTOR,"JUDY HANLEY, PHD",JUDY.HANLEY@LCSJ.ORG,LAWS LOUDOUN CITIZENS FOR SOCIAL JUSTICE,7037713398,EXECUTIVE DIRECTOR,105 EAST MARKET STREET,LEESBURG,VA,541282756,GSA_MIGRATION,No,201760000,7037774900,VA,LEESBURG,CPA,110 EAST MARKET STREET SUITE 200,201760000,USA,KARA SANTMYER,KARA@MCOCPA.COM,"MITCHELL & CO., P.C.",,541853459,,16,2024-01-27,2018-08-12,2018-08-06,2018-08-13,2018-08-12,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"93, 14, 16",non-profit,,annual,919623,UG,true,CENSUS,2018-08-12,KARA SANTMYER,CPA -238000,2017-06-CENSUS-0000136775,MONROE COUNTY DISTRICT SCHOOL BOARD,EXE DIRECTOR OF FINANCE AND PERFORMANCE,JAMES DRAKE,JAMES.DRAKE@KEYSSCHOOLS.COM,MONROE COUNTY DISTRICT SCHOOL BOARD,3052931400,EXE DIRECTOR OF FINANCE AND PERFORMANCE,241 TRUMBO ROAD,KEY WEST,FL,596000750,GSA_MIGRATION,No,330406684,8504122905,FL,TALLAHASSEE,AUDIT MANAGER,111 WEST MADISON STREET,323991450,USA,MICAH RODGERS,MICAHRODGERS@AUD.STATE.FL.US,STATE OF FLORIDA AUDITOR GENERAL,,596001874,,93,2024-01-27,2018-03-26,2018-03-27,2018-03-26,2018-03-26,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 93, 10",local,,annual,8979530,UG,true,CENSUS,2018-03-26,MICAH RODGERS,AUDIT MANAGER -239000,2017-12-CENSUS-0000241203,URBAN RESOURCE INSTITUTE AND AFFILIATES,SEVIOR VICE PRESIDENT OF FINANCE,DONOVAN MURRAY,DMURRAY@URINYC.ORG,URBAN RESOURCE INSTITUTE AND AFFILIATES,6465880040,SEVIOR VICE PRESIDENT OF FINANCE,75 BROAD STREET SUITE 505,NEW YORK,NY,112561648,GSA_MIGRATION,No,10004,7329649359,NJ,EAST BRUNSWICK,PARTNER,ONE TOWER CENTER BLVD,08816,USA,JAMES MULROY,JMULROY@WITHUM.COM,WTIHUMSMITH+BROWN,,222027092,,93,2024-01-27,2018-06-27,2018-06-28,2018-06-25,2018-06-27,2017-12-31,2016-12-31,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 14",non-profit,,annual,17891820,UG,true,CENSUS,2018-06-27,JAMES MULROY,PARTNER -240000,2017-12-CENSUS-0000209375,GENESEE VALLEY PRESBYTERIAN NURSING CTR D/B/A KIRKHAVEN 014-43199,"CONTROLLER, CMSWNY",SCOTT MAAR,SMAAR@CMSWNY.ORG,GENESEE VALLEY PRESBYTERIAN NURSING CTR D/B/A KIRKHAVEN 014-43199,5854424315,"CONTROLLER, CMSWNY",254 ALEXANDER STREET,ROCHESTER,NY,222291346,GSA_MIGRATION,No,14607,5853811000,NY,PITTSFORD,PARTNER,171 SULLY'S TRAIL,14534,USA,KELLE DEMONTE,KDEMONTE@BONADIO.COM,"BONADIO & CO., LLP",,161131146,,14,2024-01-27,2018-07-09,2018-07-10,2018-05-30,2018-07-09,2017-12-31,2016-12-31,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,4902508,UG,true,CENSUS,2018-07-09,KELLE DEMONTE,PARTNER -241000,2017-12-CENSUS-0000220835,"KENAI PENINSULA HOUSING INITIATIVES, INC.",EXECUTIVE DIRECTOR,STEVEN ROUSE,STEVEN@KPHI.NET,"KENAI PENINSULA HOUSING INITIATIVES, INC.",9072354357,EXECUTIVE DIRECTOR,3751 STERLING HIGHWAY,HOMER,AK,911823180,GSA_MIGRATION,No,99603,9072788878,AK,ANCHORAGE,MANAGING DIRECTOR,"3601 C STREET, SUITE 600",99503,USA,ALEX BECKMAN,ABECKMAN@BDO.COM,"BDO USA, LLP",,135381590,,14,2024-01-27,2018-09-29,2018-09-30,2018-09-30,2018-09-29,2017-12-31,2016-12-31,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,14,non-profit,,annual,2510066,UG,true,CENSUS,2018-09-29,ALEX BECKMAN,MANAGING DIRECTOR -242000,2016-06-CENSUS-0000111911,NORTHLAND FAMILY HELP CENTER,EXECUTIVE DIRECTOR,AILEEN FITZ,AFITZ@NORTHLANDFAMILY.ORG,NORTHLAND FAMILY HELP CENTER,9282332205,EXECUTIVE DIRECTOR,"2532 NORTH FOURTH STREET, SUITE 506",FLAGSTAFF,AZ,860351566,GSA_MIGRATION,No,86004,6022643077,AZ,SCOTTSDALE,DIRECTOR,"9019 EAST BAHIA DRIVE, SUITE 100",852601553,USA,OLIVIA BRASHER,OBRASHER@F-CPC.COM,FESTER & CHAPMAN P.C.,,860494040,,93,2024-01-27,2017-01-11,2017-01-12,2017-01-12,2017-01-11,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 16, 93",non-profit,,annual,954937,UG,true,CENSUS,2017-01-11,OLIVIA BRASHER,DIRECTOR -243000,2016-06-CENSUS-0000187474,CITY OF SANTA FE,ASSISTANT FINANCE DIRECTOR,TERESITA GARCIA,TMGARCIA@CI.SANTA-FE.NM.US,CITY OF SANTA FE,5059556532,ASSISTANT FINANCE DIRECTOR,200 LINCOLN AVENUE,SANTA FE,NM,856000168,GSA_MIGRATION,No,87504,5058832727,NM,ALBUQUERQUE,PARTNER,2424 LOUISIANA BLVD SUITE 300,87110,USA,ROBERT CORDOVA,RCORDOVA@ACGSW.COM,RPC CPAS + CONSULTANTS,,850454285,,14,2024-01-27,2017-01-22,2017-01-23,2017-01-22,2017-01-22,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"20, 14, 93, 94, 16, 15, 97",local,,annual,5201021,UG,true,CENSUS,2017-01-22,ROBERT CORDOVA,PARTNER -244000,2016-12-CENSUS-0000219565,PROMESA BEHAVIORAL HEALTH & SUBSIDIARY,CONTROLLER,SUSANNE PRUETT,SPRUETT@PROMESABEHAVIORAL.ORG,PROMESA BEHAVIORAL HEALTH & SUBSIDIARY,5594373596,CONTROLLER,7120 NORTH MARKS AVE STE 110,FRESNO,CA,770174896,GSA_MIGRATION,No,93711,5592999540,CA,CLOVIS,AUDIT MANAGER,677 SCOTT AVENUE,93612,USA,HENRY OUM,HENRY@PPCPAS.COM,PRICE PAIGE AND COMPANY,,770203007,,93,2024-01-27,2017-09-27,2017-09-27,2017-09-27,2017-09-27,2016-12-31,2016-01-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 93",non-profit,,annual,5904670,UG,true,CENSUS,2017-09-27,HENRY OUM,AUDIT MANAGER -245000,2016-06-CENSUS-0000187278,"COUNTY OF NELSON, VIRGINIA",COUNTY ADMINISTRATOR,STEPHEN CARTER,SCARTER@NELSONCOUNTY.ORG,"COUNTY OF NELSON, VIRGINIA",4342637001,COUNTY ADMINISTRATOR,PO BOX 336,LOVINGSTON,VA,546001441,GSA_MIGRATION,No,22949,4349738314,VA,CHARLOTTESVILLE,MEMBER,PO BOX 6580,22906,USA,DAVID FOLEY,DFOLEY@RFCA.COM,"ROBINSON, FARMER, COX ASSOCIATES",,541896113,,84,2024-01-27,2017-03-01,2017-03-01,2017-03-02,2017-03-01,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 14, 93, 84, 16, 20, 97",local,,annual,2601211,UG,true,CENSUS,2017-03-01,DAVID FOLEY,MEMBER -246000,2016-09-CENSUS-0000226040,"WIYN CONSORTIUM, INC.",SENIOR FINANCIAL COMPLIANCE MANAGER,JOSE MARTENS,JMARTENS@AURA-ASTRONOMY.ORG,"WIYN CONSORTIUM, INC.",5203188520,SENIOR FINANCIAL COMPLIANCE MANAGER,P.O. BOX 26732,TUCSON,AZ,860669456,GSA_MIGRATION,No,857266732,5207903500,AZ,TUCSON,PRINCIPAL,"5255 EAST WILLIAMS CIRCLE, SUITE 5000",85711,USA,DAVE PHILLIPS,DAVE.PHILLIPS@CLACONNECT.COM,CLIFTONLARSONALLEN,,410746749,,47,2024-01-27,2017-05-25,2017-05-26,2017-05-26,2017-05-25,2016-09-30,2015-10-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,47,non-profit,,annual,1248623,UG,true,CENSUS,2017-05-25,DAVE PHILLIPS,PRINCIPAL -247000,2016-12-CENSUS-0000246429,GOODWILL INDUSTRIES OF CENTRAL EAST TEXAS INC.,TIFFANY.RHODES@LUFKINGOODWILL.ORG,TIFFANY RHODES,TIFFANY.RHODES@LUFKINGOODWILL.ORG,GOODWILL INDUSTRIES OF CENTRAL EAST TEXAS INC.,9366328838,TIFFANY.RHODES@LUFKINGOODWILL.ORG,301 HILL STREET,LUFKIN,TX,752204594,GSA_MIGRATION,No,75904,9366346621,TX,LUFKIN,PARTNER,1307 SOUTH 1ST STREET,75901,USA,MOLLY ABELE,MABELE@AXLEYRODE.COM,AXLEY & RODE LLP,,750767305,,93,2024-01-27,2017-09-28,2018-01-05,2018-01-08,2017-09-28,2016-12-31,2016-01-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,93,non-profit,,annual,4942777,UG,true,CENSUS,2017-09-28,MOLLY ABELE,PARTNER -248000,2016-11-CENSUS-0000089384,"RURAL HEALTH CARE, INC., DBA AZALEA HEALTH",CHIEF FINANCIAL OFFICER,JIM REID,JREID@AZAHEALTH.ORG,"RURAL HEALTH CARE, INC., DBA AZALEA HEALTH",3863251086,CHIEF FINANCIAL OFFICER,613 ST. JOHNS AVENUE; 3RD FLOOR,PALATKA,FL,591792958,GSA_MIGRATION,No,32177,3523781331,FL,GAINESVILLE,PARTNER,5931 NW 1ST PLACE,32607,USA,JENNIFER FORRESTER,JENNIFER.FORRESTER@JMCO.COM,"JAMES MOORE & CO., P.L.",,593204548,,93,2024-01-27,2017-09-28,2017-09-29,2017-09-29,2017-09-28,2016-11-30,2015-12-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,93,non-profit,,annual,6092141,UG,true,CENSUS,2017-09-28,JENNIFER FORRESTER,PARTNER -249000,2016-12-CENSUS-0000039731,"CINCINNATI HEALTH NETWORK, INC.",CEO,KATHLEEN BENNETT,KBENNETT@CINTIHEALTHNETWORK.ORG,"CINCINNATI HEALTH NETWORK, INC.",5139610600,CEO,2825 BURNET AVE. #232,CINCINNATI,OH,311182378,GSA_MIGRATION,No,45219,8593315622,KY,FT. WRIGHT,AUDIT MANAGER,909 WRIGHT'S SUMMIT PARKWAY SUITE 120,41011,USA,VICKI SELIGMAN,VSELIGMAN@CPAWIN.COM,"ANNEKEN, HUEY & MOSER PLLC",,113799330,,93,2024-01-27,2017-09-27,2017-09-26,2017-09-28,2017-09-27,2016-12-31,2016-01-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,93,non-profit,,annual,3488002,UG,true,CENSUS,2017-09-27,VICKI SELIGMAN,AUDIT MANAGER -250000,2016-06-CENSUS-0000148123,"TOWN OF SOUTH HADLEY, MASSACHUSETTS",TOWN ACCOUNTANT,WILLIAM SUTTON,WSUTTON@SOUTHHADLEYMA.GOV,"TOWN OF SOUTH HADLEY, MASSACHUSETTS",4135385017,TOWN ACCOUNTANT,116 MAIN ST,SOUTH HADLEY,MA,046001303,GSA_MIGRATION,No,01075,4136654001,MA,SOUTH DEERFIELD,PARTNER,8 TINA DRIVE,01373,USA,"THOMAS J. SCANLON, JR.",TSCANLONJR@COMCAST.NET,"SCANLON & ASSOCIATES, LLC",,270371224,,93,2024-01-27,2017-06-28,2017-06-29,2017-06-12,2017-06-28,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"84, 93, 14, 97, 20, 10",local,,annual,1735743,UG,true,CENSUS,2017-06-28,"THOMAS J. SCANLON, JR.",PARTNER -251000,2016-08-CENSUS-0000085650,"THE CARTER CENTER, INC.","VICE PRESIDENT, FINANCE AND TREASURER",CHRISTOPHER BROWN,CHRISTOPHER.BROWN@CARTERCENTER.ORG,"THE CARTER CENTER, INC.",4044203429,"VICE PRESIDENT, FINANCE AND TREASURER","ONE COPENHILL, 453 FREEDOM PARKWAY",ATLANTA,GA,581454716,GSA_MIGRATION,No,30307,4042223258,GA,ATLANTA,PARTNER,"303 PEACHTREE ST NE, SUITE 2000",30308,USA,DASHA WALKER,DASHAWALKER@KPMG.COM,KPMG LLP,,135565207,,98,2024-01-27,2018-08-30,2018-08-31,2018-08-31,2018-08-30,2016-08-31,2015-09-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"98, 93, 19",non-profit,,annual,10395332,UG,true,CENSUS,2018-08-30,DASHA WALKER,PARTNER -252000,2016-12-CENSUS-0000194168,"STEVENS COUNTY ECONOMIC IMPROVEMENT COMMISSION, INC.",EXECUTIVE DIRECTOR,CHERYL KUHN,CHERYLSCEIC@FEDTEL.NET,"STEVENS COUNTY ECONOMIC IMPROVEMENT COMMISSION, INC.",3205852609,EXECUTIVE DIRECTOR,"215 ATLANTIC AVENUE, SUITE 110",MORRIS,MN,411383188,GSA_MIGRATION,No,56267,2187393267,MN,FERGUS FALLS,PARTNER,2450 COLLEGE WAY,56537,USA,DEAN BIRKELAND,DEANB@CARLSONHIGHLANDCPA.COM,CARLSON & COMPANY LLP,,411562398,,10,2024-01-29,2017-08-01,2017-08-02,2017-08-01,2017-08-01,2016-12-31,2016-01-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,Yes,"10, 14",non-profit,,annual,1525896,UG,true,CENSUS,2017-08-01,DEAN BIRKELAND,PARTNER -253000,2017-06-CENSUS-0000116605,OREGON COAST COMMUNITY ACTION,FINANCE DIRECTOR,ANDREW SANTOS,ASANTOS@ORCCA.US,OREGON COAST COMMUNITY ACTION,5414357755,FINANCE DIRECTOR,1855 THOMAS AVE,COOS BAY,OR,930547036,GSA_MIGRATION,No,97420,3605333370,WA,ABERDEEN,PRINCIPAL,343 W WISHKAH ST,98520,USA,LONNIE RICH,LONNIE@AIKEN-SANDERS.COM,AIKEN & SANDERS INC. PS,,910870697,,93,2024-01-28,2017-11-06,2017-11-06,2017-11-07,2017-11-06,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 97, 93, 10, 64, 14, 81",non-profit,,annual,5444526,UG,true,CENSUS,2017-11-06,LONNIE RICH,PRINCIPAL -254000,2017-09-CENSUS-0000102556,ST GEORGE'S SENIOR HOUSING INC. 115-EH057,MANAGEMENT AGENT,CONNIE QUILLEN,CQUILLEN@ALPHA-BARNES.COM,ST GEORGE'S SENIOR HOUSING INC. 115-EH057,5125813611,MANAGEMENT AGENT,1443 CORONADO HILLS DRIVE,AUSTIN,TX,742245337,GSA_MIGRATION,No,787522974,8178651360,TX,SOUTHLAKE,PARTNER,"515 W SOUTHLAKE BLVD, STE 150",760926176,USA,MICHAEL MARTIN,MMARTIN@MGROUPTX.COM,"M GROUP, LLP",,263548566,,14,2024-01-29,2018-04-11,2018-04-12,2018-04-12,2018-04-11,2017-09-30,2016-09-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,No,14,non-profit,,annual,1611512,UG,true,CENSUS,2018-04-11,MICHAEL MARTIN,PARTNER -255000,2017-06-CENSUS-0000133121,OMNITRANS,CEO/GENERAL MANAGER,P SCOTT GRAHAM,SCOTT.GRAHAM@OMNITRANS.ORG,OMNITRANS,9093797112,CEO/GENERAL MANAGER,1700 WEST FIFTH STREET,SAN BERNARDINO,CA,953035112,GSA_MIGRATION,No,92411,9094664410,CA,RANCHO CUCAMONGA,PARTNER,"10681 FOOTHILL BLVD., SUITE 300",91730,USA,ROGER ALFARO,RALFARO@VTDCPA.COM,"VAVRINEK, TRINE, DAY & CO., LLP",,952648289,,20,2024-01-29,2018-01-24,2018-01-25,2018-01-25,2018-01-24,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,1167662,Yes,20,local,,annual,38922080,UG,true,CENSUS,2018-01-24,ROGER ALFARO,PARTNER -256000,2016-06-CENSUS-0000119886,"SANTA CRUZ COUNTY PARENTS ASSOCIATION, INC.",EXECUTIVE DIRECTOR,ELAINE HENNING,STAFF@VOUCHERPROJECT.ORG,"SANTA CRUZ COUNTY PARENTS ASSOCIATION, INC.",8316882152,EXECUTIVE DIRECTOR,"9047 SOQUEL DRIVE, SUITE D",APTOS,CA,942410291,GSA_MIGRATION,No,95003,5104525051,CA,OAKLAND,MANAGING PARTNER,"266 17TH STREET, SUITE 200",946124124,USA,SANWAR HARSHWAL,PATEL@PATELCPA.COM,PATEL & ASSOCIATES LLP,,364781081,,93,2024-01-29,2016-12-12,2016-12-13,2016-12-13,2016-12-12,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,93,non-profit,,annual,1920267,UG,true,CENSUS,2016-12-12,SANWAR HARSHWAL,MANAGING PARTNER -257000,2016-06-CENSUS-0000131710,LITTLE ROCK SCHOOL DISTRICT,"DIRECTOR, FINANCE AND ACCOUNTING",JEAN RING,JEAN.RING@LRSD.ORG,LITTLE ROCK SCHOOL DISTRICT,5014471089,"DIRECTOR, FINANCE AND ACCOUNTING",810 WEST MARKHAM STREET,LITTLE ROCK,AR,716014717,GSA_MIGRATION,No,722011306,5016838600,AR,LITTLE ROCK,DEPUTY LEGISLATIVE AUDITOR,"500 WOODLANE STREET, STE 172",722011099,USA,"LARRY W. HUNTER, CPA, CFE",DCF@ARKLEGAUDIT.GOV,ARKANSAS LEGISLATIVE AUDIT,,716042275,,12,2024-01-29,2017-03-26,2017-03-24,2017-03-27,2017-03-26,2016-06-30,2015-07-01,2016,single-audit,not_gaap,regulatory_basis,Yes,unmodified_opinion,No,No,No,No,No,974680,Yes,"12, 84, 15, 10, 93",local,,annual,32489344,UG,true,CENSUS,2017-03-26,"LARRY W. HUNTER, CPA, CFE",DEPUTY LEGISLATIVE AUDITOR -258000,2016-06-CENSUS-0000133979,MODOC JOINT UNIFIED SCHOOL DISTRICT,BUSINESS MANAGER,NIKKI GUZMAN,EGUZMAN@MODOC.K12.CA.US,MODOC JOINT UNIFIED SCHOOL DISTRICT,5302337201,BUSINESS MANAGER,906 WEST FOURTH STREET,ALTURAS,CA,010560280,GSA_MIGRATION,No,96101,5302571040,CA,SUSANVILLE,AUDIT PARTNER,"1740 MAIN STREET, SUITE A",96130,USA,CLAY SINGLETON,CSINGLETON@SA-CPAS.COM,"SINGLETONAUMAN, PC",,753179488,,15,2024-01-29,2017-01-16,2017-01-15,2017-01-17,2017-01-16,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,No,"84, 93, 15, 10",local,,annual,1063186,UG,true,CENSUS,2017-01-16,CLAY SINGLETON,AUDIT PARTNER -259000,2017-06-CENSUS-0000165945,SPERRY PUBLIC SCHOOL DISTRICT NO. I-8,SUPERINTENDENT,BRIAN BEAGLES,BBEAGLES@SPERRY.K12.OK.US,SPERRY PUBLIC SCHOOL DISTRICT NO. I-8,9182886258,SUPERINTENDENT,400 WEST MAIN STREET,SPERRY,OK,736081383,GSA_MIGRATION,No,740730610,9183664440,OK,TULSA,PARTNER,10026-A SOUTH MINGO ROAD #309,741335700,USA,JACK JENKINS,JHJCPA@JACKHJENKINSCPA.COM,"JENKINS & KEMPER, CPAS, P.C.",,731514558,,84,2024-01-29,2018-03-28,2018-03-06,2018-03-29,2018-03-28,2017-06-30,2016-06-30,2017,single-audit,not_gaap,regulatory_basis,Yes,qualified_opinion,No,No,No,No,No,750000,No,"84, 10",state,,annual,1041462,UG,true,CENSUS,2018-03-28,JACK JENKINS,PARTNER -260000,2017-06-CENSUS-0000160592,KIRYAS JOEL UNION FREE SCHOOL DISTRICT,SUPERINTENDENT OF SCHOOLS,MR. JOEL PETLIN,JPETLIN@KJSD.K12.NY.US,KIRYAS JOEL UNION FREE SCHOOL DISTRICT,8457822300,SUPERINTENDENT OF SCHOOLS,48 BAKERTOWN ROAD,MONROE,NY,061301151,GSA_MIGRATION,No,10950,6314733400,NY,PORT JEFFERSON STATION,PARTNER,1650 ROUTE 112,117763060,USA,"ALAN YU, CPA",AYU@CDLLP.NET,"CULLEN & DANOWSKI, LLP",,112459188,,84,2024-01-29,2018-03-25,2018-03-26,2018-03-26,2018-03-25,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,84,local,,annual,9319153,UG,true,CENSUS,2018-03-25,"ALAN YU, CPA",PARTNER -261000,2017-12-CENSUS-0000170987,"CITY OF PIERRE, SOUTH DAKOTA",FINANCE OFFICER,TWILA HIGHT,TWILA.HIGHT@CI.PIERRE.SD.US,"CITY OF PIERRE, SOUTH DAKOTA",6057737407,FINANCE OFFICER,222 E DAKOTA AVE,PIERRE,SD,466000356,GSA_MIGRATION,No,575011253,6043481930,SD,RAPID CITY,AUDITOR DIRECTOR,"909 ST JOSEPH ST, STE 101",57701,USA,DEIDRE BUDAHL,DEIDREB2@CASEYPETERSON.COM,"CASEY PETERSON, LTD.",,460403496,,15,2024-01-29,2018-09-30,2018-09-28,2018-10-01,2018-09-30,2017-12-31,2016-12-31,2017,single-audit,unmodified_opinion,,,,No,Yes,Yes,Yes,No,750000,No,"20, 39, 15, 97",local,,annual,3155357,UG,true,CENSUS,2018-09-30,DEIDRE BUDAHL,AUDITOR DIRECTOR -262000,2016-06-CENSUS-0000164179,BAY VILLAGE CITY SCHOOL DISTRICT,TREASURER,KEVIN ROBERTSON,KEVIN.ROBERTSON@BAYSCHOOLSOHIO.ORG,BAY VILLAGE CITY SCHOOL DISTRICT,4406177312,TREASURER,377 DOVER CENTER ROAD,BAY VILLAGE,OH,346000208,GSA_MIGRATION,No,44140,2167873665,OH,COLUMBUS,SENIOR AUDIT MANAGER,88 EAST BROAD STREET,43215,USA,EVAN WITTE,ETWITTE@OHIOAUDITOR.GOV,"DAVE YOST, AUDITOR OF STATE OF OHIO",,311334821,,84,2024-01-29,2017-03-09,2017-03-10,2017-03-10,2017-03-09,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 84",local,,annual,822325,UG,true,CENSUS,2017-03-09,EVAN WITTE,SENIOR AUDIT MANAGER -263000,2016-06-CENSUS-0000175497,"TOWN OF WAITSFIELD, VERMONT",TOWN ADMINISTRATOR,VALERIE CAPELS,TOWNADMIN@GMAVT.NET,"TOWN OF WAITSFIELD, VERMONT",8024962218,TOWN ADMINISTRATOR,4144 MAIN STREET,WAITSFIELD,VT,036000726,GSA_MIGRATION,No,05673,8022232352,VT,MONTPELIER,PARTNER,PO BOX 947,05601,USA,FRED DUPLESSIS,FDUPLESSIS@SULLIVANPOWERS.COM,"SULLIVAN, POWERS & CO.",,030276150,,66,2024-01-29,2017-03-30,2017-03-30,2017-03-31,2017-03-30,2016-06-30,2015-07-01,2016,single-audit,not_gaap,cash_basis,No,qualified_opinion,No,Yes,Yes,No,Yes,750000,No,"20, 97, 14, 66",local,,annual,1553682,UG,true,CENSUS,2017-03-30,FRED DUPLESSIS,PARTNER -264000,2016-06-CENSUS-0000181767,"PLANNING AND SERVICE AREA 2, AREA AGENCY ON AGING",EXECUTIVE DIRECTOR,TERI GABRIEL,TERI@PSA2.ORG,"PLANNING AND SERVICE AREA 2, AREA AGENCY ON AGING",5308421687,EXECUTIVE DIRECTOR,PO BOX 1400,YREKA,CA,942679692,GSA_MIGRATION,No,960971400,5309263881,CA,MOUNT SHASTA,CPA,PO BOX 158,960670158,USA,STEVE DRAGESET,STEVE@AGTCPA.COM,AIELLO GOODRICH & TEUSCHER,,680146027,,10,2024-01-29,2017-01-16,2017-01-16,2017-01-10,2017-01-16,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,Yes,No,750000,Yes,"10, 93",local,,annual,1752446,UG,true,CENSUS,2017-01-16,STEVE DRAGESET,CPA -265000,2016-06-CENSUS-0000186403,"ACADEMIC DEVELOPMENT INSTITUTE, INC.",EXECUTIVE DIRECTOR,SAM REDDING,SREDDING@ADI.ORG,"ACADEMIC DEVELOPMENT INSTITUTE, INC.",2177326462,EXECUTIVE DIRECTOR,121 N. KICKAPOO,LINCOLN,IL,371153267,GSA_MIGRATION,No,62656,2175251111,IL,SPRINGFIELD,PARTNER,227 S SEVENTH STREET,62701,USA,BRADLEY J PUNKE,BPUNKE@ESPCPA.COM,"ECK, SCHAFER & PUNKE, LLP",,371335003,,84,2024-01-30,2017-02-20,2017-02-21,2017-02-21,2017-02-20,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,84,non-profit,,annual,1263571,UG,true,CENSUS,2017-02-20,BRADLEY J PUNKE,PARTNER -266000,2016-06-CENSUS-0000171430,"ROANE COUNTY, TENNESSEE",COUNTY EXECUTIVE,RON WOODY,RON.WOODY@ROANECOUNTYTN.GOV,"ROANE COUNTY, TENNESSEE",8653765671,COUNTY EXECUTIVE,"200 RACE STREET, SUITE 1",KINGSTON,TN,626000806,GSA_MIGRATION,No,377630643,6154017846,TN,NASHVILLE,AUDIT REVIEW MANAGER,SUITE 1500 JAMES K. POLK BLDG.,372431402,USA,GREG WORLEY,GREG.WORLEY@COT.TN.GOV,"COMPTROLLER OF THE TREASURY, DIVISION OF LOCAL GOVERNMENT AUDIT",,626001445,,16,2024-01-30,2016-12-13,2016-12-06,2016-12-14,2016-12-13,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"16, 97, 81, 84, 14, 20, 93, 12, 66, 11, 10",local,,annual,8898269,UG,true,CENSUS,2016-12-13,GREG WORLEY,AUDIT REVIEW MANAGER -267000,2016-06-CENSUS-0000187609,"WARNER UNIVERSITY, INC.",VP FOR FINANCE AND BUSINESS,GREGORY A. RODDEN,GREG.RODDEN@WARNER.EDU,"WARNER UNIVERSITY, INC.",8636387215,VP FOR FINANCE AND BUSINESS,13895 US HWY 27,LAKE WALES,FL,591275800,GSA_MIGRATION,No,338598797,8636767981,FL,LAKE WALES,PARTNER,230 E. TILLMAN AVE.,338533714,USA,"WILLIAM M. JACOBS, CPA",BILL.JACOBS@BTICPA.COM,"BUNTING, TRIPP & INGLEY, LLP",,590673514,,84,2024-01-30,2017-03-29,2017-03-30,2017-03-29,2017-03-29,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,84,higher-ed,,annual,12049053,UG,true,CENSUS,2017-03-29,"WILLIAM M. JACOBS, CPA",PARTNER -268000,2016-09-CENSUS-0000235718,TMC DEVELOPMENT WORKING SOLUTIONS,CHIEF EXECUTIVE OFFICER,EMILY GASNER,EMILY@WORKINGSOLUTIONS.ORG,TMC DEVELOPMENT WORKING SOLUTIONS,4156555433,CHIEF EXECUTIVE OFFICER,"930 MONTGOMERY STREET, STE 400",SAN FRANCISCO,CA,911951777,GSA_MIGRATION,No,94133,4159251120,CA,LARKSPUR,PARTNER,101 LARKSPUR LANDING CIRCLE # 200,949391750,USA,MICHAEL SMITH,MSMITH@WMSHB.COM,"WILSON MARKLE STUCKEY HARDESTY & BOTT, LLP",,263789391,,21,2024-01-30,2017-04-30,2017-05-01,2017-04-28,2017-04-30,2016-09-30,2015-10-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,Yes,"11, 14, 21, 59",non-profit,,annual,2206380,UG,true,CENSUS,2017-04-30,MICHAEL SMITH,PARTNER -269000,2016-06-CENSUS-0000193820,CITY OF TRENTON,COMPTROLLER,JANET SCHOENHAAR,JSCHOENHAAR@TRENTONNJ.ORG,CITY OF TRENTON,6099894248,COMPTROLLER,319 EAST STATE STREET,TRENTON,NJ,216001242,GSA_MIGRATION,No,086082099,6096899700,NJ,HAMILTON,PRINCIPAL,3625 QUAKERBRIDGE ROAD,086197648,USA,"WARREN A. BROUDY, CPA, RMA, CGFM",WBROUDY@MERCADIEN.COM,"MERCADIEN, P.C. CPAS",,223271712,,97,2024-01-30,2017-04-23,2017-04-24,2017-04-20,2017-04-23,2016-06-30,2015-07-01,2016,single-audit,not_gaap,regulatory_basis,Yes,unmodified_opinion,No,Yes,Yes,No,Yes,750000,No,"20, 14, 10, 97, 66, 16, 93",local,,annual,13250034,UG,true,CENSUS,2017-04-23,"WARREN A. BROUDY, CPA, RMA, CGFM",PRINCIPAL -270000,2016-06-CENSUS-0000201467,MOUNT VERNON CITY SCHOOL DISTRICT,TREASURER,JUDY FORNEY,JFORNEY@MVCSD.US,MOUNT VERNON CITY SCHOOL DISTRICT,7403977422,TREASURER,300 NEWARK ROAD,MOUNT VERNON,OH,310718901,GSA_MIGRATION,No,43050,8004439275,OH,COLUMBUS,AUDIT MANAGER,88 EAST BROAD STREET,43215,USA,MELISSA KANDEL,MKKANDEL@OHIOAUDITOR.GOV,DAVE YOST,,311334820,,84,2024-01-30,2017-03-20,2017-03-21,2017-03-20,2017-03-20,2016-06-30,2015-07-01,2016,single-audit,not_gaap,cash_basis,No,unmodified_opinion,No,Yes,Yes,Yes,No,750000,No,"84, 10",local,,annual,4041049,UG,true,CENSUS,2017-03-20,MELISSA KANDEL,AUDIT MANAGER -271000,2016-06-CENSUS-0000204030,"STEPPING OUT HOUSING, INC. 121-EH308-NP-CMI-L8",ACCOUNTING MANAGER,SANDRA MARONEY,SANDY.MARONEY@ENCOMPASSCS.ORG,"STEPPING OUT HOUSING, INC. 121-EH308-NP-CMI-L8",8314691700,ACCOUNTING MANAGER,"380 ENCINAL STREET, SUITE 200",SANTA CRUZ,CA,770157232,GSA_MIGRATION,No,95060,8317630997,CA,WATSONVILLE,OWNER,36 B ASPEN WAY,95076,USA,JOHN TEUTEBERG,JJTCPA@CRUZIO.COM,"JOHN J. TEUTEBERG, CPA",,770142495,,14,2024-01-30,2016-12-01,2016-12-02,2016-12-02,2016-12-01,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,865372,UG,true,CENSUS,2016-12-01,JOHN TEUTEBERG,OWNER -272000,2016-06-CENSUS-0000206116,MCINTOSH TRAIL COMMUNITY SERVICE BOARD,CFO,STEFANIE JACKSON,SJACKSON@MCTRAIL.ORG,MCINTOSH TRAIL COMMUNITY SERVICE BOARD,7703588254,CFO,1501-A KALAMAZOO DRIVE,GRIFFIN,GA,582098758,GSA_MIGRATION,No,30224,2294463600,GA,ALBANY,MEMBER,2303 DAWSON ROAD,31708,USA,RYAN INLOW,RINLOW@MJCPA.COM,RYAN INLOW,,580692043,,93,2024-01-30,2017-03-23,2017-03-24,2017-01-31,2017-03-23,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"14, 93, 84",local,,annual,2435662,UG,true,CENSUS,2017-03-23,RYAN INLOW,MEMBER -273000,2016-12-CENSUS-0000219732,GOODWILL VILLAGE WEST II 117-HD018,"PRESIDENT, GORMAN MANAGEMENT COMPANY",THOMAS A. GORMAN,TAGORMAN@AOL.COM,GOODWILL VILLAGE WEST II 117-HD018,9183332244,"PRESIDENT, GORMAN MANAGEMENT COMPANY",398060 WEST 2200 ROAD,BARTLESVILLE,OK,731529929,GSA_MIGRATION,No,74006,9183360008,OK,BARTLESVILLE,PARTNER,302 SE OSAGE AVENUE,74003,USA,DEBORAH E. MUEGGENBORG,DEBBIE@ARCHAMBOFINANCIAL.COM,"STOTTS, ARCHAMBO, MUEGGENBORG & BARCLAY, PC",,471209122,,14,2024-01-30,2017-03-30,2017-03-31,2017-03-30,2017-03-30,2016-12-31,2016-01-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,939880,UG,true,CENSUS,2017-03-30,DEBORAH E. MUEGGENBORG,PARTNER -274000,2016-06-CENSUS-0000246480,"PROTOTYPES MERGED WITH HEALTHRIGHT 360 EFF MARCH 30,2016",CFO,TONY DUONG,TDUONG@HEALTHRIGHT360.ORG,"PROTOTYPES MERGED WITH HEALTHRIGHT 360 EFF MARCH 30,2016",4157623717,CFO,1735 MISSION STREET,SAN FRANCISCO,CA,946129071,GSA_MIGRATION,No,94103,4155683291,CA,SAN FRANCISCO,PARTNER,44 MONTGOMERY STREET,94104,USA,GRANT LAM,GRANT.LAM@ARMANINOLLP.COM,ARMANINO LLP,,946214841,,93,2024-01-30,2017-10-10,2017-09-29,2017-10-11,2017-10-10,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,750000,Yes,"93, 16, 14",non-profit,,annual,9649915,UG,true,CENSUS,2017-10-10,GRANT LAM,PARTNER -275000,2016-06-CENSUS-0000220944,"RICHLAND RESIDENCES, INC. NFP",CFO,MARY BUCK,MBUCK56@HOTMAIL.COM,"RICHLAND RESIDENCES, INC. NFP",6183954309,CFO,RR1 4 MICAH DRIVE,OLNEY,IL,371221757,GSA_MIGRATION,No,62450,6183824151,IL,CARMI,CPA,113 E MAIN STREET,62821,USA,H. KEITH BOTSCH,CPA@BOTSCH.COM,"BOTSCH & ASSOCIATES, CPA'S, LLC",,371412029,,14,2024-01-30,2016-11-02,2016-11-03,2016-11-03,2016-11-02,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,Yes,No,750000,Yes,14,non-profit,,annual,1910588,UG,true,CENSUS,2016-11-02,H. KEITH BOTSCH,CPA -276000,2016-06-CENSUS-0000231359,SPANISH EDUCATION DEVELOPMENT (SED) CENTER,EXECUTIVE DIRECTOR,MARTHA C EGAS,MARTHAEGAS@SEDCENTER.ORG,SPANISH EDUCATION DEVELOPMENT (SED) CENTER,2027224404,EXECUTIVE DIRECTOR,4110 KANSAS AVENUE NW,WASHINGTON,DC,237147887,GSA_MIGRATION,No,200115704,7037266475,VA,ASHBURN,PRESIDENT,43482 CROSS BREEZE PLACE,20147,USA,JOHN BEHRENS,JOHN@BEHRENSCPA.COM,JOHN BEHRENS,,541556200,,93,2024-01-30,2016-11-22,2016-11-22,2016-11-21,2016-11-22,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"98, 93, 99",non-profit,,annual,2757173,UG,true,CENSUS,2016-11-22,JOHN BEHRENS,PRESIDENT -277000,2016-09-CENSUS-0000214499,GUAM DEPARTMENT OF EDUCATION,"DEPUTY SUPERINTENDENT, FINANCE & ADMIN",TALING M. TAITANO,TMTAITANO@GDOE.NET,GUAM DEPARTMENT OF EDUCATION,6713001556,"DEPUTY SUPERINTENDENT, FINANCE & ADMIN",500 MARINER AVENUE,BARRIGADA,GU,660491518,GSA_MIGRATION,No,969131608,6716463884,GU,TAMUNING,PARTNER,361 SOUTH MARINE CORPS DRIVE,96913,USA,DANIEL S. FITZGERALD,DAFITZGERALD@DELOITTE.COM,DELOITTE & TOUCHE LLP,,980047535,84,,2024-01-30,2017-05-16,2017-05-07,2017-05-16,2017-05-16,2016-09-30,2015-10-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,1930497,No,"84, 10, 93, 15, 12",state,,annual,64349925,UG,true,CENSUS,2017-05-16,DANIEL S. FITZGERALD,PARTNER -278000,2016-06-CENSUS-0000238046,PIEDMONT INTERNATIONAL UNIVERSITY,CFO,CHRIS RONK,RONKC@PIEDMONTU.EDU,PIEDMONT INTERNATIONAL UNIVERSITY,3367147987,CFO,420 S. BROAD ST,WINSTON SALEM,NC,560594591,GSA_MIGRATION,No,271015025,3367250635,NC,WINSTON SALEM,AUDIT PARTNER,2160 COUNTRY CLUB RD,27104,USA,RICHARD J. TAMER,RTAMER@CANNON.PRO,"CANNON & COMPANY, L.L.P.",,560727655,,84,2024-01-30,2017-01-19,2017-01-19,2017-01-19,2017-01-19,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,84,non-profit,,annual,4156569,UG,true,CENSUS,2017-01-19,RICHARD J. TAMER,AUDIT PARTNER -279000,2017-06-CENSUS-0000078360,RANDOLPH-MACON COLLEGE,DIRECTOR OF BUDGET AND FINANCIAL ANALYSI,CAROLINE BUSCH,CBUSCH@RMC.EDU,RANDOLPH-MACON COLLEGE,8047523267,DIRECTOR OF BUDGET AND FINANCIAL ANALYSI,PO BOX 5005,ASHLAND,VA,540505940,GSA_MIGRATION,No,23005,5403450936,VA,ROANOKE,PARTNER,319 MCCLANAHAN STREET,24014,USA,JOHN HASH,JHASH@BECPAS.COM,BROWN EDWARDS,,540504608,,84,2024-01-30,2018-03-25,2018-03-26,2018-03-26,2018-03-25,2017-06-30,2016-06-30,2017,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"07, 93, 84, 47, 45, 15",higher-ed,,annual,13274702,UG,true,CENSUS,2018-03-25,JOHN HASH,PARTNER -280000,2016-06-CENSUS-0000244215,HEARTLAND CENTER FOR BEHAVIORAL CHANGE,CFO,ANNE JETTER,AJETTER@HEARTLANDCBC.ORG,HEARTLAND CENTER FOR BEHAVIORAL CHANGE,8162149505,CFO,1730 PROSPECT AVENUE,KANSAS CITY,MO,431262765,GSA_MIGRATION,No,64127,8167562620,MO,KANSAS CITY,MANAGING MEMBER,"920 MAIN STREET, SUITE 640",64105,USA,BRIAN WELCH,BWELCH@WELCHCPAFIRM.COM,"WELCH & ASSOCIATES, L.L.C.",,431794646,,93,2024-01-30,2017-03-29,2017-03-24,2017-03-30,2017-03-29,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,93,non-profit,,annual,1448045,UG,true,CENSUS,2017-03-29,BRIAN WELCH,MANAGING MEMBER -281000,2016-12-CENSUS-0000025062,MIDATLANTIC EMPLOYERS' ASSOCIATION,CHIEF OPERATING OFFICER,SHAWN MCGUIRE,SMCGUIRE@MEAINFO.ORG,MIDATLANTIC EMPLOYERS' ASSOCIATION,6109947636,CHIEF OPERATING OFFICER,"234 MALL BLVD., SUITE 200",KING OF PRUSSIA,PA,230837700,GSA_MIGRATION,No,19406,6105653930,PA,MEDIA,PARTNER,"2 WEST BALTIMORE AVE, SUITE 210",19063,USA,COLLEEN VARALLO,CVARALLO@WIPFLI.COM,WIPFLI LLP,,390758449,,11,2024-01-30,2017-05-09,2017-05-10,2017-05-10,2017-05-09,2016-12-31,2016-01-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"11, 17",non-profit,,annual,1450137,UG,true,CENSUS,2017-05-09,COLLEEN VARALLO,PARTNER -282000,2016-06-CENSUS-0000245690,ROMAN CATHOLIC BISHOP OF SPRINGFIELD,FINANCE OFFICER,WILLIAM LABROAD,WILLIAM.LABROAD@DIOSPRINGFIELD.ORG,ROMAN CATHOLIC BISHOP OF SPRINGFIELD,4134520687,FINANCE OFFICER,P. O. BOX 1730,SPRINGFIELD,MA,042106751,GSA_MIGRATION,No,011021730,4137266852,MA,BOSTON,MEMBER OF THE FIRM,99 HIGH STREET,021102320,USA,MARTIN CAINE,MCAINE@WOLFANDCO.COM,"WOLF & COMPANY, P.C.",,042689883,,97,2024-01-30,2017-02-06,2017-02-07,2017-02-07,2017-02-06,2016-06-30,2015-07-01,2016,program-specific,unmodified_opinion,,,,No,No,No,No,No,750000,No,97,non-profit,,annual,805767,UG,true,CENSUS,2017-02-06,MARTIN CAINE,MEMBER OF THE FIRM -283000,2016-06-CENSUS-0000006625,FAMILY CENTERS INC.,VICE PRESIDENT,PIERA IORI,PIORI@FAMILYCENTERS.ORG,FAMILY CENTERS INC.,2038694848,VICE PRESIDENT,40 ARCH STREET,GREENWICH,CT,060646656,GSA_MIGRATION,No,06836,2033245117,CT,STAMFORD,PARTNER,666 SUMMER STREET,06901,USA,MICHAEL V BUZZEO,MIKEBUZZEOCPA@OPTIMUM.NET,HAIMS BUZZEO & COMPANY P.C.,,061135365,,93,2024-01-30,2017-01-17,2016-12-28,2017-01-18,2017-01-17,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"14, 10, 97, 84, 93",non-profit,,annual,2166580,UG,true,CENSUS,2017-01-17,MICHAEL V BUZZEO,PARTNER -284000,2016-06-CENSUS-0000045573,"PLANNED PARENTHOOD OF INDIANA AND KENTUCKY, INC.",CHIEF FINANCIAL OFFICER,CHERYL GATZMER,CHERYL.GATZMER@PPINK.ORG,"PLANNED PARENTHOOD OF INDIANA AND KENTUCKY, INC.",3176374378,CHIEF FINANCIAL OFFICER,"200 SOUTH MERIDIAN STREET, SUITE 400",INDIANAPOLIS,IN,350874276,GSA_MIGRATION,No,46225,5029923822,KY,LOUISVILLE,DIRECTOR,2650 EASTPOINT PKWY STE. 300,40223,USA,STEVE JONES,SJONES@BLUEANDCO.COM,"BLUE & CO., LLC",,351178661,,93,2024-01-30,2017-03-26,2017-03-24,2017-03-27,2017-03-26,2016-06-30,2015-07-01,2016,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,No,93,non-profit,,annual,1675441,UG,true,CENSUS,2017-03-26,STEVE JONES,DIRECTOR -285000,2016-12-CENSUS-0000036766,"PLUM PRESBYTERIAN SENIOR HOUSING, INC. DBA PLUM CREEK ACRES 033-11073",DIRECTOR OF BUDGET & FCIAL COMPLIANCE,MIKE STOKES,MSTOKES@SRCARE.ORG,"PLUM PRESBYTERIAN SENIOR HOUSING, INC. DBA PLUM CREEK ACRES 033-11073",4128266529,DIRECTOR OF BUDGET & FCIAL COMPLIANCE,1215 HULTON ROAD,OAKMONT,PA,251471814,GSA_MIGRATION,No,15139,4126976405,PA,PITTSBURGH,PARTNER,20 STANWIX STREET,15222,USA,TODD BOSLAU,TODD.BOSLAU@BAKERTILLY.COM,BAKER TILLY VIRCHOW KRAUSE,,390859910,,14,2024-01-30,2017-04-19,2017-04-18,2017-04-20,2017-04-19,2016-12-31,2016-01-01,2016,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,2218929,UG,true,CENSUS,2017-04-19,TODD BOSLAU,PARTNER -286000,2018-06-CENSUS-0000135674,"CITY OF NEW BRITAIN, CONNECTICUT",ASSISTANT FINANCE DIRECTOR,REBECCA SALERNI,BSALERNI@NEWBRITAINCT.GOV,"CITY OF NEW BRITAIN, CONNECTICUT",8608263434,ASSISTANT FINANCE DIRECTOR,27 WEST MAIN STREET,NEW BRITAIN,CT,066001874,GSA_MIGRATION,No,06051,8605614000,CT,WEST HARTFORD,PARTNER,29 SOUTH MAIN STREET,06107,USA,LESLIE ZOLL,LZOLL@BLUMSHAPIRO.COM,"BLUM, SHAPIRO & COMPANY, P.C.",,061009205,,14,2024-01-30,2019-01-23,2019-01-18,2019-01-16,2019-01-23,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,809334,No,"16, 97, 10, 20, 14, 84, 93",local,,annual,26977789,UG,true,CENSUS,2019-01-23,LESLIE ZOLL,PARTNER -287000,2018-06-CENSUS-0000156753,EAST ORANGE BOARD OF EDUCATION,BOARD SECRETARY/SCHOOL BUS. ADMIN.,CRAIG SMITH,CRAIG.SMITH@EASTORANGE.K12.NJ.US,EAST ORANGE BOARD OF EDUCATION,9732665700,BOARD SECRETARY/SCHOOL BUS. ADMIN.,199 FOURTH AVENUE,EAST ORANGE,NJ,226001770,GSA_MIGRATION,No,07017,2017917100,NJ,FAIR LAWN,PARTNER,17-17 ROUTE 208 NORTH,07410,USA,DIETER P. LERCH,DLERCH@LVHCPA.COM,"LERCH, VINCI & HIGGINS, LLP",,223015339,,84,2024-01-30,2019-03-06,2019-03-07,2019-03-06,2019-03-06,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,No,Yes,Yes,750000,No,"10, 84, 93",local,,annual,15147143,UG,true,CENSUS,2019-03-06,DIETER P. LERCH,PARTNER -288000,2018-06-CENSUS-0000182887,NORTHWEST OREGON HOUSING AUTHORITY,FINANCE DIRECTOR,JUDY PAGAN,JUDY@NWOHA.ORG,NORTHWEST OREGON HOUSING AUTHORITY,5039945023,FINANCE DIRECTOR,147 SOUTH MAIN AVE,WARRENTON,OR,450487504,GSA_MIGRATION,No,971460000,8018729470,UT,BOUNTIFUL,CPA,PO BOX 1516,840110000,USA,BRENT STRATTON,BRENT@B2ACPA.COM,STRATTON & MILLER LLC,,811010413,,14,2024-01-30,2020-02-04,2020-02-05,2019-03-21,2020-02-04,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,No,Yes,No,No,750000,No,"10, 14",state,,annual,8300469,UG,true,CENSUS,2020-02-04,BRENT STRATTON,CPA -289000,2018-12-CENSUS-0000194351,CITY OF MARION,CONTROLLER,JULIE FLORES,JFLORES@CITYOFMARION.IN.GOV,CITY OF MARION,7656684460,CONTROLLER,301 SOUTH BRANSON STREET,MARION,IN,356001102,GSA_MIGRATION,No,46952,3172322514,IN,INDIANAPOLIS,STATE EXAMINER,"302 WEST WASHINGTON STREET, ROOM E418",46204,USA,"PAUL D JOYCE, CPA",FEDDATA@SBOA.IN.GOV,INDIANA STATE BOARD OF ACCOUNTS,,356000158,,20,2024-01-30,2019-09-19,2019-09-20,2019-09-20,2019-09-19,2018-12-31,2017-12-31,2018,single-audit,not_gaap,regulatory_basis,No,unmodified_opinion,No,No,Yes,No,No,750000,No,"16, 14, 20",local,,annual,1181051,UG,true,CENSUS,2019-09-19,"PAUL D JOYCE, CPA",STATE EXAMINER -290000,2018-09-CENSUS-0000221874,"CATHERINE BOOTH FRIENDSHIP HOUSE RESIDENCE, INC. 113-EE021","HUD SPECIALIST, THQ (LEGAL)",SRIPARNA MITRA,SRIPARNA.MITRA@USS.SALVATIONARMY.ORG,"CATHERINE BOOTH FRIENDSHIP HOUSE RESIDENCE, INC. 113-EE021",4047286700,"HUD SPECIALIST, THQ (LEGAL)",1901 EAST SEMINARY DRIVE,FORT WORTH,TX,030494708,GSA_MIGRATION,No,76119,4048479447,GA,ATLANTA,PARTNER,"3560 LENOX RD NE, SUITE 2800",30326,USA,DAN WORRALL,DAN.WORRALL@COHNREZNICK.COM,"COHNREZNICK, LLP",,221478099,,14,2024-01-30,2019-06-23,2019-06-20,2019-06-24,2019-06-23,2018-09-30,2017-09-30,2018,single-audit,unmodified_opinion,,,,No,No,Yes,No,No,750000,No,14,non-profit,,annual,6544865,UG,true,CENSUS,2019-06-23,DAN WORRALL,PARTNER -291000,2018-06-CENSUS-0000247643,WALHALLA COMMUNITY HOSPITAL ASSOCIATION,ADMINISTRATOR,DEBRA FRASER,DEBRA.FRASER@PEMBILIERNC.COM,WALHALLA COMMUNITY HOSPITAL ASSOCIATION,7015493310,ADMINISTRATOR,500 DELANO AVE,WALHALLA,ND,450278093,GSA_MIGRATION,No,582829704,7012398543,ND,FARGO,PARTNER,4310 17TH AVE S,58103,USA,ASHLEY BRANDT-DUDA,ABRANDTDUDA@EIDEBAILLY.COM,EIDE BAILLY LLP,,450250958,,10,2024-01-30,2018-10-23,2018-10-24,2018-10-24,2018-10-23,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,Yes,No,No,Yes,750000,No,10,non-profit,,annual,4322234,UG,true,CENSUS,2018-10-23,ASHLEY BRANDT-DUDA,PARTNER -292000,2018-06-CENSUS-0000247915,"CAREER ACADEMY OF SOUTH BEND, INC.",BUSINESS MANAGER,KIM RICHARDSON,KLNER@COMCAST.NET,"CAREER ACADEMY OF SOUTH BEND, INC.",5742999800,BUSINESS MANAGER,3801 CRESCENT CIRCLE,SOUTH BEND,IN,273113436,GSA_MIGRATION,No,46628,5742894011,IN,SOUTH BEND,PARTNER,"210 SOUTH MICHIGAN ST, STE 200",46601,USA,MARGENE ZINK,MZINK@KLCPAS.COM,"KRUGGEL, LAWTON & COMPANY, LLC",,351307701,,84,2024-01-30,2020-03-08,2020-03-06,2020-03-06,2020-03-08,2018-06-30,2017-06-30,2018,single-audit,unmodified_opinion,,,,No,Yes,Yes,Yes,Yes,750000,No,"84, 10",non-profit,,annual,1505032,UG,true,CENSUS,2020-03-08,MARGENE ZINK,PARTNER -293000,2023-06-GSAFAC-0000021497,Sean Witte,Associate VP Financial Operations and Controller,SEAN WITTE,WITTES@DICKINSON.EDU,DICKINSON COLLEGE,7172548149,ASSOC VP FOR FINANCIAL OPERATIONS/CONTROLLER,28 N. COLLEGE STREET,CARLISLE,PA,231365954,G4KQDKZBL848,No,17013,6177610600,MA,BOSTON,SHAREHOLDER,500 BOYLSTON STREET,02116,USA,PATRICK QUINN,PQUINN@CBIZ.COM,MAYER HOFFMAN MCCANN PC,,431947695,,84,2024-01-25,2024-01-25,2024-02-08,2024-02-08,2024-02-08,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,higher-ed,,annual,12213900,UG,true,GSAFAC,2024-02-08,PATRICK QUINN,SHAREHOLDER -294000,2023-08-GSAFAC-0000024423,Alfredo Vela,Chief Financial Officer,Alfredo Vela,alfredo.vela@cotullaisd.net,Cotulla Independent School District,8308793073,Finance Director,310 N. Main Street,Cotulla,TX,746003466,UE6CQEMSBJY3,No,78014,2102271389,TX,San Antonio,Manager,207 Arden Grove St,78215,USA,Ruben Martinez,rmartinez@gga-cpa.biz,Garza/Gonzalez & Associates,,741867466,,84,2024-02-09,2024-02-12,2024-02-12,2024-02-14,2024-02-14,2023-08-31,2022-09-01,2023,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,Yes,00,local,,annual,3521699,UG,true,GSAFAC,2024-02-14,RUBEN MARTINEZ,AUDIT MANAGER -295000,2023-08-GSAFAC-0000023208,Vicki Taylor,Director of Business Services,Vicki Taylor,vicki.taylor@esu6.org,Educational Service Unit No. 6,4027617011,Director of Business Services,210 5th Street,Milford,NE,470495481,DGKDW9G4J6W5,No,68405,4024799300,NE,Lincoln,Auditor,"1248 O Street, Suite 500",68508,USA,Kerry Gustafsson,gustafsson@danacole.com,"Dana F. Cole & Company, LLP",,470526649,,84,2024-02-02,2024-02-06,2024-02-21,2024-02-21,2024-02-21,2023-08-31,2022-09-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,00,local,,annual,1500123,UG,true,GSAFAC,2024-02-21,"KERRY GUSTAFSSON, CPA",PARTNER -296000,2023-06-GSAFAC-0000001380,Sherri Rowland,Controller,Sarah Balas,sbalas@clemson.edu,Clemson University,8646566598,Sr. Grant Supervisor,G06 Sikes Hall,Clemson,SC,576000254,H2BMNX7DSKU8,No,29634,4103088064,MD,Timonium,Principal,"1966 Greenspring Drive, Suite 300",21093,USA,Christina Bowman,christina.bowman@claconnect.com,Clifton Larson Allen LLP (CLA),,410746749,84,,2023-10-09,2023-11-08,2024-02-16,2024-02-27,2024-02-27,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,Yes,3000000,Yes,00,higher-ed,,annual,302699610,UG,true,GSAFAC,2024-02-27,Christina Bowman,Principal -297000,2023-06-GSAFAC-0000028111,Charles Peckio,Superintendent,Richard Peckio,rpeckio@frink.k12.ok.us,Frink-Chambers School District,9184232434,Superintendent,PO Box 699,McAlester,OK,770648127,YR6NN8E1FP55,No,74502,9182508838,OK,Broken Arrow,Partner,2101 N Willow Ave,74012,USA,Kerry Patten,kerry.patten@pattencpa.net,"Patten & Odom, CPAs, PLLC",,931904036,,84,2024-03-04,2024-03-04,2024-03-04,2024-03-04,2024-03-04,2023-06-30,2022-07-01,2023,single-audit,not_gaap,other_basis,No,qualified_opinion,No,No,No,No,No,750000,No,00,local,,annual,908838,UG,true,GSAFAC,2024-03-04,Kerry Patten,Partner -298000,2023-05-GSAFAC-0000027922,Timothy Walker,COO,Timothy Walker,tw@eechealth.com,Elaine Ellis Center of Health,2028032340,COO,1627 Kenilworth AVE NE,Washington,DC,273048576,RFJKYDVW7NJ3,No,20019,2023935600,DC,Washington,Partner,1090 Vermont AVE Suite 250 NW,20005,USA,Stephanie Lane,slane@bertsmithco.com,Bert Smith & CO,,521094722,,93,2024-03-01,2024-03-11,2024-03-11,2024-03-11,2024-03-11,2023-05-31,2022-06-01,2023,single-audit,unmodified_opinion,,,,No,Yes,No,No,Yes,750000,No,00,non-profit,,annual,4182911,UG,true,GSAFAC,2024-03-11,Stephanie Lane,Partner -299000,2023-06-GSAFAC-0000030499,Melinda Brouillard,Interim Finance Director,Melinda Brouillard,mbrouillard@cumberlandri.org,"Town of Cumberland, Rhode Island",4017282400,Deputy Finance Director,45 Broad Street,Cumberland,RI,056000115,QRL4RNF9E5L8,No,02864,4016004722,RI,Providence,Partner,"100 Westminster Street, Suite 500",02903,USA,Kyle Connors,kyle.connors@marcumllp.com,Marcum LLP,,111986323,,84,2024-03-15,2024-03-15,2024-03-15,2024-03-15,2024-03-15,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,Yes,No,No,750000,No,00,local,,annual,9686909,UG,true,GSAFAC,2024-03-15,Kyle Connors,Partner -300000,2022-12-GSAFAC-0000008542,Sabine Cox,Comptroller,Sabine Cox,scox@ehmchm.org,Chestnut Hill Housing Inc,2032304809,Comptroller EHM,40 Edgemont Lane,Wolcott,CT,061451581,JH36NMXLMH88,No,06716,2032977700,CT,Woodbridge,Partner,16 Lunar Drive,06525,USA,Brian S Borgerson CPA,bborg@baileymoore.com,Bailey Moore Glazer Schaeffer & Proto LLP,,060674931,,14,2023-11-22,2024-03-21,2024-03-21,2024-03-21,2024-03-21,2022-12-31,2022-01-01,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,3395066,UG,true,GSAFAC,2024-03-21,"Brian S Borgerson, CPA",Partner -301000,2023-06-GSAFAC-0000005142,C. Shawn Boehringer,Executive Director,SHAWN BOEHRINGER,SBOEHRINGER@LASP.ORG,"LEGAL AID OF SOUTHEASTERN PENNSYLVANIA, INC.",6102755400,EXECUTIVE DIRECTOR,625 SWEDE STREET,NORRISTOWN,PA,231901014,MNVEJFEMKAJ4,No,19401,7172321230,PA,HARRISBURG,ENGAGEMENT PARTNER,"1800 LINGLESTOWN ROAD, SUITE 306",17110,USA,LISA RITTER,LRITTER@MD-CPAS.COM,"MAHER DUESSEL, CPAS",,251622758,,09,2023-11-02,2024-03-14,2024-03-14,2024-03-25,2024-03-25,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,00,non-profit,,annual,3578623,UG,true,GSAFAC,2024-03-25,Lisa Ritter,Engagement Partner -302000,2023-06-GSAFAC-0000023768,Tina Leingang,Business Manager,Tina Leingang,TINA.LEINGANG@K12.ND.US,STANDING ROCK COMMUNITY GRANT SCHOOL,7018542142,Business Manager,9889 HWY 24,Fort Yates,ND,450407044,EYC8CFBB3N89,No,58538,9897321156,MI,GAYLORD,SENIOR AUDIT PARTNER,215 S COURT AVE,49735,USA,J. Kurt Tucker,KURT@MWP.US.COM,MIDWEST PROFESSIONALS,,383571340,,15,2024-02-06,2024-03-26,2024-03-26,2024-03-27,2024-03-27,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,tribal,,annual,14627976,UG,true,GSAFAC,2024-03-27,Christopher Preville,Auditor -303000,2023-06-GSAFAC-0000029224,Dennis Meyer,Budget & Fiscal Director,Dennis Meyer,DMMEYER@LANCASTER.NE.GOV,"Lancaster County, Nebraska",4024416869,Budget and Fiscal Officer,"555 South 10th Street, Room 110",Lincoln,NE,476006482,PGJKWY8LBAR1,No,68508,8167514027,NE,Omaha,Partner,1299 Farnam St.,68102,USA,Kevin Smith,kevin.smith@rsmus.com,RSM US LLP,,420714325,,21,2024-03-08,2024-03-28,2024-03-28,2024-03-28,2024-03-28,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,local,,annual,20064033,UG,true,GSAFAC,2024-03-28,Kevin Smith,Partner -304000,2023-06-GSAFAC-0000033720,Diana Snyder,Finance Director,Diana Snyder,dsnyder@hrdewv.org,"Marion Unity Apartments, Inc.",3042968223,Finance Director,1369 Stewartstown Road,Morgantown,WV,550678214,RNCDTKSYJ5K2,No,26505,3043434603,WV,Charleston,Partner,P.O. Box 1991,25327,USA,R. Gabe Rushden,gabe.rushden@blaircpas.com,Blair & Company,,844083294,,14,2024-03-26,2024-03-28,2024-03-28,2024-03-29,2024-03-29,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,5072057,UG,true,GSAFAC,2024-03-29,R. Gabe Rushden,Partner -305000,2022-06-GSAFAC-0000035727,Christy Ramirez,Finance Director,CHRISTY RAMIREZ,CRAMIREZ@SPCITY.ORG,CITY OF SANTA PAULA,8059334211,FINANCE DIRECTOR,970 E. VENTURA STREET,SANTA PAULA,CA,956000791,TDLLJXMQYPJ9,No,93060,9098890871,CA,SAN BERNARDINO,PARTNER,735 E. CARNEGIE DRIVE SUITE 100,92408,USA,BRIANNA SCHULTZ,BSCHULTZ@RAMSCPA.NET,"ROGERS, ANDERSON, MALODY & SCOTT, LLP.",,952662063,,97,2024-03-29,2024-03-29,2024-03-29,2024-03-29,2024-03-29,2022-06-30,2021-07-01,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,00,local,,annual,3002983,UG,true,GSAFAC,2024-03-29,Brianna Schultz,Partner -306000,2023-06-GSAFAC-0000036275,Yolanda Brumfield,Director of Finance,YOLANDA BRUMFIELD,YBRUMFIELD@SOWIB.ORG,SOUTHWESTERN OREGON WORKFORCE INVESTMENT BOARD,8445326893,FINANCE DIRECTOR,PO BOX 415,COOS BAY,OR,473873798,G3LVWKKZN9L9,No,97420,3605333370,WA,Montesano,PRINCIPAL,"324 S Main Street, Unit A",98563,USA,Lonnie Rich,lonnie@aiken-sanders.com,AIKEN & SANDERS INC. PS,,910870697,,11,2024-04-01,2024-04-01,2024-04-01,2024-04-01,2024-04-01,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,non-profit,,annual,4946972,UG,true,GSAFAC,2024-04-01,Lonnie Rich,Principal -307000,2023-09-GSAFAC-0000037293,Timothy S Brown,CFO,Tim Brown,tbrown@pinesbhs.org,Branch County Community Mental Health Authority,5172782129,Chief Financial Officer,200 Orleans Blvd.,Coldwater,MI,383622335,P27VJPZMNB76,No,49036,9894636123,MI,Alma,Partner,"525 West Warwick Drive, Suite A",48801,USA,Christina Schaub,christina.schaub@rpccpas.com,"Roslund, Prestage & Company, P.C.",,382462587,,93,2024-04-10,2024-04-10,2024-04-10,2024-04-10,2024-04-10,2023-09-30,2022-10-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,00,local,,annual,2470496,UG,true,GSAFAC,2024-04-10,Christina Schaub,Partner -308000,2023-12-GSAFAC-0000036438,Takisha Artis,Controller,TAKISHA ARTIS,takisha@hsrpro.com,"SILENT COOPERATIVE APARTMENTS, INC.",8158069990,OUTSOURCED CFO,2500 WEST BELMONT AVENUE,CHICAGO,IL,363390850,NELBSURV5F29,No,60618,8475804138,IL,LINCOLNSHIRE,PARTNER,100 TRI STATE INTERNATIONAL STE 300,60069,USA,ALEX PEKLER,ALEX.PEKLER@WIPFLI.COM,WIPFLI LLP,,390758449,,14,2024-04-02,2024-04-22,2024-04-22,2024-04-22,2024-04-22,2023-12-31,2023-01-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,non-profit,,annual,6389958,UG,true,GSAFAC,2024-04-22,Alex Pekler,Partner -309000,2023-12-GSAFAC-0000023806,Julie Reed,Housing Accounting Manager,JULIE REED,JREED@UCHINC.ORG,"UCC II, INC. D/B/A STERLING PLACE 067-EE047-WAH",7403824885,MANAGER OF HOUSING ACCOUNTING,170 East Center Street P.O. Box 1806,MARION,OH,341789176,P9YSGYCPJNV1,No,43302,6148493000,OH,COLUMBUS,ENGAGEMENT PARTNER,250 SOUTH HIGH STREET SUITE 100,43215,USA,JASON REES,Jason.rees@plantemoran.com,"PLANTE & MORAN, PLLC",,381357951,,14,2024-02-07,2024-04-28,2024-04-28,2024-04-30,2024-04-30,2023-12-31,2023-01-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,non-profit,,annual,4141930,UG,true,GSAFAC,2024-04-30,Jason T Rees,Partner -310000,2023-06-GSAFAC-0000036113,Eric C Hern,CFO,Erin Hern,ehern@guidancecenter.net,The Guidance Center,8143626536,Senior Director of Operations & Chief Financial Officer,110 Campus Drive,Bradford,PA,251469392,PB38PFWQMND7,No,16701,4123677102,PA,Pittsburgh,Partner,"3800 McKnight East Drive, Suite 3805",15237,USA,Brian Chruscial,bchruscial@zallc.org,Zelenkofske Axelrod LLC,,233022325,,93,2024-04-01,2024-04-01,2024-04-01,2024-04-09,2024-05-13,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,00,non-profit,,annual,3035973,UG,true,GSAFAC,2024-05-13,Brian Chruscial,Partner -311000,2023-12-GSAFAC-0000038239,Beth Long,CFO,Beth Long,beth.long@tphtrust.org,"South Main Street Housing Trust, Inc.",8022917000,CFO,"226 Holiday Drive, Suite 20",White River Junction,VT,030364340,FWBVY17ZZU35,No,05001,2077801100,ME,South Portland,Partner,324 Gannett Drive,04106,USA,Bridget Sylvester,bsylvester@otisatwell.com,Otis Atwell,,203690847,,10,2024-04-18,2024-05-23,2024-05-23,2024-05-23,2024-05-23,2023-12-31,2023-01-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,non-profit,,annual,13424945,UG,true,GSAFAC,2024-05-23,Bridget Sylvester,Partner -312000,2023-12-GSAFAC-0000042594,Peter Halle,President,Peter Halle,peterhalle@yahoo.com,"Salisbury Housing Committee, Inc.",8604350049,President,"34 Cobble Street, P.O. Box 10",Salisbury,CT,115309016,LCCQX834NCQ7,No,06068,2032489341,CT,North Haven,Partner,296 State St,06473,USA,Michele Loso Boisvert,mloso@sewardmonde.com,Seward and Monde,,060530830,,10,2024-06-03,2024-06-04,2024-06-04,2024-06-04,2024-06-04,2023-12-31,2023-01-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,10,non-profit,,annual,1124543,UG,true,GSAFAC,2024-06-04,Michele Loso Boisvert,Partner -313000,2023-06-GSAFAC-0000042985,Lori Willis,Business Manager,LORI WILLIS,LORI.WILLIS@SMACKOVER.NET,SMACKOVER-NORPHLET SCHOOL DISTRICT NO. 39,8707253132,BUSINESS MANAGER,112 E 8TH ST,SMACKOVER,AR,716021546,HTUUC34B8SX5,No,71762,5012252133,AR,LITTLE ROCK,PARTNER,P.O. BOX 21675,72221,USA,MICHAEL L. COBB,MCOBB@COBBANDSUSKIE.COM,"COBB AND SUSKIE, LTD.",,710671623,,84,2024-06-05,2024-06-10,2024-06-10,2024-06-17,2024-06-17,2023-06-30,2022-07-01,2023,single-audit,not_gaap,other_basis,Yes,unmodified_opinion,No,No,No,No,No,750000,No,00,local,,annual,2424712,UG,true,GSAFAC,2024-06-17,Melissa Hodgson,Auditor Certifying Official -314000,2023-09-GSAFAC-0000044342,Todd Blount,Fiscal Officer,Todd Blount,tblount@ecpdd.org,"EAST CENTRAL PLANNING & DEVELOPMENT DISTRICT, INC.",6016831220,Fiscal Officer,PO BOX 499,NEWTON,MS,640468881,HCG8JZ9N18K7,No,39345,6624485885,MS,Houston,Partner,109 N Jackson St - PO box 391,38851,USA,Jason Brooks,Jason.Brooks@wws.cpa,"Watkins, Ward and Stafford, PLLC",,640394922,,11,2024-06-18,2024-06-18,2024-06-24,2024-06-25,2024-06-25,2023-09-30,2022-10-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,local,,annual,5683747,UG,true,GSAFAC,2024-06-25,Jason Brooks,Partner -315000,2023-09-GSAFAC-0000046014,Jawan Warren Simpson,Chief Financial Officer,JAWAN SIMPSON,jsimpson@waynecountymi.gov,WAYNE COUNTY HEALTHY COMMUNITIES,3138263320,INTERIM CHEIF FINANCIAL OFFICER,9011 JOSEPH CAMPAU,HAMTRAMCK,MI,300672911,KZMDRDLPTD44,No,48212,3139645270,MI,DETROIT,PRESIDENT,PO BOX 32605,48232,USA,GLEN OLIVACHE,olivache@sbcglobal.net,GLEN OLIVACHE CPA PC,,382770838,,93,2024-06-27,2024-06-28,2024-06-28,2024-06-28,2024-06-28,2023-09-30,2022-10-01,2023,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,No,93,non-profit,,annual,2321019,UG,true,GSAFAC,2024-06-28,GLEN OLIVACHE,PRESIDENT -3000,2023-06-GSAFAC-0000002638,Mike Andreshak,Director of Business Services/CSBO,Mike L Andreshak,mandreshak@ksd140.org,Kirby School District 140,7085326462,Director of Business Services/CSBO,16931 S Grissom Drive,Tinley Park,IL,366004355,XT85V8G9QLM8,No,60477,6305668574,IL,Rockford,"Partner-in-Charge, Government Services","401 W State Street,Suite 509",61101,USA,Anthony Cervini,anthony.cervini@sikich.com,Sikich,,363168081,,84,2023-10-18,2023-10-18,2023-10-18,2023-10-18,2023-10-18,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,00,local,,annual,2880895,UG,true,GSAFAC,2023-10-18,DEFAULT_AUDITOR_CERTIFY_NAME,DEFAULT_AUDITOR_CERTIFY_TITLE -4000,2023-01-GSAFAC-0000003440,ORLANDO TORRES,FINANCE DEPARTMENT DIRECTOR,ORLANDO TORRES,orlando.torres@chs015.com,"CAMUY HEALTH SERVICES, INC.",7872626603,FINANCE DIRECTOR,PO BOX 660,CAMUY,PR,660428652,NJSNXFBJ2LP5,No,00627,7877935353,PR,San Juan,PRESIDENT,PO BOX 194806,00919,USA,JOSE LUIS CARDONA GONZALEZ,jlc@jlcardona.com,José L. Cardona & Co. PSC,,660567451,,93,2023-10-23,2023-10-24,2023-10-24,2023-10-24,2023-10-24,2023-01-31,2022-02-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,non-profit,,annual,4241758,UG,true,GSAFAC,2023-10-24,DEFAULT_AUDITOR_CERTIFY_NAME,DEFAULT_AUDITOR_CERTIFY_TITLE -5000,2023-03-GSAFAC-0000004663,Tadd S. Greenfield,Chief Executive Officer,Gayl Smith,gsmi4@cabinetpeaks.org,St. John's Lutheran Hospital DBA Cabinet Peaks Medical Center,4062837139,Controller,209 Health Park Dr.,Libby,MT,810241755,YMEAZVHDGJ56,No,59923,4068962449,MT,BILLINGS,Partner,401 N 31ST ST STE 1120,59103,USA,Scott Nelson,snelson@eidebailly.com,Eide Bailly LLP,,450250958,,14,2023-10-31,2023-11-01,2023-11-01,2023-11-01,2023-11-01,2023-03-31,2022-04-01,2023,single-audit,unmodified_opinion,,,,No,Yes,Yes,No,No,750000,No,00,non-profit,,annual,22622437,UG,true,GSAFAC,2023-11-01,DEFAULT_AUDITOR_CERTIFY_NAME,DEFAULT_AUDITOR_CERTIFY_TITLE -6000,2023-06-GSAFAC-0000006066,Laura Parnell,Director of Finance,LAURA PARNELL,LBPARNELL@ASD2.ORG,ANDERSON COUNTY SCHOOL DISTRICT TWO,8643697364,DIRECTOR OF FINANCE,10990 BELTON HONEA PATH HIGHWAY,HONEA PATH,SC,576003892,XNC8JL9DLSX5,No,29654,8642325204,SC,Mauldin,PARTNER,"211 E Butler Road, Suite C-6",29662,USA,KEN MEADOWS,KEN.MEADOWS@GFC.CPA,"Greene Finney Cauley, LLP",,522212837,,84,2023-11-08,2023-11-09,2023-11-10,2023-11-14,2023-11-14,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,00,local,,annual,7617713,UG,true,GSAFAC,2023-11-14,DEFAULT_AUDITOR_CERTIFY_NAME,DEFAULT_AUDITOR_CERTIFY_TITLE -7000,2023-06-GSAFAC-0000007575,Jeffrey Carraway,CFO,Jeffrey Carraway,Jeffrey.Carraway@lutheranseniorlife.org,Titusville Senior Housing Corporation,7247422246,CFO,127 Beechwood Drive,Titusville,PA,270115829,N457WM5LA4V1,No,16354,3302664124,OH,Canton,Principal,4334 Munson St.,44718,USA,Kyle Bowers,kyle.bowers@claconnect.com,CliftonLarsonAllen,,410746749,,14,2023-11-16,2023-11-22,2023-11-22,2023-11-27,2023-11-27,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,No,14,non-profit,,annual,1821192,UG,true,GSAFAC,2023-11-27,DEFAULT_AUDITOR_CERTIFY_NAME,DEFAULT_AUDITOR_CERTIFY_TITLE -8000,2023-06-GSAFAC-0000005626,Jessica S. Rogers,Chief Financial Officer,Jessica Rogers,jessica.roger1@nelson.kyschools.us,Nelson County Board of Education,5023314310,Chief Financial Officer,1244 Tom Greer Road,Bardstown,KY,616001240,CBD2FKG94EZ3,No,40004,8598081329,KY,Winchester,Partner,124 Candlewood Drive,40391,USA,Tammy Patrick,tammy@patrickassoc.com,"Patrick & Associates, LLC",,831439494,,84,2023-11-06,2023-12-01,2023-12-01,2023-12-04,2023-12-04,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,unknown,,annual,9280538,UG,true,GSAFAC,2023-12-04,DEFAULT_AUDITOR_CERTIFY_NAME,DEFAULT_AUDITOR_CERTIFY_TITLE -9000,2022-06-GSAFAC-0000010150,Daniel Amato,Executive Director,Daniel Amato,damato@mvap.org,Merrimack Valley Assistance Program,6032260607,Executive Director,170 Lowell Street,Manchester,NH,223396725,MP9VTSBDMVG3,No,03104,6036227070,NH,Manchester,Audit Manager,608 Chestnut Street,03104,USA,Cory Philbrick,cphilbrick@vachonclukay.com,Vachon Clukay & Company PC,,020400031,,14,2023-12-04,2023-12-04,2023-12-04,2023-12-11,2023-12-11,2022-06-30,2021-07-01,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,00,non-profit,,annual,819015,UG,true,GSAFAC,2023-12-11,DEFAULT_AUDITOR_CERTIFY_NAME,DEFAULT_AUDITOR_CERTIFY_TITLE -10000,2023-06-GSAFAC-0000011690,Pamela Rocha,Chief Financial Officer,Pamela Rocha,pamelarocha@nafi.com,"NFI Vermont, Inc.",9788824868,CFO,30 Airport Road,Burlington,VT,030304434,SQT2TJZXMW81,No,05403,5084413300,MA,Quincy,Principal,"4 Batterymarch Park, Suite 100",02169,USA,Jennifer Olivier,jen.olivier@claconnect.com,CliftonLarsonAllen LLP,,410746749,,93,2023-12-08,2023-12-15,2023-12-15,2023-12-18,2023-12-18,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,non-profit,,annual,1227738,UG,true,GSAFAC,2023-12-18,DEFAULT_AUDITOR_CERTIFY_NAME,DEFAULT_AUDITOR_CERTIFY_TITLE -11000,2023-06-GSAFAC-0000014202,Chris Baird,Controller,Chris Baird,chris.baird@g-b.com,Edison Terrace South,5093444904,CFO,"818 W Riverside, Suite 300",Spokane,WA,264362933,EJ98S93Z13T6,No,99201,5097472158,WA,Spokane,CPA,"422 W Riverside Ave, Suite 1420",99201,USA,Darcy Skjothaug,dskjothaug@schoedel.com,Schoedel & Schoedel CPAs PLLC,,910614823,,14,2023-12-19,2023-12-19,2023-12-19,2023-12-21,2023-12-21,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,non-profit,,annual,1726491,UG,true,GSAFAC,2023-12-21,DEFAULT_AUDITOR_CERTIFY_NAME,DEFAULT_AUDITOR_CERTIFY_TITLE -12000,2023-06-GSAFAC-0000015058,Anna Kasprzyk,Chief School Business Official,ANNA KASPRZYK,anna.kasprzyk@district41.org,LAKE VILLA COMMUNITY CONSOLIDATED SCHOOL DISTRICT NO. 41,8473562385,CHIEF SCHOOL BUSINESS OFFICIAL,131 MCKINLEY AVENUE,LAKE VILLA,IL,366004850,HSDYSNMJBCX7,No,60046,8153441300,IL,MCHENRY,PARTNER,"5400 WEST ELM STREET, SUITE 203",60050,USA,CHERYDEN JUERGENSEN,cjuergensen@eccezion.com,ECCEZION,,363614997,,84,2023-12-22,2023-12-22,2023-12-22,2024-01-02,2024-01-02,2023-06-30,2022-07-01,2023,single-audit,"adverse_opinion, not_gaap",cash_basis,No,unmodified_opinion,No,No,No,No,No,750000,No,00,local,,annual,2703240,UG,true,GSAFAC,2024-01-02,DEFAULT_AUDITOR_CERTIFY_NAME,DEFAULT_AUDITOR_CERTIFY_TITLE -13000,2023-06-GSAFAC-0000013380,Suwen Su,"Director, Fiscal Services",SUWEN SU,SSU@ROSEMEAD.K12.CA.US,ROSEMEAD SCHOOL DISTRICT,6263122900,DIRECTOR OF FISCAL SERVICES,3907 ROSEMEAD BLVD,ROSEMEAD,CA,956002586,CQ4LFQM3M1J5,No,91770,3106702745,CA,CULVER CITY,PARTNER,"5800 HANNUM AVE, SUITE E",90230,USA,HADLEY HUI,HHUI@MLHCPAS.COM,"MOSS, LEVY & HARTZHEIM, LLP",,753194011,,84,2023-12-15,2024-01-11,2024-01-11,2024-01-12,2024-01-12,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,local,,annual,7492901,UG,true,GSAFAC,2024-01-12,HADLEY HUI,partner -14000,2023-06-GSAFAC-0000016731,Megan Leonard,Director of Management & Finance,MEGAN LEONARD,MLEONARD@NORPC.ORG,"REGIONAL PLANNING COMMISSION FOR JEFFERSON, ORLEANS, PLAQUEMINES, ST.",5044838525,DIRECTOR OF MANAGEMENT AND FINANCE,10 VETERANS BLVD,NEW ORLEANS,LA,720595531,YJJLYGNK6FE5,No,70124,5048332436,LA,Metairie,Partner,"111 VETERANS BOULEVARD, SUITE 350",70005,USA,Becky Hammond,BHAMMOND@CRICPA.COM,"CARR, RIGGS & INGRAM, LLC",,721396621,,20,2024-01-03,2024-01-11,2024-01-11,2024-01-11,2024-01-11,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,local,,annual,2859431,UG,true,GSAFAC,2024-01-11,Becky Hammond,Partner -15000,2023-06-GSAFAC-0000014261,Denise M Mooney,Treasurer,DENISE MOONEY,MOONEY.D@VANTAGECAREERCENTER.COM,VANTAGE CAREER CENTER,4192385411,TREASURER,818 NORTH FRANKLIN STREET,VAN WERT,OH,341146218,D47MDVJZKMF5,No,45891,9372856677,OH,Columbus,SENIOR AUDIT MANAGER,88 East Broad Street,43215,USA,KATIE EDDY,KAEDDY@OHIOAUDITOR.GOV,Ohio Auditor of State,,311334820,,84,2023-12-20,2023-12-28,2023-12-28,2024-01-17,2024-01-17,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,Yes,Yes,No,750000,No,21,local,,annual,855829,UG,true,GSAFAC,2024-01-17,Katie Eddy,Senior Audit Manager -16000,2023-06-GSAFAC-0000016547,Jordan Tuten,Finance/Human Resources Director,Jordan Tuten,jtuten@ridgelandsc.gov,Town of Ridgeland,8437267503,Finance/HR Director,One Town Square,Ridgeland,SC,566001100,V26EFCE28S31,No,29936,8037393090,SC,West Columbia,Partner,501 State Street,29169,USA,William Hancock,whancock@tbgcpas.com,"The Brittingham Group, LLP",,464116137,,21,2024-01-03,2024-01-03,2024-01-09,2024-01-10,2024-01-23,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,00,local,,annual,863743,UG,true,GSAFAC,2024-01-23,William H Hancock,Engagement Partner -17000,2023-09-GSAFAC-0000021853,Angela Daigle,"Senior Director, Finance and Affordable Housing",Angie Daigle,adaigle@goodwillacadiana.org,"Village du Vie, Inc.",3372615811,Senior Director Finance and Affordable Housing,1104 Tarleton Street,Jeanerette,LA,582089102,E8SEUJ2L75Y9,No,70544,3372323312,LA,Lafayette,Partner,2000 Kaliste Saloom Rd Suite 300,70508,USA,Christine Dunn,christied@dsfcpas.com,"Darnall, Sikes and Frederick CPAs",,720738838,,14,2024-01-26,2024-01-26,2024-01-26,2024-01-27,2024-01-27,2023-09-30,2022-10-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,non-profit,,annual,1031087,UG,true,GSAFAC,2024-01-27,Christine R. Dunn,Partner -18000,2023-06-GSAFAC-0000021348,Joseph Sterbank,Business Administrator,JOSEPH STERBANK,JSTERBANK@ONONDAGACSD.ORG,ONONDAGA CENTRAL SCHOOL DISTRICT,3155525001,SCHOOL BUSINESS ADMINISTRATOR,4466 SOUTH ONONDAGA RD,NEDROW,NY,156002303,VL79J2GZ6JK3,No,13120,5854231860,NY,rochester,partner,"100 chestnut street, suite 1200",14604,USA,thomas zuber,jnobles@mmb-co.com,mmb+co.,,161092347,,84,2024-01-24,2024-01-25,2024-01-29,2024-01-29,2024-02-01,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,00,state,,annual,1617460,UG,true,GSAFAC,2024-02-01,thomas zuber,partner -19000,2023-06-GSAFAC-0000006751,Kristin Rossow,VP of Accounting,Kristin Rossow,kristin.rossow@mosaicinfo.org,Mosaic Housing Corporation VII,4028963884,VP of Accounting,4980 South 118th St,Omaha,NE,470828015,WU6JS45L3Z36,No,68137,7012398627,ND,Fargo,Partner,4310 17th Ave S PO Box 2545,58108,USA,Reggie MacMaster,rmacmaster@eidebailly.com,Eide Bailly LLP,,450250958,,14,2023-11-13,2023-11-13,2024-02-05,2024-02-07,2024-02-07,2023-06-30,2022-07-01,2023,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,00,non-profit,,annual,903868,UG,true,GSAFAC,2024-02-07,Reggie MacMaster,Partner -20000,2022-12-CENSUS-0000258232,VILLAGE OF HAWTHORN WOODS,CHIEF FINANCIAL OFFICER,KATREINA YORK,KYORK@VHW.ORG,VILLAGE OF HAWTHORN WOODS,8474385500,CHIEF FINANCIAL OFFICER,2 LAGOON DRIVE,HAWTHORN WOODS,IL,362726116,WT7DKVMQLN86,No,60047,6305668400,IL,NAPERVILLE,PARTNER,"1415 W. DIEHL ROAD, SUITE 400",60563,USA,BRIAN LEFEVRE,BRIAN.LEFEVRE@SIKICH.COM,SIKICH LLP,,366168081,,21,2024-01-19,2023-05-30,2023-05-30,2023-05-30,2023-05-30,2022-12-31,2021-12-31,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,21,local,,annual,1179310,UG,true,CENSUS,2023-05-30,BRIAN LEFEVRE,PARTNER -21000,2022-09-CENSUS-0000042728,GREAT LAKES COMMUNITY ACTION PARTNERSHIP,CHIEF FINANCIAL OFFICER,DAVID KIPPLEN,DRKIPPLEN@GLCAP.ORG,GREAT LAKES COMMUNITY ACTION PARTNERSHIP,4193348919,CHIEF FINANCIAL OFFICER,127 FRONT STREET,FREMONT,OH,340975934,NJFNHGSBUJU1,No,43420,6082741980,WI,MADISON,PARTNER,PO BOX 8700,537168700,USA,"JOHN HEMMING, CPA",JHEMMING@WIPFLI.COM,WIPFLI LLP,,390758449,,93,2024-01-19,2023-06-07,2023-06-08,2023-06-07,2023-06-07,2022-09-30,2021-09-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,1339061,Yes,"66, 10, 14, 21, 17, 20, 84, 19, 93, 64, 23, 81",non-profit,,annual,48219750,UG,true,CENSUS,2023-06-07,"JOHN HEMMING, CPA",PARTNER -22000,2022-06-CENSUS-0000231406,"EDKEY, INC.",CFO,PATRIC GREER,PGREER@EDKEY.ORG,"EDKEY, INC.",4804613200,CFO,2345 N HORNE,MESA,AZ,743033931,VN5GRJGAAXJ4,No,852031823,4807527728,AZ,TEMPE,PARTNER,2055 E WARNER ROAD STE 101,85284,USA,COLETTE KAMPS,COLETTE.KAMPS@BAKERTILLY.COM,"BAKER TILLY U.S., LLP",,390859910,,84,2024-01-19,2023-01-15,2023-01-13,2023-01-16,2023-01-15,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"84, 10, 93, 99",non-profit,,annual,20290249,UG,true,CENSUS,2023-01-15,COLETTE KAMPS,PARTNER -23000,2022-06-CENSUS-0000218633,"ASI - FLORENCE, INC.",ACCOUNTING MANAGER,CHUCK REUTER,CREUTER@ACCESSIBLESPACE.ORG,"ASI - FLORENCE, INC.",6516457271,ACCOUNTING MANAGER,2550 UNIVERSITY AVE. W. SUITE 330 N,ST. PAUL,MN,631251743,Q12WMUTM7CK6,No,55114,9525428010,MN,MINNEAPOLIS,SHAREHOLDER,"1000 SHELARD PARKWAY, STE. 110",55426,USA,BOB BAKER,BOB@HINRICHSASSOCIATES.COM,"HINRICHS & ASSOCIATES, LTD",,411629155,,14,2024-01-19,2023-02-28,2023-03-01,2023-02-25,2023-02-28,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,14,non-profit,,annual,1663370,UG,true,CENSUS,2023-02-28,BOB BAKER,SHAREHOLDER -24000,2022-06-CENSUS-0000251368,"CATHOLIC CHARITIES OF CENTRAL COLORADO, INC.",CFO,LORI KAPU,LKAPU@CCHARITIESCC.ORG,"CATHOLIC CHARITIES OF CENTRAL COLORADO, INC.",7198666423,CFO,228 NORTH CASCADE AVE,COLORADO SPRINGS,CO,840586169,J86JMKV4H2B6,No,80903,7196362321,CO,COLORADO SPRINGS,PARTNER,601 NORTH NEVADA AVENUE,80903,USA,MITCHELL DOWNS,MITCH@EBKCPA.COM,"ERICKSON, BROWN & KLOSTER, LLC",,840957308,,21,2024-01-19,2023-01-03,2023-01-04,2022-12-16,2023-01-03,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"14, 10, 16, 93, 97, 21",non-profit,,annual,4453358,UG,true,CENSUS,2023-01-03,MITCHELL DOWNS,PARTNER -25000,2022-06-CENSUS-0000232685,PLUMAS LAKE ELEMENTARY SCHOOL DISTRICT,DIRECTOR OF BUSINESS SERVICES,AJIT KANG,AKANG@PLUSD.ORG,PLUMAS LAKE ELEMENTARY SCHOOL DISTRICT,5307431408,DIRECTOR OF BUSINESS SERVICES,2743 PLUMAS SCHOOL ROAD,PLUMAS LAKE,CA,510636774,TRNJJ7CF5JH5,No,95961,9096898219,CA,RANCHO CUCAMONGA,PROPRIETOR,12223 HIGHLAND AVE. #106-625,91739,USA,ROBERT T DENNIS,RTDENNIS@DENNIS-CPA.COM,RT DENNIS ACCOUNTANCY,,464664250,,10,2024-01-19,2023-02-13,2023-02-14,2023-02-14,2023-02-13,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"84, 10, 93",local,,annual,1734870,UG,true,CENSUS,2023-02-13,ROBERT T DENNIS,PROPRIETOR -26000,2022-06-CENSUS-0000213785,RANDALLSTOWN NON-PROFIT HOUSING CORPORATION,CFO,CINDY LAMB,CINDY.LAMB@CSI.COOP,RANDALLSTOWN NON-PROFIT HOUSING CORPORATION,5867539002,CFO,8425 E. TWELVE MILE ROAD SUITE 100,WARREN,MI,522148647,H6N9FGNML4K5,No,48093,3178485700,IN,CARMEL,MEMBER,501 CONGRESSIONAL BLVD.,46032,USA,JACOB BUEHLER,JBUEHLER@DOZLLC.COM,"DAUBY O'CONNOR & ZALESKI, LLC",,351750664,,14,2024-01-19,2023-01-16,2023-01-17,2023-01-12,2023-01-16,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,6280641,UG,true,CENSUS,2023-01-16,JACOB BUEHLER,MEMBER -27000,2022-06-CENSUS-0000222455,FREEDOM HOUSING CORPORATION,EXECUTIVE DIRECTOR,CHRISTIAN BELDEN,CBELDEN@CCHCNEWPORT.ORG,FREEDOM HOUSING CORPORATION,4018465114,EXECUTIVE DIRECTOR,50 WASHINGTON SQUARE,NEWPORT,RI,134236650,HBTMMW93AE44,No,02840,4014742300,RI,WOONSOCKET,OWNER,531 HARRIS AVENUE,02895,USA,CRAIG D'AMBRA,CRAIGSDAMBRA@GMAIL.COM,D'AMBRA CPA,,050503395,,14,2024-01-19,2022-10-18,2022-10-19,2022-10-19,2022-10-18,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,1502431,UG,true,CENSUS,2022-10-18,CRAIG D'AMBRA,OWNER -28000,2022-06-CENSUS-0000232107,EASTERN AROOSTOOK REGIONAL SCHOOL UNIT #39,BUSINESS MANAGER,MARK BOUCHARD,MBOUCHARD@RSU39.ORG,EASTERN AROOSTOOK REGIONAL SCHOOL UNIT #39,2074966311,BUSINESS MANAGER,75 GLENN STREET,CARIBOU,ME,264505834,MAJDRQ5KQNE5,No,04736,2074983176,ME,CARIBOU,PARTNER,P.O. BOX 906,04736,USA,"GISELE MACDONALD, CPA",GMACDONALD@FELCHCPA.COM,FELCH & COMPANY LLC,,010540055,,84,2024-01-19,2023-01-05,2023-01-05,2022-12-30,2023-01-05,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 84",local,,annual,3377002,UG,true,CENSUS,2023-01-05,"GISELE MACDONALD, CPA",PARTNER -29000,2022-04-CENSUS-0000038992,"NARTHEX, INC.",HOUSING ACCOUNTING SUPPORT SUPERVISOR,ELIZABETH KING,NCRAUDITTEAM@NATIONALRESIDENCES.ORG,"NARTHEX, INC.",6142733577,HOUSING ACCOUNTING SUPPORT SUPERVISOR,2335 NORTH BANK DRIVE,COLUMBUS,OH,311013799,NYH3S7GZFEQ3,No,43220,6145281440,OH,COLUMBUS,PARTNER,"4249 EASTON WAY, SUITE 210",43219,USA,ANDREW GANTZER,ANDY.GANTZER@TIDWELLGROUP.COM,"TIDWELL GROUP, LLC",,271490692,,14,2024-01-19,2023-05-08,2023-04-24,2023-05-09,2023-05-08,2022-04-11,2021-04-11,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,other,2122349,UG,true,CENSUS,2023-05-08,ANDREW GANTZER,PARTNER -30000,2022-06-CENSUS-0000257720,STIEFEL THEATRE FOR THE PERFORMING ARTS FOUNDATION,EXECUTIVE DIRECTOR,JANE GATES,JGATES@STIEFELTHEATRE.ORG,STIEFEL THEATRE FOR THE PERFORMING ARTS FOUNDATION,7858271998,EXECUTIVE DIRECTOR,151 S. SANTA FE AVE,SALINA,KS,311537194,FTWNJFQLF463,No,67401,7858251561,KS,SALINA,CPA,3030 CORTLAND CIRCLE,67401,USA,CORLENE R. LANGE,LANGE@KCOE.COM,"KCOE ISOM, LLP",,480567703,,59,2024-01-19,2023-03-30,2023-03-31,2023-03-31,2023-03-30,2022-06-30,2021-06-30,2022,program-specific,unmodified_opinion,,,,No,No,No,No,Yes,750000,No,59,non-profit,,annual,1085901,UG,true,CENSUS,2023-03-30,CORLENE R. LANGE,CPA -31000,2022-06-CENSUS-0000248016,ACLED ANALYSIS INCORPORATED,SENIOR FINANCE MANAGER,AMANDA BERTRAN LLOVERA,A.BERTRAN.LLOVERA@ACLEDDATA.COM,ACLED ANALYSIS INCORPORATED,2623020379,SENIOR FINANCE MANAGER,361 FALLS RD #501,GRAFTON,WI,470972683,QDN9FURLZ8J3,No,53024,5104679506,CA,OAKLAND,PRESIDENT,1000 BROADWAY 200G,94607,USA,IRYNA ORESHKOVA,IO@IRYNACPA.COM,IRYNA ACCOUNTANCY CORPORATION,,204994635,,19,2024-01-19,2023-01-25,2023-01-26,2023-01-26,2023-01-25,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"98, 19",non-profit,,annual,777580,UG,true,CENSUS,2023-01-25,IRYNA ORESHKOVA,PRESIDENT -32000,2022-12-CENSUS-0000192624,STEEL VALLEY REGIONAL TRANSIT AUTHORITY,FINANCE MANAGER,TIM TURNER,TTURNER@SVRTA.COM,STEEL VALLEY REGIONAL TRANSIT AUTHORITY,7402826145,FINANCE MANAGER,555 ADAMS STREET,STEUBENVILLE,OH,341815642,L8SLLRZ77EP8,No,43952,7403730056,OH,MARIETTA,AUDIT MANAGER,313 SECOND STREET,45750,USA,MARIAH RICHARDS,MRICHARDS@PERRYCPAS.NET,PERRY AND ASSOCIATES CPAS AC,,550771624,,20,2024-01-19,2023-09-26,2023-09-27,2023-09-27,2023-09-26,2022-12-31,2021-12-31,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,20,local,,annual,824231,UG,true,CENSUS,2023-09-26,MARIAH RICHARDS,AUDIT MANAGER -33000,2022-12-CENSUS-0000250381,"BURROUGHS MANOR, INC.","PRESIDENT, GORMAN MANAGEMENT COMPANY",THOMAS A. GORMAN,TAGORMAN@AOL.COM,"BURROUGHS MANOR, INC.",9183332244,"PRESIDENT, GORMAN MANAGEMENT COMPANY",398060 WEST 2200 ROAD,BARTLESVILLE,OK,731124865,EULPKNCVLXM3,No,74006,9183360008,OK,BARTLESVILLE,ENGAGEMENT PARTNER,302 SE OSAGE AVENUE,74003,USA,JIM G. NICHOLS,JIM.NICHOLS@SAMBCPAS.COM,"STOTTS, ARCHAMBO, MUEGGENBORG & BARCLAY, PC",,471209122,,14,2024-01-19,2023-03-18,2023-03-17,2023-03-17,2023-03-18,2022-12-31,2021-12-31,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,1499783,UG,true,CENSUS,2023-03-18,JIM G. NICHOLS,ENGAGEMENT PARTNER -34000,2022-09-CENSUS-0000250713,THE SALVATION ARMY LAS VEGAS SOCIAL SERVICES,CONTROLLER,JEFFREY WHITE,JEFF.WHITE@USW.SALVATIONARMY.ORG,THE SALVATION ARMY LAS VEGAS SOCIAL SERVICES,6022674198,CONTROLLER,2900 PALOMINO LANE,LAS VEGAS,NV,941156347,WKZ9BWP1U5L3,No,89107,7149748143,CA,ORANGE,CPA,"2808 E. KATELLA AVENUE, SUITE 200",92867,USA,MARK FREEMAN,MJFREEMAN27@GMAIL.COM,FREEMAN & FREEMAN,,330112006,,64,2024-01-20,2023-06-22,2023-06-23,2023-06-23,2023-06-22,2022-09-30,2021-09-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"97, 14, 64, 16, 21",non-profit,,annual,8508867,UG,true,CENSUS,2023-06-22,MARK FREEMAN,CPA -35000,2022-06-CENSUS-0000232734,HUDSON COMMUNITY SCHOOL DISTRICT,SCHOOL BUSINESS OFFICIAL,CHAD WAGNER,CHADWAGNER@HUDSCHOOLS.ORG,HUDSON COMMUNITY SCHOOL DISTRICT,3199883233,SCHOOL BUSINESS OFFICIAL,136 S WASHINGTON ST,HUDSON,IA,426025403,F7EHCEHN3H64,No,50638,6417921910,IA,NEWTON,PRESIDENT,115 N 3RD AVE W,50208,USA,NANCY JANSSEN,INFO@NCJCPAIA.COM,"NOLTE, CORNMAN & JOHNSON",,421255013,,10,2024-01-20,2023-04-25,2023-04-25,2023-04-26,2023-04-25,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,Yes,Yes,No,No,750000,No,"84, 10",local,,annual,843708,UG,true,CENSUS,2023-04-25,NANCY JANSSEN,PRESIDENT -36000,2022-12-CENSUS-0000114049,WASHINGTON GORGE ACTION PROGRAMS,EXECUTIVE DIRECTOR,LESLIE NARAMORE,LESLIE@WAGAP.ORG,WASHINGTON GORGE ACTION PROGRAMS,5094932662,EXECUTIVE DIRECTOR,PO BOX 805,BINGEN,WA,910793062,G142DZKFX3B8,No,98605,5095751040,WA,YAKIMA,MEMBER OF THE FIRM,3702 KERN ROAD,98902,USA,ANGELA PRATT,ANGELA@PETERSENADVISORS.COM,"PETERSEN CPAS AND ADVISORS, PLLC",,261262413,,21,2024-01-19,2023-09-28,2023-09-29,2023-09-29,2023-09-28,2022-12-31,2021-12-31,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 16, 14, 93, 21",non-profit,,annual,4820177,UG,true,CENSUS,2023-09-28,ANGELA PRATT,MEMBER OF THE FIRM -37000,2022-06-CENSUS-0000089494,"REFUGE HOUSE, INC.",EXECUTIVE DIRECTOR,EMILY MITCHEM,EMITCHEM@REFUGEHOUSE.COM,"REFUGE HOUSE, INC.",8509226062,EXECUTIVE DIRECTOR,P.O. BOX 20910,TALLAHASSEE,FL,591869324,NKEWXG2SEUN8,No,32316,8503857444,FL,TALLAHASSEE,AUDIT PARTNER,3375 G CAPITAL CIRCLE NE,32308,USA,JOHN KIRK,JKIRK@TBL-CPA.COM,THOMSON BROCK LUGER & COMPANY,,202259573,,16,2024-01-20,2023-03-30,2023-03-31,2023-03-31,2023-03-30,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,No,"97, 16, 14, 93",non-profit,,annual,1330756,UG,true,CENSUS,2023-03-30,JOHN KIRK,AUDIT PARTNER -38000,2022-06-CENSUS-0000211066,CLEVELAND HEIGHTS-UNIVERSITY HEIGHTS CITY SCHOOL DISTRICT,TREASURER/CFO,SCOTT GAINER,S_GAINER@CHUH.ORG,CLEVELAND HEIGHTS-UNIVERSITY HEIGHTS CITY SCHOOL DISTRICT,2163717171,TREASURER/CFO,2155 MIRAMAR BOULEVARD,UNIVERSITY HEIGHTS,OH,346000687,NHLKJ8WKAZ36,No,44118,2167873665,OH,COLUMBUS,SENIOR AUDIT MANAGER,88 EAST BROAD STREET,43215,USA,MATTHEW GOLDMAN,MAGOLDMAN@OHIOAUDITOR.GOV,"KEITH FABER, AUDITOR OF STATE",,311334820,,84,2024-01-20,2023-01-16,2023-01-17,2023-01-17,2023-01-16,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 10, 32, 21",local,,annual,21449156,UG,true,CENSUS,2023-01-16,MATTHEW GOLDMAN,SENIOR AUDIT MANAGER -39000,2022-06-CENSUS-0000183906,VERMILLION HOUSING AND REDEVELOPMENT COMMISSION,EXECUTIVE DIRECTOR,DAVE THIESSE,VERMILLIONHOUSING@YAHOO.COM,VERMILLION HOUSING AND REDEVELOPMENT COMMISSION,6056777191,EXECUTIVE DIRECTOR,25 CENTER ST,VERMILLION,SD,460413120,MV5VTL3QLJY7,No,57069,6059967717,SD,MITCHELL,PARTNER,PO BOX 249,57301,USA,TERRI POST,TERRI.POST@ELOCPA.COM,ELO CPAS & ADVISORS,,460434947,,14,2024-01-20,2023-02-26,2023-02-27,2023-02-27,2023-02-26,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,No,14,local,,annual,951537,UG,true,CENSUS,2023-02-26,TERRI POST,PARTNER -40000,2022-12-CENSUS-0000231482,"FIRST RICHLAND MORROW HOUSING, INC. X",CONTROLLER,SHELLEY DARFUS,SDARFUS@BARCUSCOMPANY.COM,"FIRST RICHLAND MORROW HOUSING, INC. X",6144519000,CONTROLLER,1601 BETHEL ROAD,COLUMBUS,OH,341930439,LNJSEWR9BYF7,No,43220,6142240955,OH,COLUMBUS,CPA,"370 S FIFTH STREET, SUITE 1",43215,USA,TRACIE BAUGUS,TBAUGUS@SMS-CPAS.COM,"STEMEN, MERTENS, STICKLER CPAS & ASSOCIATES",,205511881,,14,2024-01-20,2023-04-18,2023-04-19,2023-04-19,2023-04-18,2022-12-31,2021-12-31,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,1435297,UG,true,CENSUS,2023-04-18,TRACIE BAUGUS,CPA -41000,2022-06-CENSUS-0000160804,WAVERLY CENTRAL SCHOOL DISTRICT,BUSINESS ADMINISTRATOR,KATHY ROTE,KROTE@GSTBOCES.ORG,WAVERLY CENTRAL SCHOOL DISTRICT,6075652841,BUSINESS ADMINISTRATOR,15 FREDRICK STREET,WAVERLY,NY,156002429,PD2KK5PY4JL3,No,14892,6072724444,NY,ITHACA,PARTNER,20 THORNWOOD DRIVE SUITE 200,14850,USA,LESLIE SPURGIN,DCF@INSEROCPA.COM,"INSERO & CO. CPA'S, LLP",,475324570,,84,2024-01-20,2022-11-07,2022-11-08,2022-11-08,2022-11-07,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,Yes,No,750000,Yes,"10, 84",local,,annual,3948327,UG,true,CENSUS,2022-11-07,LESLIE SPURGIN,PARTNER -42000,2022-06-CENSUS-0000250238,"DRUG RECOVERY, INC. (DBA CATALYST BEHAVIORAL SERVICES, INC)",EXECUTIVE DIRECTOR,EDIE NAYFA,ENAYFA@CATALYSTOK.ORG,"DRUG RECOVERY, INC. (DBA CATALYST BEHAVIORAL SERVICES, INC)",4052329804,EXECUTIVE DIRECTOR,"3033 N. WALNUT, WEST BUILDING",OKLAHOMA CITY,OK,730968383,CDCJZ69PCEL8,No,73105,4055942041,OK,OKLAHOMA CITY,VDUTTON@EIDEBAILLY.COM,"621 N. ROBINSON, STE 200",731026232,USA,VANESSA DUTTON,VDUTTON@EIDEBAILLY.COM,EIDE BAILLY LLP,,450250958,,93,2024-01-20,2023-03-30,2023-03-31,2023-03-31,2023-03-30,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,No,"16, 93",non-profit,,annual,2844302,UG,true,CENSUS,2023-03-30,VANESSA DUTTON,VDUTTON@EIDEBAILLY.COM -43000,2022-06-CENSUS-0000160312,DUTCHESS COUNTY BOCES,SCHOOL BUSINESS ADMINISTRATOR,MATTHEW METZGER,MATTHEW.METZGER@DCBOCES.ORG,DUTCHESS COUNTY BOCES,8454864800,SCHOOL BUSINESS ADMINISTRATOR,5 BOCES ROAD,POUGHKEEPSIE,NY,146012196,GNQGKNTDVDS6,No,12601,5188284616,NY,HUDSON,DIRECTOR,"4071 ROUTE 9, STOP 1",12534,USA,"VICTOR V. CHURCHILL, CPA",VCHURCHILL@RBTCPAS.COM,"RBT CPA'S, LLP",,141604297,,84,2024-01-20,2023-03-30,2023-03-31,2023-03-31,2023-03-30,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"10, 84",local,,annual,2951989,UG,true,CENSUS,2023-03-30,"VICTOR V. CHURCHILL, CPA",DIRECTOR -44000,2022-06-CENSUS-0000235281,CATALYST SCHOOLS,DIRECTOR OF FINANCE AND COMPLIANCE,MICHELE NUNEZ,MNUNEZ@CATALYSTSCHOOLS.ORG,CATALYST SCHOOLS,7738975825,DIRECTOR OF FINANCE AND COMPLIANCE,6727 S CALIFORNIA AVE,CHICAGO,IL,204069346,XB1KKBRLHQJ9,No,60629,3126344414,IL,CHICAGO,SENIOR DIRECTOR,30 SOUTH WACKER DR,60606,USA,MANDY PITTMAN,MANDY.PITTMAN@RSMUS.COM,RSM US LLP,,420714325,,84,2024-01-20,2022-11-03,2022-11-04,2022-11-04,2022-11-03,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"84, 10",non-profit,,annual,4121154,UG,true,CENSUS,2022-11-03,MANDY PITTMAN,SENIOR DIRECTOR -45000,2022-06-CENSUS-0000164186,CLEVELAND MUNICIPAL SCHOOL DISTRICT,INTERIM CHIEF FINANCIAL OFFICER,NATHAN MORTIMER,NATHAN.MORTIMER@CLEVELANDMETROSCHOOLS.ORG,CLEVELAND MUNICIPAL SCHOOL DISTRICT,2168380391,INTERIM CHIEF FINANCIAL OFFICER,"1111 SUPERIOR AVENUE, SUITE 1800",CLEVELAND,OH,346000662,ESYAYXNJYTD9,No,44114,2167873665,OH,COLUMBUS,SENIOR AUDIT MANAGER,88 EAST BROAD STREET,43216,USA,DOUGLAS CHAMOT,DWCHAMOT@OHIOAUDITOR.GOV,"KEITH FABER, AUDITOR OF STATE",,311334820,84,,2024-01-20,2023-01-24,2023-01-24,2023-01-25,2023-01-24,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,Yes,No,No,3000000,No,"84, 12, 10, 93",local,,annual,200303287,UG,true,CENSUS,2023-01-24,DOUGLAS CHAMOT,SENIOR AUDIT MANAGER -46000,2022-06-CENSUS-0000229656,"MOHN STREET ACCESSIBLE HOUSING, INC.",PRESIDENT,LEAH EPPINGER,LEPPINGER@DAUPHINHOUSING.ORG,"MOHN STREET ACCESSIBLE HOUSING, INC.",7179399301,PRESIDENT,"501 MOHN STREET ACCESSIBLE HOUSING, INC.",STEELTON,PA,870762383,XP1NMSKDMM81,No,17113,7175619200,PA,HARRISBURG,PARTNER,"830 SIR THOMAS COURT, SUITE 100",17109,USA,KIMBERLY STANK,KSTANK@ZALLC.ORG,ZELENKOFSKE AXELROD LLC,,233022325,,14,2024-01-20,2023-02-21,2023-02-13,2023-02-17,2023-02-21,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,14,non-profit,,annual,1598234,UG,true,CENSUS,2023-02-21,KIMBERLY STANK,PARTNER -47000,2022-06-CENSUS-0000206378,"TURNING POINT, INC.",EXECUTIVE DIRECTOR,SARAH PONITZ,SPONITZ@TURNPT.ORG,"TURNING POINT, INC.",8153388081,EXECUTIVE DIRECTOR,PO BOX 273,WOODSTOCK,IL,363163296,JL4FNMZDKSK8,No,60098,8153441300,IL,MCHENRY,PARTNER,"5400 W ELM ST, SUITE 203",60050,USA,CHERYDEN JUERGENSEN,CPAS@EDERCASELLA.COM,"EDER, CASELLA & CO",,363614997,,93,2024-01-20,2022-11-09,2022-11-10,2022-11-09,2022-11-09,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"14, 16, 93, 97",non-profit,,annual,946382,UG,true,CENSUS,2022-11-09,CHERYDEN JUERGENSEN,PARTNER -48000,2022-12-CENSUS-0000247951,EXTENSION FOUNDATION,CONTROLLER,ARIELLE SMITH,ARIELLESMITH@EXTENSION.ORG,EXTENSION FOUNDATION,8327365157,CONTROLLER,ONE KANSAS CITY PL 1200 MAIN S,KANSAS CITY,MO,204781422,DNZBHAA5N4G3,No,641052122,4024799321,NE,LINCOLN,CPA/PARTNER,1248 O STREET SUITE 500,68508,USA,KERRY GUSTAFSSON,GUSTAFSSON@DANACOLE.COM,DANA F. COLE & COMPANY LLP,,470526649,,10,2024-01-20,2023-09-28,2023-09-29,2023-09-29,2023-09-28,2022-12-31,2021-12-31,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 66",non-profit,,annual,5096909,UG,true,CENSUS,2023-09-28,KERRY GUSTAFSSON,CPA/PARTNER -49000,2022-06-CENSUS-0000150455,SHELBY PUBLIC SCHOOLS,BUSINESS MANAGER,AMY BUDDE,ABUDDE@SHELBY.K12.MI.US,SHELBY PUBLIC SCHOOLS,2318615211,BUSINESS MANAGER,525 N. STATE STREET,SHELBY,MI,386003167,T3DJNDK73JY8,No,49455,6169493200,MI,GRAND RAPIDS,SENIOR MANAGER,2910 LUCERNE DRIVE SE,49546,USA,MARC SAWYERS,MSAWYERS@HUNGERFORDNICHOLS.COM,HUNGERFORD NICHOLS,,382184825,,10,2024-01-20,2023-01-13,2023-01-14,2022-11-30,2023-01-13,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 84, 93",local,,annual,4702717,UG,true,CENSUS,2023-01-13,MARC SAWYERS,SENIOR MANAGER -50000,2022-09-CENSUS-0000102763,"EDINBURG CHILD CARE, INC",EXECUTIVE DIRECTOR,GYLMA GARZA,GMG.ECCI@GMAIL.COM,"EDINBURG CHILD CARE, INC",9563836789,EXECUTIVE DIRECTOR,1338 W UNIVERSITY,EDINBURG,TX,742358732,M3PGGA5FRAF6,No,785392930,9567879909,TX,PHARR,AUDITOR,208 W FERGUSON UNIT 1,78577,USA,JANET ROBLES,JANETROBLES@ORGCPA.COM,OSCAR R GONZALEZ CPA AND ASSOCIATES,,273582518,,10,2024-01-20,2023-06-28,2023-06-29,2023-06-29,2023-06-28,2022-09-30,2021-09-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,10,non-profit,,annual,4343763,UG,true,CENSUS,2023-06-28,JANET ROBLES,AUDITOR -51000,2022-06-CENSUS-0000166110,CITY OF ROSEBURG,FINANCE DIRECTOR,RON HARKER,RHARKER@CITYOFROSEBURG.ORG,CITY OF ROSEBURG,5414926710,FINANCE DIRECTOR,900 SE DOUGLAS AVE,ROSEBURG,OR,936002247,ZFPTXNQ1CVH8,No,97470,5416724886,OR,ROSEBURG,CERTIFIED PUBLIC ACCOUNTANT,2500 W HARVARD AVE,97471,USA,JEFFREY R. COOLEY,JEFF@NEUNERDAVIDSON.COM,NEUNER DAVIDSON & CO,,930764156,,14,2024-01-20,2023-01-16,2023-01-17,2023-01-17,2023-01-16,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,No,"14, 21, 16, 45, 20",state,,annual,1236483,UG,true,CENSUS,2023-01-16,JEFFREY R. COOLEY,CERTIFIED PUBLIC ACCOUNTANT -52000,2022-06-CENSUS-0000205148,SCHOOL DISTRICT OF OSCEOLA,ADMINISTRATOR,MARK LUEBKER,LUEBKERM@OSCEOLAK12.ORG,SCHOOL DISTRICT OF OSCEOLA,7152944140,ADMINISTRATOR,PO BOX 128,OSCEOLA,WI,396003796,HB1AL86ZYTT3,No,54020,7158521100,WI,EAU CLAIRE,PRINCIPAL,"3402 OAKWOOD MALL DRIVE, SUITE 100",54701,USA,APRIL ANDERSON,APRIL.ANDERSON@CLACONNECT.COM,CLIFTONLARSONALLEN LLP,,410746749,,10,2024-01-20,2023-03-29,2023-03-30,2023-03-30,2023-03-29,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,Yes,No,No,No,750000,No,"84, 93, 32, 10",unknown,,annual,2877490,UG,true,CENSUS,2023-03-29,APRIL ANDERSON,PRINCIPAL -53000,2022-03-CENSUS-0000230532,"PAGEDALE RHF HOUSING, INC. DBA MOUNT BEULAH TERRACE 085-EE-090",CONTROLLER,CHRISTOPHER PURCELL,CHRISTOPHER.PURCELL@RHF.ORG,"PAGEDALE RHF HOUSING, INC. DBA MOUNT BEULAH TERRACE 085-EE-090",5622575100,CONTROLLER,911 NORTH STUDEBAKER ROAD,LONG BEACH,CA,205267298,E67LXYHYDCG3,No,90815,3178485700,IN,CARMEL,PRINCIPAL,501 CONGRESSIONAL BLVD.,46032,USA,MATT CATLIN,MCATLIN@DOZLLC.COM,"DAUBY O'CONNOR & ZALESKI, LLC",,351750664,,14,2024-01-20,2022-12-19,2022-12-20,2022-12-20,2022-12-19,2022-03-31,2021-03-31,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"10, 14",non-profit,,annual,5248382,UG,true,CENSUS,2022-12-19,MATT CATLIN,PRINCIPAL -54000,2022-12-CENSUS-0000159275,TOWN OF TONAWANDA,COMPTROLLER,DEBORAH SCHWARZ,DSCHWARZ@TONAWANDA.NY.US,TOWN OF TONAWANDA,7168778810,COMPTROLLER,2919 DELAWARE AVE,KENMORE,NY,166002385,W9YVK5RR6SN7,No,14217,7165652299,NY,ELMA,PARTNER,2721 TRANSIT RD SUITE 111,14059,USA,MATTHEW MONTALBO,MMONTALBO@DRESCHERMALECKI.COM,DRESCHER & MALECKI LLP,,743118183,,14,2024-01-20,2023-07-24,2023-07-25,2023-07-24,2023-07-24,2022-12-31,2021-12-31,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"21, 14, 20, 16",local,,annual,6646790,UG,true,CENSUS,2023-07-24,MATTHEW MONTALBO,PARTNER -55000,2022-06-CENSUS-0000181993,"CHILD AND FAMILY RESOURCES, INC. AND SUBSIDIARY",CHIEF FINACIAL OFFICER,CONNIE CURNETT,CCURNETT@CFRAZ.ORG,"CHILD AND FAMILY RESOURCES, INC. AND SUBSIDIARY",5208818940,CHIEF FINACIAL OFFICER,2800 E. BROADWAY,TUCSON,AZ,860251984,YBNKJ1BNH4Z9,No,85716,5206248229,AZ,TUCSON,PARTNER,"4801 E. BROADWAY BLVD, STE 501",857113648,USA,SUSAN VOS,SVOS@RCM.CPA,"REGIER, CARR, & MONROE, LLP CPAS",,480573184,,93,2024-01-20,2022-11-22,2022-11-14,2022-11-23,2022-11-22,2022-06-30,2021-06-30,2022,single-audit,unmodified_opinion,,,,No,No,No,No,No,750000,Yes,"93, 10",non-profit,,annual,4649326,UG,true,CENSUS,2022-11-22,SUSAN VOS,PARTNER -187000,2018-06-CENSUS-0000171150,BON HOMME SCHOOL DISTRICT NO. 4-2,BUSINESS MANAGER,GARY KORTAN,GARY.KORTAN@K12.SD.US,BON HOMME SCHOOL DISTRICT NO. 4-2,6055893388,BUSINESS MANAGER,PO BOX 28,TYNDALL,SD,460311996,GSA_MIGRATION,No,570660028,6059287241,SD,PARKSTON,CPA,"PO BOX 247, 105 E MAIN ST",573660247,USA,RANDY SCHOENFISH,WJSCPA@SANTEL.NET,"SCHOENFISH & CO, INC",,460353724,,84,2024-01-26,2019-03-26,2019-03-26,2019-03-27,2019-03-26,2018-06-30,2017-06-30,2018,single-audit,qualified_opinion,,,,No,Yes,No,Yes,No,750000,No,"10, 84",local,,annual,807077,UG,true,CENSUS,2019-03-26,RANDY SCHOENFISH,CPA diff --git a/backend/dissemination/sql/api/api_v1_1_0/sling.md b/backend/dissemination/sql/api/api_v1_1_0/sling.md deleted file mode 100644 index 9974a5d676..0000000000 --- a/backend/dissemination/sql/api/api_v1_1_0/sling.md +++ /dev/null @@ -1,41 +0,0 @@ -# sling - -Point to the DB - -``` -export PG='postgresql://postgres@localhost:5432/postgres?sslmode=disable' -``` - -This gives me the general table as a CSV to STDOUT: - -``` -sling run --src-conn $PG --src-stream 'public.dissemination_general' --stdout -``` - -With a configuration file (YAML): - -`config1.yaml`: - -``` -source: postgresql://postgres@localhost:5432/postgres?sslmode=disable -target: LOCAL - -defaults: - target_options: - format: csv - -streams: - public.dissemination_general: - object: file:///tmp/dg.csv - - public.dissemination_federalaward: - object: file:///tmp/dfa/*.csv - target_options: - file_max_rows: 100000 -``` - -and then - -``` -sling run -r config1.yaml -``` diff --git a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/base.sql b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/base.sql deleted file mode 100644 index c6f8853bfa..0000000000 --- a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/base.sql +++ /dev/null @@ -1,36 +0,0 @@ -DO -$do$ -BEGIN - IF EXISTS ( - SELECT FROM pg_catalog.pg_roles - WHERE rolname = 'authenticator') THEN - RAISE NOTICE 'Role "authenticator" already exists. Skipping.'; - ELSE - CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; - END IF; -END -$do$; - -DO -$do$ -BEGIN - IF EXISTS ( - SELECT FROM pg_catalog.pg_roles - WHERE rolname = 'api_fac_gov') THEN - RAISE NOTICE 'Role "api_fac_gov" already exists. Skipping.'; - ELSE - CREATE ROLE api_fac_gov NOLOGIN; - END IF; -END -$do$; - -GRANT api_fac_gov TO authenticator; - -CREATE SEQUENCE IF NOT EXISTS public_data_general - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE; - - -NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/create_functions.sql b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/create_functions.sql deleted file mode 100644 index 5c4db5d713..0000000000 --- a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/create_functions.sql +++ /dev/null @@ -1,4 +0,0 @@ --- Under the new approach, we don't need --- any functions here. - -NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/create_schema.sql b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/create_schema.sql deleted file mode 100644 index 0c0d6d58bf..0000000000 --- a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/create_schema.sql +++ /dev/null @@ -1,42 +0,0 @@ -begin; - -do -$$ -BEGIN - DROP SCHEMA IF EXISTS public_api_v2_0_0_alpha CASCADE; - DROP SCHEMA IF EXISTS public_api_v2_0_0_alpha_functions CASCADE; - - CREATE SCHEMA IF NOT EXISTS public_api_v2_0_0_alpha; - CREATE SCHEMA IF NOT EXISTS public_api_v2_0_0_alpha_functions; - - GRANT USAGE ON SCHEMA public_api_v2_0_0_alpha_functions TO api_fac_gov; - - -- Grant access to tables and views - ALTER DEFAULT PRIVILEGES - IN SCHEMA public_api_v2_0_0_alpha - GRANT SELECT - -- this includes views - ON tables - TO api_fac_gov; - - -- Grant access to sequences, if we have them - GRANT USAGE ON SCHEMA public_api_v2_0_0_alpha to api_fac_gov; - GRANT SELECT, USAGE - ON ALL SEQUENCES - IN SCHEMA public_api_v2_0_0_alpha - TO api_fac_gov; - - -- ALTER DEFAULT PRIVILEGES - -- IN SCHEMA public_api_v2_0_0_alpha - -- GRANT SELECT, USAGE - -- ON sequences - -- TO api_fac_gov; -END -$$ -; - -COMMIT; - -notify pgrst, - 'reload schema'; - diff --git a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/create_views.sql b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/create_views.sql deleted file mode 100644 index c98acde746..0000000000 --- a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/create_views.sql +++ /dev/null @@ -1,182 +0,0 @@ -BEGIN; - -CREATE VIEW public_api_v2_0_0_alpha.additional_eins AS - SELECT * FROM public_data.additional_eins -; - ---------------------------------------- --- additional_ueis ---------------------------------------- -create view public_api_v2_0_0_alpha.additional_ueis AS - SELECT * FROM additional_ueis -; - ---------------------------------------- --- corrective_action_plan ---------------------------------------- -CREATE VIEW public_api_v2_0_0_alpha.corrective_action_plans AS - SELECT * FROM public_data.corrective_action_plans - ORDER BY public_data.corrective_action_plans.id -; - ---------------------------------------- --- finding ---------------------------------------- -CREATE VIEW public_api_v2_0_0_alpha.findings AS - SELECT * FROM public_data.findings -; - ---------------------------------------- --- finding_text ---------------------------------------- -CREATE VIEW public_api_v2_0_0_alpha.findings_text AS - SELECT * FROM public_data.findings_text ft - ORDER BY ft.id -; - ---------------------------------------- --- federal award ---------------------------------------- -CREATE VIEW public_api_v2_0_0_alpha.federal_awards AS - SELECT * FROM public_data.federal_awards -; - ---------------------------------------- --- general ---------------------------------------- -CREATE VIEW public_api_v2_0_0_alpha.general AS - SELECT * FROM public_data.general -; - ---------------------------------------- --- notes_to_sefa ---------------------------------------- -create view public_api_v2_0_0_alpha.notes_to_sefa AS - SELECT * FROM public_data.notes_to_sefa nts - ORDER BY nts.id -; - ---------------------------------------- --- passthrough ---------------------------------------- -CREATE VIEW public_api_v2_0_0_alpha.passthrough AS - SELECT * FROM public_data.passthrough -; - - ---------------------------------------- --- auditor (secondary auditor) ---------------------------------------- -CREATE VIEW public_api_v2_0_0_alpha.secondary_auditors AS - SELECT * FROM public_data.secondary_auditors - ; - --- Specify every field in dissemination_combined, omitting the id. --- Generated fields like ALN are done in the creation of the table, not here. -create view public_api_v2_0_0_alpha.combined as - select - combined.report_id, - combined.award_reference, - combined.reference_number, - combined.aln, - combined.agencies_with_prior_findings, - combined.audit_period_covered, - combined.audit_type, - combined.audit_year, - combined.auditee_address_line_1, - combined.auditee_certified_date, - combined.auditee_certify_name, - combined.auditee_certify_title, - combined.auditee_city, - combined.auditee_contact_name, - combined.auditee_contact_title, - combined.auditee_ein, - combined.auditee_email, - combined.auditee_name, - combined.auditee_phone, - combined.auditee_state, - combined.auditee_uei, - combined.auditee_zip, - combined.auditor_address_line_1, - combined.auditor_certified_date, - combined.auditor_certify_name, - combined.auditor_certify_title, - combined.auditor_city, - combined.auditor_contact_name, - combined.auditor_contact_title, - combined.auditor_country, - combined.auditor_ein, - combined.auditor_email, - combined.auditor_firm_name, - combined.auditor_foreign_address, - combined.auditor_phone, - combined.auditor_state, - combined.auditor_zip, - combined.cognizant_agency, - combined.data_source, - combined.date_created, - combined.dollar_threshold, - combined.entity_type, - combined.fac_accepted_date, - combined.fy_end_date, - combined.fy_start_date, - combined.gaap_results, - combined.is_additional_ueis, - combined.is_aicpa_audit_guide_included, - combined.is_going_concern_included, - combined.is_internal_control_deficiency_disclosed, - combined.is_internal_control_material_weakness_disclosed, - combined.is_low_risk_auditee, - combined.is_material_noncompliance_disclosed, - combined.is_public, - combined.is_sp_framework_required, - combined.number_months, - combined.oversight_agency, - combined.ready_for_certification_date, - combined.sp_framework_basis, - combined.sp_framework_opinions, - combined.submitted_date, - combined.total_amount_expended, - combined.type_audit_code, - combined.additional_award_identification, - combined.amount_expended, - combined.cluster_name, - combined.cluster_total, - combined.federal_agency_prefix, - combined.federal_award_extension, - combined.federal_program_name, - combined.federal_program_total, - combined.findings_count, - combined.is_direct, - combined.is_loan, - combined.is_major, - combined.is_passthrough_award, - combined.loan_balance, - combined.audit_report_type, - combined.other_cluster_name, - combined.passthrough_amount, - combined.state_cluster_name, - combined.is_material_weakness, - combined.is_modified_opinion, - combined.is_other_findings, - combined.is_other_matters, - combined.is_questioned_costs, - combined.is_repeat_finding, - combined.is_significant_deficiency, - combined.prior_finding_ref_numbers, - combined.type_requirement, - combined.passthrough_name, - combined.passthrough_id - from - dissemination_combined combined - where - (combined.is_public = true - or (combined.is_public = false and public_api_v2_0_0_alpha_functions.has_tribal_data_access())) - order by combined.id -; - -COMMIT; - -notify pgrst, - 'reload schema'; - diff --git a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/drop_schema.sql b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/drop_schema.sql deleted file mode 100644 index cb24063256..0000000000 --- a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/drop_schema.sql +++ /dev/null @@ -1,12 +0,0 @@ - -begin; - -DROP SCHEMA IF EXISTS public_api_v2_0_0_alpha CASCADE; - --- DROP ROLE IF EXISTS authenticator; --- DROP ROLE IF EXISTS api_fac_gov; - -commit; - -notify pgrst, - 'reload schema'; diff --git a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/drop_views.sql b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/drop_views.sql deleted file mode 100644 index c95c0fc139..0000000000 --- a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/drop_views.sql +++ /dev/null @@ -1,18 +0,0 @@ -begin; - drop table if exists api_v1_1_1.metadata; - drop view if exists api_v1_1_1.general; - drop view if exists api_v1_1_1.auditor; - drop view if exists api_v1_1_1.federal_awards; - drop view if exists api_v1_1_1.findings; - drop view if exists api_v1_1_1.findings_text; - drop view if exists api_v1_1_1.corrective_action_plans; - drop view if exists api_v1_1_1.additional_ueis; - drop view if exists api_v1_1_1.notes_to_sefa; - drop view if exists api_v1_1_1.passthrough; - drop view if exists api_v1_1_1.secondary_auditors; - drop view if exists api_v1_1_1.additional_eins; - drop view if exists api_v1_1_1.combined; -commit; - -notify pgrst, - 'reload schema'; diff --git a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/setup.sql b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/setup.sql deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml b/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml index d5db7a613d..ab9af6f700 100644 --- a/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml +++ b/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml @@ -1,7 +1,7 @@ # This uses an env var called PG to # set the connection string. source: FAC_DB_URI -target: FAC_DB_URI +target: FAC_SNAPSHOT_URI env: SLING_ALLOW_EMPTY: true diff --git a/backend/dissemination/sql/sling/public_data_v1_0_0/standup.sql b/backend/dissemination/sql/sling/public_data_v1_0_0/standup.sql index e69de29bb2..f1280eab4e 100644 --- a/backend/dissemination/sql/sling/public_data_v1_0_0/standup.sql +++ b/backend/dissemination/sql/sling/public_data_v1_0_0/standup.sql @@ -0,0 +1,6 @@ +-- WE DO NEED THE TABLES, THOUGH. +-- In theory, `sling` will be creating these tables. But, if they don't exist, +-- then the views will crash. So, lets go ahead and create the tables if they don't exist. +-- This means we have to stay in sync with the sling YAML. + +CREATE TABLE IF NOT EXISTS public_data_v1_0_0 diff --git a/backend/tools/setup_cgov_env.sh b/backend/tools/setup_cgov_env.sh index 553a3c636f..007e4aa0aa 100644 --- a/backend/tools/setup_cgov_env.sh +++ b/backend/tools/setup_cgov_env.sh @@ -45,7 +45,7 @@ function setup_cgov_env { export FAC_DB_URI="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "fac-db" ".[][] | select(.name == \$service_name) | .credentials.uri")" export FAC_SNAPSHOT_URI="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "fac-snapshot-db" ".[][] | select(.name == \$service_name) | .credentials.uri")" # https://stackoverflow.com/questions/37072245/check-return-status-of-psql-command-in-unix-shell-scripting - export PSQL_EXE='/home/vcap/deps/0/apt/usr/lib/postgresql/*/bin/psql' + export PSQL_EXE='/home/vcap/deps/0/apt/usr/lib/postgresql/*/bin/psql -v ON_ERROR_STOP=on' return 0 } From b2263e779de8a87085fcb23666be4da059dc2da5 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Sun, 29 Sep 2024 16:49:45 -0400 Subject: [PATCH 03/89] In-progress. --- .../all_audits_that_failed_migrations.csv | 3329 ----------------- .../tools/load_public_dissem_data/Dockerfile | 27 + .../tools/load_public_dissem_data/README.md | 50 + .../load_public_dissem_data/data/README.md | 3 + .../load_public_data_locally.sh | 126 + tools/load_public_dissem_data/.gitignore | 5 + .../load_public_data_locally.sh | 109 + 7 files changed, 320 insertions(+), 3329 deletions(-) delete mode 100644 backend/tools/historic_migrator/all_audits_that_failed_migrations.csv create mode 100644 backend/tools/load_public_dissem_data/Dockerfile create mode 100644 backend/tools/load_public_dissem_data/README.md create mode 100644 backend/tools/load_public_dissem_data/data/README.md create mode 100755 backend/tools/load_public_dissem_data/load_public_data_locally.sh create mode 100644 tools/load_public_dissem_data/.gitignore create mode 100755 tools/load_public_dissem_data/load_public_data_locally.sh diff --git a/backend/tools/historic_migrator/all_audits_that_failed_migrations.csv b/backend/tools/historic_migrator/all_audits_that_failed_migrations.csv deleted file mode 100644 index b09f16662a..0000000000 --- a/backend/tools/historic_migrator/all_audits_that_failed_migrations.csv +++ /dev/null @@ -1,3329 +0,0 @@ -dbkey,audit_year -245186,2016 -181884,2016 -238896,2016 -187529,2016 -237862,2016 -184042,2016 -74828,2016 -245454,2016 -188100,2016 -184711,2016 -187796,2016 -229655,2016 -141780,2016 -110841,2016 -112120,2016 -243942,2016 -247316,2016 -9156,2016 -188358,2016 -246222,2016 -82235,2016 -73891,2016 -202062,2016 -64586,2016 -241362,2016 -239399,2016 -204681,2016 -183865,2016 -202639,2016 -166571,2016 -138640,2016 -43776,2016 -165496,2016 -49070,2016 -170759,2016 -219517,2016 -112265,2016 -104928,2016 -183596,2016 -233733,2016 -245635,2016 -245991,2016 -183478,2016 -243590,2016 -211245,2016 -196078,2016 -241815,2016 -227102,2016 -154684,2016 -181446,2016 -141521,2016 -170497,2016 -169599,2016 -221915,2016 -132198,2016 -82235,2017 -180129,2017 -183392,2017 -247316,2017 -221360,2017 -95902,2017 -131921,2017 -147959,2017 -242069,2017 -247312,2017 -248094,2017 -89892,2017 -99963,2017 -80841,2017 -188343,2017 -82235,2018 -131409,2017 -125677,2017 -219241,2017 -243141,2017 -81941,2017 -136308,2017 -234451,2017 -247217,2017 -247311,2017 -161183,2017 -196218,2017 -150045,2017 -208410,2017 -148858,2017 -177871,2017 -196850,2017 -226934,2017 -152705,2017 -132024,2017 -154119,2017 -198094,2017 -164895,2017 -230305,2017 -223363,2018 -247318,2018 -181817,2018 -180129,2018 -243141,2018 -227561,2018 -106920,2018 -193734,2018 -183834,2018 -224521,2018 -236580,2018 -228625,2018 -223994,2018 -238803,2018 -221747,2018 -225866,2018 -230095,2018 -137693,2018 -193746,2018 -208149,2018 -200258,2018 -125496,2018 -191699,2018 -188378,2018 -216179,2018 -216448,2018 -218911,2018 -249402,2018 -248094,2018 -201593,2018 -243205,2018 -247016,2018 -189688,2018 -246031,2018 -213520,2018 -238398,2018 -212927,2018 -225532,2018 -147058,2018 -190995,2018 -176780,2018 -170743,2018 -197329,2018 -240474,2019 -109739,2019 -236396,2019 -244532,2019 -148983,2019 -137649,2019 -192847,2019 -148189,2019 -169846,2019 -110826,2019 -249817,2019 -212124,2019 -82235,2019 -179407,2019 -247990,2019 -214832,2019 -156391,2019 -240680,2019 -258913,2019 -232146,2019 -185474,2019 -141198,2019 -94819,2019 -216437,2019 -207030,2019 -234392,2019 -213771,2019 -4412,2019 -193917,2019 -177168,2020 -225104,2020 -237101,2020 -232622,2020 -186672,2020 -82235,2020 -205254,2020 -94819,2020 -250998,2020 -236344,2020 -187642,2020 -221641,2020 -51747,2020 -190034,2020 -227711,2020 -241491,2020 -144498,2020 -249752,2020 -241192,2020 -214832,2020 -240474,2020 -227822,2020 -251218,2020 -183763,2020 -228694,2020 -251079,2020 -224100,2020 -250906,2020 -56449,2020 -250210,2020 -196299,2020 -34075,2020 -83733,2020 -181006,2020 -214506,2020 -246935,2020 -242070,2020 -147059,2020 -205842,2021 -170083,2021 -11980,2021 -115254,2021 -246683,2021 -228629,2021 -214763,2021 -212495,2021 -213143,2021 -218226,2021 -129947,2021 -254402,2021 -187782,2021 -208597,2021 -82235,2021 -56000,2021 -64989,2021 -111001,2021 -182850,2021 -254484,2021 -249752,2021 -240080,2021 -249033,2021 -210061,2021 -241733,2021 -240680,2021 -201811,2021 -238619,2021 -226682,2021 -248260,2021 -187428,2021 -236079,2021 -144393,2021 -213806,2021 -207104,2021 -206963,2021 -248943,2021 -206188,2021 -14104,2021 -253671,2021 -249104,2021 -253519,2021 -254745,2021 -255251,2021 -192984,2021 -246540,2021 -253889,2021 -254695,2021 -252762,2021 -243601,2021 -227361,2021 -254747,2021 -240075,2021 -229304,2021 -253719,2021 -232364,2021 -155547,2021 -38009,2021 -211967,2021 -36043,2022 -197787,2022 -185619,2022 -91930,2022 -252746,2022 -98016,2022 -223085,2022 -247273,2022 -236344,2022 -150090,2022 -205092,2022 -250965,2022 -184872,2022 -257726,2022 -39718,2022 -82235,2022 -222924,2022 -94819,2022 -231532,2022 -236079,2022 -198496,2022 -158343,2022 -257749,2022 -195719,2022 -256451,2022 -103261,2022 -238967,2022 -257210,2022 -256821,2022 -219582,2016 -111783,2016 -244792,2016 -229938,2016 -231297,2016 -64400,2016 -21005,2016 -198263,2016 -246358,2016 -204828,2016 -150961,2016 -247285,2017 -190034,2017 -219294,2017 -225051,2017 -177934,2017 -171681,2017 -173830,2017 -235561,2017 -50668,2018 -198658,2018 -248049,2018 -248635,2018 -106968,2018 -248270,2018 -217703,2018 -131972,2018 -98042,2019 -88611,2019 -217403,2019 -234387,2019 -189782,2020 -191094,2020 -191946,2020 -190848,2020 -191610,2020 -190163,2020 -183651,2020 -241193,2020 -235850,2020 -170380,2020 -250450,2020 -247318,2020 -189790,2020 -191186,2020 -201820,2020 -191185,2020 -225053,2020 -601,2020 -191010,2020 -255580,2020 -192315,2020 -191949,2020 -188733,2020 -94530,2021 -256745,2021 -247154,2021 -212124,2021 -225036,2021 -255075,2021 -251420,2021 -205821,2021 -193349,2022 -30647,2022 -242128,2022 -211875,2022 -55683,2022 -255075,2022 -193089,2016 -55815,2016 -197885,2017 -65462,2017 -242643,2017 -143139,2017 -251060,2017 -237083,2018 -95902,2018 -186072,2018 -235247,2018 -141202,2019 -104235,2019 -54936,2020 -247863,2020 -190364,2020 -128661,2020 -130520,2020 -117467,2020 -240432,2021 -128661,2021 -159008,2021 -191201,2021 -212267,2021 -240801,2021 -253527,2021 -256874,2021 -253836,2021 -156005,2022 -182241,2022 -148220,2016 -181306,2016 -190267,2016 -1447,2016 -186831,2016 -141352,2016 -233246,2016 -202361,2016 -187405,2016 -97820,2016 -247811,2016 -177312,2017 -2976,2017 -180895,2017 -170265,2017 -111045,2017 -119705,2018 -182848,2018 -180876,2018 -172606,2018 -78466,2018 -251060,2018 -170375,2018 -152320,2018 -221406,2019 -160877,2019 -181306,2019 -176944,2019 -193816,2019 -240544,2019 -54901,2020 -144780,2020 -38009,2020 -3308,2021 -204846,2021 -134330,2021 -247687,2021 -248409,2021 -200435,2022 -142576,2022 -252761,2022 -158089,2018 -203886,2016 -141944,2016 -222159,2016 -176280,2016 -78825,2018 -248053,2017 -195049,2017 -66214,2017 -173029,2018 -61486,2018 -152112,2018 -2565,2019 -177168,2019 -238213,2019 -245783,2019 -142069,2019 -48702,2020 -146898,2021 -211930,2021 -255454,2021 -197714,2021 -179990,2021 -251967,2022 -195307,2022 -133844,2022 -196375,2022 -141793,2016 -238213,2016 -197953,2016 -1445,2017 -148563,2017 -232513,2017 -68497,2017 -148031,2018 -48352,2018 -131885,2018 -177109,2018 -73371,2018 -230189,2018 -220500,2018 -147945,2019 -125375,2019 -204846,2020 -251178,2020 -239085,2020 -141787,2021 -133572,2021 -66346,2022 -230361,2016 -205150,2016 -177093,2017 -205901,2017 -143455,2017 -111566,2018 -73238,2018 -196589,2019 -162361,2019 -176408,2020 -225626,2020 -50668,2022 -85010,2022 -181829,2022 -170804,2016 -185474,2016 -145147,2016 -148279,2017 -192817,2017 -192208,2017 -212629,2017 -193564,2018 -196078,2018 -162361,2018 -111566,2019 -211619,2021 -53337,2021 -114688,2022 -232822,2022 -204449,2022 -245844,2016 -214503,2017 -205150,2017 -117856,2018 -160306,2019 -143473,2019 -111566,2020 -210571,2021 -187415,2021 -176492,2022 -165461,2022 -189881,2017 -174118,2016 -169685,2016 -50864,2017 -177342,2017 -119031,2017 -88948,2017 -50864,2018 -200278,2018 -176434,2018 -222740,2019 -17874,2020 -204005,2020 -191159,2020 -170786,2021 -229054,2021 -250210,2021 -240535,2017 -188579,2021 -204724,2021 -187283,2021 -191788,2022 -134750,2016 -189543,2017 -147758,2017 -132239,2018 -150367,2018 -177018,2021 -200157,2021 -253719,2022 -191786,2022 -240934,2016 -187736,2016 -208308,2017 -189687,2018 -152759,2018 -56585,2019 -243348,2020 -152574,2021 -181054,2022 -8851,2016 -180127,2016 -108096,2017 -177159,2017 -155932,2017 -187661,2020 -207318,2022 -147111,2017 -212631,2018 -171124,2019 -150609,2020 -147758,2016 -158078,2016 -150489,2016 -177400,2019 -249252,2019 -133572,2020 -137797,2021 -186565,2021 -114174,2022 -161163,2016 -111783,2017 -167280,2018 -166842,2018 -166834,2018 -181663,2018 -166662,2021 -116381,2022 -109204,2022 -152112,2017 -187645,2018 -160973,2019 -164362,2020 -147941,2016 -160947,2016 -192054,2021 -100238,2022 -2437,2016 -94539,2018 -166773,2018 -87296,2017 -186567,2018 -155935,2018 -181663,2020 -232299,2022 -132357,2017 -196888,2020 -180812,2022 -147111,2016 -131864,2017 -70455,2021 -187661,2018 -250085,2021 -111758,2016 -189354,2018 -161165,2022 -190888,2021 -130216,2022 -111758,2017 -190848,2022 -195773,2022 -176120,2021 -246937,2016 -186878,2017 -205740,2017 -133864,2019 -194305,2020 -189227,2020 -189671,2022 -191146,2022 -156062,2021 -190163,2022 -161202,2017 -202936,2018 -143898,2016 -186321,2018 -169951,2021 -115793,2016 -171359,2019 -147073,2021 -205740,2016 -75188,2018 -148881,2019 -193596,2021 -125845,2018 -187872,2016 -170795,2021 -182086,2017 -184257,2018 -161057,2017 -161171,2022 -177880,2016 -193137,2021 -198172,2021 -189690,2022 -161080,2016 -155935,2022 -187750,2021 -161140,2016 -161042,2017 -161067,2017 -147964,2019 -135669,2021 -131915,2017 -189799,2020 -161107,2017 -81941,2020 -111110,2020 -161032,2017 -171359,2018 -158089,2021 -129783,2022 -161202,2016 -171359,2020 -181758,2019 -161089,2016 -137875,2021 -166794,2022 -192887,2016 -130213,2017 -132045,2018 -182191,2017 -131826,2017 -187872,2017 -3379,2017 -131873,2016 -65449,2022 -131826,2018 -212942,2022 -181667,2019 -130749,2018 -175887,2020 -181658,2020 -137965,2022 -147725,2021 -171359,2017 -181658,2022 -138467,2017 -134720,2020 -237887,2022 -188372,2022 -181663,2021 -181658,2016 -181658,2021 -181663,2017 -129783,2018 -193062,2020 -193062,2017 -152133,2017 -181663,2016 -171359,2016 -63579,2020 -118321,2016 -147719,2020 -170831,2017 -131804,2017 -211692,2020 -146632,2022 -180900,2022 -35765,2018 -180900,2021 -136829,2016 -15175,2018 -25545,2016 -219554,2019 -141337,2019 -138368,2019 -164682,2019 -206347,2019 -246159,2020 -154249,2020 -144484,2020 -194570,2021 -148708,2021 -239297,2021 -206033,2021 -162957,2022 -242150,2022 -244629,2022 -105561,2019 -192812,2019 -248836,2019 -223588,2019 -194250,2019 -81810,2019 -35712,2019 -135673,2019 -246203,2019 -138410,2019 -211695,2019 -243955,2019 -249687,2019 -208586,2019 -177336,2019 -232413,2019 -186332,2019 -213481,2019 -158768,2019 -233507,2019 -53220,2019 -95223,2019 -245853,2019 -30558,2020 -200553,2020 -249815,2020 -252604,2020 -256695,2020 -171150,2020 -171170,2020 -251053,2020 -186994,2020 -171082,2020 -102688,2020 -251128,2020 -11678,2020 -230397,2020 -136276,2020 -252824,2020 -91688,2020 -161805,2020 -131844,2020 -235987,2020 -196804,2020 -161297,2020 -246449,2020 -25727,2020 -21005,2020 -131978,2020 -119131,2020 -208000,2020 -241554,2020 -249360,2020 -129809,2020 -313,2020 -130204,2020 -55860,2021 -205103,2021 -205541,2021 -196597,2021 -228286,2021 -245506,2021 -241491,2021 -147743,2021 -252104,2021 -48987,2021 -254977,2021 -244891,2021 -250045,2021 -223855,2021 -77427,2021 -253621,2021 -26131,2021 -246542,2021 -255441,2021 -77932,2021 -83716,2021 -249360,2021 -238126,2021 -141966,2021 -147973,2021 -151594,2021 -196207,2021 -220979,2021 -256331,2021 -234022,2021 -210151,2021 -8526,2022 -190316,2022 -249120,2022 -254103,2022 -194335,2022 -185929,2022 -221846,2022 -228259,2022 -248768,2022 -153686,2022 -259060,2022 -233131,2022 -135791,2022 -189772,2022 -147084,2022 -257664,2022 -180929,2022 -252986,2022 -224118,2022 -196283,2022 -72641,2022 -239085,2022 -62590,2022 -258226,2022 -198573,2022 -258011,2022 -243458,2022 -252566,2022 -171944,2018 -904650,2021 -61410,2019 -6348,2019 -64332,2019 -160196,2019 -150003,2019 -146605,2019 -5982,2019 -101232,2019 -248692,2019 -160749,2019 -25511,2019 -101294,2019 -33900,2019 -85330,2019 -166588,2019 -61612,2019 -119271,2019 -151731,2019 -223983,2018 -3008,2017 -187355,2017 -234878,2018 -131594,2017 -38141,2017 -155834,2017 -239057,2017 -243612,2017 -114495,2017 -205148,2018 -165504,2018 -85763,2018 -27643,2018 -164346,2018 -179198,2018 -179551,2018 -170139,2017 -179363,2018 -179361,2018 -179558,2018 -240578,2017 -69041,2018 -233490,2018 -239473,2018 -234743,2017 -66214,2018 -228981,2018 -201672,2018 -201039,2017 -208320,2017 -177754,2017 -181494,2017 -229886,2017 -225001,2017 -123541,2016 -177877,2017 -36101,2016 -136325,2018 -182069,2017 -179272,2018 -188402,2017 -227433,2018 -174516,2017 -179591,2018 -135998,2018 -64655,2016 -238189,2017 -247897,2018 -186612,2018 -227668,2016 -206142,2016 -187566,2016 -54768,2016 -120051,2017 -164737,2018 -120267,2018 -160193,2016 -204858,2017 -177919,2017 -169355,2017 -150048,2017 -49508,2016 -179660,2018 -204843,2017 -153106,2017 -177835,2017 -144069,2017 -57590,2017 -179597,2018 -136699,2017 -229103,2018 -213972,2017 -179265,2018 -179290,2018 -221285,2017 -214225,2018 -132082,2017 -185706,2017 -5865,2018 -247583,2017 -179240,2018 -177356,2017 -220052,2017 -246845,2018 -247856,2018 -26012,2018 -209841,2017 -3659,2017 -179267,2018 -179372,2018 -179369,2018 -164319,2018 -147045,2018 -179271,2017 -243996,2017 -177993,2017 -183829,2017 -215301,2018 -215892,2017 -95993,2018 -177925,2017 -227642,2017 -134944,2017 -91989,2017 -216575,2017 -152086,2017 -154042,2017 -177936,2017 -205770,2018 -165467,2018 -211645,2017 -201383,2017 -111012,2017 -239644,2018 -205036,2018 -179292,2018 -93943,2017 -191791,2017 -191784,2017 -191116,2017 -244303,2017 -164389,2017 -197932,2017 -118330,2018 -244693,2017 -242586,2017 -215956,2017 -227101,2016 -230503,2018 -246207,2017 -222140,2016 -160805,2016 -209740,2018 -227271,2016 -200941,2016 -190761,2017 -219328,2016 -193631,2017 -186639,2017 -202585,2016 -38553,2016 -176357,2016 -95258,2016 -35877,2016 -179663,2016 -186507,2016 -86973,2016 -132247,2016 -236543,2016 -230363,2016 -228496,2016 -200730,2016 -128399,2016 -221796,2016 -186855,2016 -191871,2016 -142030,2016 -179359,2016 -219855,2016 -136162,2016 -96074,2016 -238336,2016 -230371,2016 -166685,2016 -163887,2016 -232779,2016 -192794,2016 -244983,2016 -155756,2016 -230448,2016 -159782,2016 -221331,2016 -191516,2016 -223655,2016 -199628,2016 -173305,2016 -131847,2016 -162489,2016 -18903,2016 -152762,2016 -99406,2016 -174600,2016 -243734,2016 -197454,2016 -193318,2016 -116086,2016 -88676,2016 -237007,2016 -165563,2022 -203473,2022 -212317,2022 -165750,2022 -258918,2022 -256749,2022 -195907,2022 -141688,2022 -142837,2021 -187625,2021 -141946,2022 -189688,2022 -177270,2021 -211616,2019 -165775,2020 -141758,2019 -209984,2019 -165769,2019 -227061,2018 -34158,2021 -228317,2022 -247746,2022 -34158,2022 -15452,2021 -15452,2022 -71648,2022 -71648,2021 -15452,2020 -71648,2020 -30108,2020 -34158,2020 -243753,2019 -15452,2019 -34158,2019 -204888,2022 -244299,2021 -134158,2017 -134054,2017 -137849,2017 -227934,2017 -227471,2017 -189972,2017 -25545,2017 -133877,2017 -15138,2017 -147719,2017 -134025,2017 -133875,2017 -131872,2017 -182630,2017 -134083,2017 -134057,2017 -134155,2017 -72985,2017 -133834,2017 -134105,2017 -133782,2017 -147733,2017 -134164,2017 -134152,2017 -134279,2017 -134045,2017 -202931,2017 -217558,2017 -134295,2017 -133823,2017 -134168,2017 -133862,2017 -237721,2017 -134138,2017 -38560,2018 -188493,2018 -242588,2018 -38560,2017 -188493,2017 -242588,2017 -185892,2018 -227682,2016 -231373,2016 -219160,2016 -154504,2016 -144163,2016 -170065,2016 -151697,2016 -181006,2016 -184335,2016 -246679,2017 -59643,2016 -238797,2016 -142663,2016 -118366,2019 -120776,2019 -238195,2022 -197262,2019 -245528,2019 -214191,2016 -119702,2016 -49357,2016 -151882,2016 -148279,2016 -72956,2017 -234338,2018 -120869,2017 -213708,2017 -246797,2017 -192128,2021 -130478,2018 -186616,2017 -161731,2018 -192484,2017 -228606,2017 -232644,2017 -232566,2017 -185069,2017 -197454,2018 -230630,2017 -192640,2017 -137668,2017 -246591,2017 -152848,2017 -230150,2016 -205572,2021 -173977,2021 -173924,2020 -173977,2020 -110936,2022 -110936,2021 -240046,2020 -205123,2019 -45469,2021 -45469,2022 -250419,2021 -250418,2020 -45469,2020 -250419,2020 -229622,2020 -111152,2020 -111152,2019 -250084,2021 -250084,2022 -111152,2022 -214068,2022 -170159,2022 -141858,2021 -111152,2021 -187745,2020 -141782,2020 -205272,2019 -250084,2020 -172606,2020 -203196,2019 -247784,2022 -225015,2022 -182626,2021 -200207,2021 -199369,2020 -175289,2019 -191782,2019 -43755,2019 -183990,2019 -252699,2020 -221010,2017 -207338,2017 -6075,2017 -133904,2017 -194426,2017 -56843,2017 -29637,2017 -246936,2017 -196273,2017 -70485,2020 -187338,2021 -250418,2021 -173977,2022 -187338,2022 -174341,2021 -110936,2020 -250838,2020 -141477,2021 -228837,2022 -141782,2022 -242213,2022 -242213,2021 -141782,2021 -152638,2016 -182173,2021 -141541,2021 -230565,2021 -230565,2020 -142028,2022 -86892,2021 -183243,2021 -245053,2021 -203580,2022 -37904,2021 -183243,2022 -39290,2022 -211851,2022 -138363,2022 -39295,2022 -43453,2022 -39295,2021 -39026,2022 -39026,2021 -211851,2021 -195481,2021 -43453,2021 -39290,2021 -138363,2021 -251297,2020 -197187,2019 -39290,2020 -248502,2020 -43453,2020 -186039,2020 -36393,2020 -245921,2020 -187723,2020 -39295,2020 -172651,2020 -241523,2020 -241944,2020 -241185,2020 -39026,2020 -11543,2020 -183243,2020 -186039,2021 -37904,2020 -18603,2019 -191692,2019 -247880,2020 -86892,2020 -138363,2020 -211851,2020 -165469,2019 -165467,2019 -246281,2019 -43453,2019 -118672,2019 -241144,2019 -138667,2019 -248503,2020 -30108,2019 -39295,2019 -138363,2019 -211851,2019 -204977,2019 -182458,2019 -183243,2019 -136113,2019 -186039,2019 -110926,2019 -187723,2019 -165468,2019 -229234,2019 -37904,2019 -165488,2019 -200326,2019 -86892,2019 -39026,2019 -216106,2019 -249647,2019 -241523,2019 -203580,2019 -145908,2019 -200191,2020 -215459,2020 -120130,2021 -216160,2021 -138666,2021 -258219,2022 -130575,2022 -131821,2022 -133942,2022 -137050,2022 -144156,2022 -155930,2022 -178024,2022 -182362,2022 -182955,2022 -199759,2022 -199799,2022 -221412,2022 -225564,2022 -232747,2022 -238531,2022 -245252,2022 -250417,2022 -256322,2022 -257491,2022 -258492,2022 -4184,2022 -66058,2022 -7633,2022 -133623,2022 -133922,2022 -133950,2022 -133959,2022 -134435,2022 -136939,2022 -144315,2022 -158000,2022 -158350,2022 -162511,2022 -170749,2022 -171967,2022 -180356,2022 -181243,2022 -182347,2022 -183607,2022 -183955,2022 -184907,2022 -192692,2022 -193190,2022 -198212,2022 -205292,2022 -213657,2022 -226095,2022 -228034,2022 -233660,2022 -243901,2022 -258159,2022 -259083,2022 -48219,2022 -184907,2021 -186200,2021 -187327,2021 -133578,2021 -144156,2021 -158350,2021 -170749,2021 -180879,2021 -197034,2021 -199945,2021 -208670,2021 -246036,2021 -246139,2021 -248482,2021 -248483,2021 -251582,2021 -254332,2021 -255073,2021 -257009,2021 -258544,2021 -7633,2021 -904721,2021 -130575,2021 -131821,2021 -131886,2021 -132580,2021 -128390,2020 -134446,2020 -144156,2020 -144315,2020 -156539,2020 -180356,2020 -184907,2020 -18842,2020 -193962,2020 -195097,2020 -198076,2020 -200095,2020 -211951,2020 -215924,2020 -238203,2020 -239561,2020 -240828,2020 -242384,2020 -243901,2020 -249987,2020 -250219,2020 -250420,2020 -250438,2020 -250488,2020 -250610,2020 -250882,2020 -250927,2020 -251948,2020 -252508,2020 -254501,2020 -130575,2020 -141279,2020 -158537,2020 -174205,2020 -182226,2020 -182362,2020 -191838,2020 -199485,2020 -224063,2020 -228034,2020 -244738,2020 -249855,2020 -250303,2020 -250324,2020 -250440,2020 -250683,2020 -250899,2020 -251245,2020 -252693,2020 -254776,2020 -256729,2020 -66058,2020 -66064,2020 -66070,2020 -7505,2020 -904475,2020 -129783,2019 -132580,2019 -136308,2019 -162507,2019 -131821,2019 -147944,2019 -180337,2019 -182362,2019 -183394,2019 -184907,2019 -188848,2019 -193190,2019 -196436,2019 -197680,2019 -208670,2019 -211951,2019 -223594,2019 -238531,2019 -245252,2019 -249141,2019 -249724,2019 -250001,2019 -251266,2019 -251840,2019 -136939,2021 -144315,2021 -155930,2021 -162181,2021 -182362,2021 -193962,2021 -194914,2021 -199759,2021 -208014,2021 -221412,2021 -221809,2021 -251782,2021 -252508,2021 -252820,2021 -252887,2021 -254173,2021 -254898,2021 -255262,2021 -255833,2021 -256127,2021 -904604,2021 -170831,2019 -185678,2019 -18842,2019 -192079,2019 -201962,2019 -250845,2019 -250927,2019 -251867,2019 -256729,2019 -162181,2020 -162442,2019 -59154,2020 -177867,2019 -183227,2019 -149997,2019 -234848,2019 -189799,2019 -243307,2020 -207609,2021 -211578,2017 -211578,2016 -200598,2022 -169236,2016 -160549,2022 -169236,2018 -160609,2022 -97814,2021 -97814,2019 -160439,2022 -19913,2022 -219415,2020 -250957,2020 -36465,2019 -169236,2017 -19002,2022 -97814,2022 -128545,2021 -231040,2019 -34371,2021 -232307,2022 -212452,2021 -97814,2020 -98002,2017 -61372,2017 -213144,2019 -9900,2019 -9389,2019 -220293,2019 -98002,2018 -255789,2022 -252211,2022 -232700,2022 -179933,2021 -252671,2021 -250347,2020 -227975,2020 -227404,2019 -240535,2018 -231141,2019 -184882,2017 -201754,2017 -97978,2018 -245353,2016 -201754,2016 -256075,2021 -242628,2021 -254662,2022 -237472,2018 -242898,2021 -231040,2021 -231040,2020 -250571,2020 -88591,2020 -98002,2020 -195952,2020 -185776,2019 -246532,2018 -105747,2018 -98002,2019 -226845,2016 -237472,2017 -246658,2017 -219895,2017 -244340,2017 -224186,2016 -175278,2016 -238093,2016 -198009,2016 -203015,2021 -234451,2020 -234451,2019 -96927,2020 -176362,2020 -205298,2018 -186063,2017 -148755,2016 -77803,2021 -78068,2022 -87913,2018 -237832,2020 -126995,2017 -25515,2020 -80052,2018 -87913,2019 -127155,2017 -87913,2016 -2854,2019 -237832,2019 -2854,2021 -2854,2020 -237832,2018 -179684,2021 -77673,2017 -25515,2018 -191719,2021 -150053,2020 -134043,2019 -188008,2021 -201693,2018 -188008,2020 -188008,2018 -188008,2019 -147725,2022 -81328,2021 -133933,2019 -198499,2020 -125011,2020 -181663,2019 -243753,2018 -218330,2017 -195177,2021 -249608,2022 -246694,2017 -68000,2017 -91983,2016 -133993,2019 -87913,2017 -206851,2019 -134208,2021 -133883,2022 -68000,2020 -210107,2016 -118338,2018 -170653,2017 -201083,2016 -5982,2020 -197622,2022 -134200,2021 -5982,2021 -245327,2017 -131879,2021 -68000,2019 -68000,2021 -93398,2018 -93398,2017 -131879,2022 -68000,2022 -120244,2022 -119104,2018 -195408,2016 -25515,2019 -245299,2016 -25515,2021 -15491,2021 -179604,2020 -224465,2018 -42338,2020 -63509,2017 -97826,2016 -237832,2017 -251210,2020 -182165,2022 -171382,2019 -184315,2019 -205745,2021 -181178,2017 -132044,2022 -136772,2019 -249397,2020 -32660,2016 -202,2022 -171424,2019 -181178,2019 -175013,2022 -182926,2020 -180985,2021 -63579,2022 -162412,2019 -154226,2017 -182454,2020 -183764,2019 -250736,2022 -180900,2017 -72985,2019 -138590,2021 -175013,2020 -123466,2019 -136772,2022 -213388,2020 -32660,2018 -171977,2018 -195724,2019 -18081,2021 -32660,2017 -77883,2021 -171421,2019 -196737,2020 -15175,2020 -15175,2021 -92331,2019 -148727,2018 -123694,2019 -219107,2019 -245476,2019 -192928,2021 -32660,2019 -176851,2020 -147719,2018 -6735,2022 -136772,2021 -170808,2019 -69267,2022 -162405,2021 -162405,2020 -248888,2018 -198478,2019 -2131,2019 -194793,2020 -122370,2022 -120172,2022 -120172,2021 -211897,2019 -80598,2020 -181758,2018 -203870,2022 -72948,2020 -193986,2018 -72948,2021 -148729,2018 -36348,2018 -195222,2022 -182396,2018 -162405,2019 -195222,2021 -69267,2021 -194704,2021 -162405,2022 -2961,2020 -136308,2022 -131847,2021 -137921,2021 -161052,2020 -25395,2021 -213388,2022 -187424,2019 -185929,2019 -181661,2019 -233569,2021 -6735,2019 -166795,2019 -158391,2018 -213388,2021 -131807,2022 -166054,2020 -189972,2022 -162414,2022 -45510,2020 -131847,2020 -258457,2022 -6735,2018 -93230,2021 -158380,2018 -65449,2018 -166054,2022 -2961,2021 -155652,2019 -176088,2019 -6735,2021 -181758,2021 -148751,2018 -129783,2020 -42780,2021 -182884,2020 -123694,2020 -78408,2022 -72712,2018 -1465,2022 -188084,2018 -91200,2019 -136063,2022 -176379,2018 -152133,2020 -166019,2021 -145856,2018 -52086,2020 -181667,2022 -194305,2019 -130209,2022 -37861,2022 -258472,2022 -181554,2019 -72105,2019 -179223,2020 -100050,2018 -189201,2018 -18064,2017 -250736,2020 -221192,2018 -8827,2018 -162411,2022 -195537,2021 -124244,2021 -244592,2019 -124244,2022 -78063,2022 -156529,2018 -131871,2020 -80286,2022 -80286,2020 -80286,2021 -136521,2021 -181554,2017 -182074,2021 -233569,2018 -186914,2021 -200449,2018 -33299,2017 -193947,2020 -10724,2019 -258431,2022 -80286,2019 -118676,2022 -212928,2019 -17157,2019 -18565,2022 -187507,2018 -170309,2020 -185800,2020 -147731,2022 -201412,2021 -166477,2022 -132057,2022 -181658,2019 -170804,2019 -182968,2019 -179867,2020 -29193,2018 -147731,2021 -182191,2022 -160989,2019 -193634,2022 -185929,2021 -170831,2020 -187704,2018 -132155,2021 -174986,2021 -100050,2020 -156972,2020 -204769,2017 -17122,2021 -187317,2021 -166399,2021 -185800,2021 -182755,2022 -78063,2021 -187704,2019 -143605,2021 -199484,2018 -85608,2020 -182454,2022 -170263,2017 -141524,2022 -101742,2022 -144797,2021 -182916,2019 -70282,2017 -181427,2022 -152133,2019 -176394,2018 -123466,2021 -160969,2019 -211695,2022 -65853,2019 -179712,2018 -121369,2018 -135840,2021 -166670,2019 -174778,2022 -197909,2022 -220190,2020 -186438,2021 -190313,2022 -146779,2020 -232373,2021 -134126,2019 -176254,2021 -220190,2022 -185406,2020 -134136,2020 -202341,2021 -63212,2016 -209451,2020 -135760,2020 -205106,2021 -133895,2020 -161094,2021 -177931,2017 -171698,2021 -209663,2021 -186474,2016 -245496,2021 -157629,2022 -133870,2021 -135828,2022 -174774,2022 -188181,2021 -197194,2022 -170805,2022 -81346,2020 -157844,2022 -181050,2021 -15322,2022 -181657,2022 -200266,2022 -231939,2022 -243857,2021 -157792,2022 -161067,2020 -161151,2022 -227662,2021 -133911,2022 -232433,2019 -188065,2022 -133911,2020 -169988,2016 -92979,2019 -92979,2018 -161027,2021 -157786,2022 -161177,2022 -92979,2020 -186157,2022 -161042,2022 -161032,2020 -158417,2019 -161018,2020 -161067,2021 -150467,2021 -198496,2020 -161018,2019 -157757,2022 -219566,2022 -160998,2021 -133767,2022 -157852,2022 -247901,2021 -139184,2021 -211897,2021 -211897,2022 -134122,2020 -151697,2017 -151697,2018 -248480,2016 -248480,2018 -164868,2016 -212625,2018 -219129,2018 -227927,2018 -232762,2018 -212625,2017 -227927,2017 -219113,2018 -164868,2017 -219129,2017 -232762,2017 -212625,2016 -219129,2016 -227927,2016 -232762,2016 -223003,2016 -181006,2017 -214191,2018 -243110,2018 -141321,2017 -214191,2017 -223003,2018 -38675,2018 -108177,2018 -125014,2018 -209721,2018 -218813,2018 -209721,2017 -191960,2018 -216877,2017 -202715,2018 -216877,2018 -164912,2017 -249560,2018 -218813,2017 -224092,2018 -202715,2017 -38675,2017 -223003,2017 -151882,2017 -68500,2016 -164912,2016 -202715,2016 -181423,2016 -38675,2016 -213293,2016 -170120,2020 -222514,2020 -206360,2020 -247298,2022 -174545,2019 -245949,2022 -169637,2020 -174271,2022 -166556,2021 -231060,2022 -247713,2022 -188608,2020 -249073,2020 -257557,2022 -166559,2021 -186212,2021 -253539,2020 -207099,2022 -171671,2022 -96959,2019 -182940,2022 -247095,2022 -161152,2022 -133605,2021 -186212,2022 -249737,2019 -257848,2021 -193581,2019 -242983,2020 -166039,2021 -165674,2020 -166480,2022 -223389,2022 -133605,2022 -204031,2022 -197909,2019 -249439,2020 -162437,2022 -146779,2019 -49685,2022 -195929,2021 -249439,2019 -250934,2020 -166761,2021 -133602,2022 -166761,2022 -157547,2021 -166536,2021 -192862,2021 -187723,2021 -166604,2021 -131902,2019 -197909,2020 -238203,2019 -195487,2022 -160983,2020 -161186,2021 -160989,2022 -157850,2022 -249505,2019 -133882,2021 -204417,2022 -161082,2021 -229253,2020 -161093,2019 -157784,2022 -160946,2021 -161160,2020 -161089,2021 -219113,2017 -219113,2016 -202121,2016 -181006,2018 -177452,2017 -217833,2017 -218813,2016 -224092,2017 -221304,2018 -29466,2017 -164882,2016 -234631,2021 -141226,2019 -246870,2022 -232386,2019 -157761,2022 -170349,2019 -170349,2020 -205409,2021 -136080,2019 -249073,2019 -141542,2022 -3830,2022 -206688,2020 -205409,2020 -185443,2021 -245887,2022 -161156,2022 -174996,2020 -177408,2018 -176278,2022 -193043,2016 -144797,2020 -158024,2019 -108096,2020 -45510,2022 -220190,2021 -242168,2018 -224092,2016 -161099,2021 -248803,2022 -232041,2022 -205536,2022 -223716,2022 -230486,2021 -157642,2021 -145507,2019 -242181,2021 -14842,2021 -134269,2021 -231945,2022 -226279,2019 -160498,2021 -54219,2021 -144373,2022 -19530,2021 -177052,2020 -161166,2021 -158396,2018 -161075,2021 -137901,2018 -86512,2022 -196861,2017 -164845,2017 -154077,2019 -162449,2018 -135829,2022 -252761,2021 -250108,2022 -236889,2020 -191833,2019 -133943,2019 -133870,2020 -197291,2021 -175159,2021 -252076,2021 -190779,2022 -133870,2019 -236161,2020 -131876,2016 -188059,2022 -188059,2021 -133827,2022 -133937,2019 -133828,2022 -134142,2022 -133937,2022 -133845,2021 -136123,2019 -241521,2019 -133937,2020 -134077,2021 -188059,2020 -232604,2022 -134144,2020 -176289,2022 -216437,2018 -186480,2022 -186661,2020 -206178,2022 -187886,2018 -185341,2021 -240953,2019 -187338,2019 -134096,2022 -134580,2022 -255765,2022 -193543,2022 -186388,2020 -181086,2022 -134142,2021 -133886,2022 -157646,2021 -133950,2019 -133557,2019 -134586,2022 -134144,2022 -134130,2019 -201641,2020 -134130,2020 -134045,2022 -197291,2020 -134142,2019 -241356,2020 -133943,2022 -133845,2020 -134045,2020 -133882,2020 -133885,2019 -133845,2022 -160178,2022 -182678,2019 -133943,2020 -134139,2020 -156039,2019 -258861,2022 -176362,2018 -244461,2022 -135836,2020 -151898,2021 -1815,2022 -134139,2019 -134431,2020 -134586,2019 -134096,2019 -44083,2021 -133885,2022 -133882,2019 -188059,2019 -133882,2022 -157481,2021 -161129,2022 -2619,2020 -134431,2019 -133950,2020 -157662,2021 -86512,2021 -134431,2022 -141785,2022 -237824,2021 -134132,2021 -133885,2020 -197909,2021 -201528,2021 -242303,2019 -242303,2020 -134104,2021 -145635,2019 -247711,2019 -134048,2021 -134182,2022 -134136,2019 -134182,2020 -134146,2022 -133830,2019 -187036,2021 -188608,2022 -134096,2020 -193698,2018 -247972,2021 -223527,2020 -223527,2021 -201017,2021 -133830,2020 -233179,2019 -157663,2021 -176283,2022 -156590,2022 -157507,2020 -133839,2022 -134092,2020 -144816,2020 -151705,2020 -133911,2019 -258552,2021 -161070,2021 -92997,2022 -244898,2021 -237824,2022 -131874,2021 -146878,2022 -134136,2021 -134136,2022 -134104,2020 -196951,2022 -134142,2020 -134048,2022 -144816,2019 -197291,2022 -230344,2020 -176362,2016 -211695,2021 -230344,2021 -135840,2020 -249505,2022 -158405,2018 -186157,2021 -132225,2021 -205143,2022 -195146,2019 -157472,2021 -207821,2020 -134092,2019 -157670,2022 -134402,2022 -157448,2022 -156753,2021 -134132,2020 -156753,2019 -131874,2022 -91317,2018 -197070,2019 -176362,2017 -200579,2018 -248411,2022 -194244,2017 -182355,2016 -59870,2016 -25950,2022 -214309,2016 -255101,2021 -255101,2022 -125765,2019 -125765,2020 -166849,2017 -166849,2018 -206033,2016 -166851,2018 -147871,2020 -131882,2018 -22967,2022 -2227,2022 -158396,2021 -131886,2017 -134589,2017 -190859,2016 -191092,2016 -190159,2016 -190163,2016 -143452,2016 -184588,2016 -190847,2016 -184590,2016 -194229,2016 -187222,2016 -189685,2016 -189687,2016 -194335,2016 -192109,2016 -188726,2016 -190848,2016 -194314,2016 -191415,2016 -191768,2016 -232308,2016 -192315,2016 -192108,2016 -190979,2016 -154502,2016 -191950,2016 -190247,2016 -191012,2016 -188248,2016 -186065,2016 -191610,2016 -191146,2016 -189117,2016 -189782,2016 -189790,2016 -191409,2016 -190204,2016 -190318,2016 -188733,2016 -202163,2016 -227779,2016 -188727,2016 -192173,2016 -223387,2016 -223747,2016 -143586,2016 -190177,2016 -191949,2016 -211681,2016 -191727,2016 -202938,2016 -190245,2016 -186838,2016 -192677,2016 -190180,2016 -143329,2016 -190995,2016 -188565,2016 -191104,2016 -192675,2016 -143546,2016 -189735,2016 -211158,2016 -188734,2016 -189688,2016 -188340,2016 -191416,2016 -191708,2016 -190978,2016 -190846,2016 -143343,2016 -191090,2016 -191186,2016 -188735,2016 -189732,2016 -189791,2016 -207267,2016 -191213,2016 -190986,2016 -190364,2016 -191159,2016 -191021,2016 -189775,2016 -191621,2016 -191407,2016 -201915,2016 -191951,2016 -227561,2016 -189690,2016 -243869,2016 -189819,2016 -190845,2016 -232619,2016 -193682,2016 -189772,2016 -197685,2016 -201820,2016 -190175,2016 -191152,2016 -192111,2016 -190164,2016 -191412,2016 -190182,2016 -189774,2016 -191881,2016 -94572,2017 -190861,2016 -196899,2016 -191141,2016 -191010,2016 -187492,2016 -191210,2016 -211277,2016 -192796,2016 -192171,2016 -94572,2016 -191145,2016 -233155,2016 -232929,2016 -190989,2016 -191205,2016 -186781,2016 -227614,2016 -232794,2016 -190210,2016 -191102,2016 -94770,2016 -94770,2017 -190983,2016 -212212,2016 -211510,2016 -191948,2016 -191094,2016 -202046,2016 -191724,2016 -190521,2016 -190316,2016 -232299,2016 -190862,2016 -197740,2016 -232617,2016 -197735,2016 -202048,2016 -212220,2016 -232847,2016 -232910,2016 -187688,2016 -232930,2016 -211666,2016 -213650,2016 -188657,2016 -191946,2016 -232792,2016 -211511,2016 -202876,2016 -94572,2018 -191147,2016 -202047,2016 -94770,2018 -184271,2018 -184271,2017 -184271,2016 -211062,2016 -74813,2016 -147919,2016 -200682,2017 -231770,2016 -82017,2016 -245399,2016 -240674,2018 -184725,2017 -184725,2018 -206194,2018 -184722,2018 -184722,2017 -206194,2017 -184723,2018 -184726,2018 -240674,2017 -221412,2017 -184828,2018 -184720,2018 -184723,2017 -184726,2017 -184828,2017 -184720,2017 -234415,2016 -240674,2016 -166670,2017 -221412,2016 -221412,2018 -184725,2016 -246804,2016 -248904,2016 -246205,2016 -184723,2016 -184720,2016 -184722,2016 -184726,2016 -184828,2016 -206194,2016 -224263,2016 -246525,2017 -245657,2016 -11674,2017 -134101,2017 -235343,2017 -191936,2019 -231334,2019 -251231,2019 -82834,2019 -203817,2019 -13836,2018 -184525,2021 -248282,2018 -115801,2021 -32699,2019 -218847,2021 -240506,2021 -195558,2020 -180133,2022 -223772,2021 -223772,2020 -258158,2022 -258074,2021 -180177,2018 -230558,2020 -257745,2022 -904751,2022 -904753,2022 -904739,2022 -904748,2022 -904749,2022 -904752,2022 -257168,2022 -904706,2022 -904750,2022 -904763,2022 -904764,2022 -904776,2022 -239186,2021 -239456,2021 -243104,2021 -75839,2021 -904769,2021 -904774,2021 -149070,2021 -154246,2021 -154247,2021 -215977,2021 -904639,2021 -904644,2021 -255084,2021 -255204,2021 -255900,2021 -255956,2021 -256443,2021 -904573,2021 -904581,2021 -904603,2021 -904605,2021 -904610,2021 -904667,2021 -904676,2021 -242836,2021 -77249,2021 -904686,2021 -904715,2021 -904653,2021 -904654,2021 -904660,2021 -251089,2021 -251650,2021 -252346,2021 -252461,2021 -904647,2021 -904648,2021 -904777,2021 -904779,2021 -235314,2021 -250946,2021 -251025,2021 -251076,2021 -254211,2021 -254414,2021 -254122,2021 -254714,2021 -252358,2020 -252394,2020 -252832,2020 -252834,2020 -252846,2020 -904559,2020 -225532,2020 -220815,2020 -237347,2020 -904551,2020 -904666,2020 -904690,2020 -251235,2020 -252835,2020 -252847,2020 -252893,2020 -252950,2020 -253125,2020 -249245,2020 -249479,2020 -250294,2020 -251197,2020 -904484,2020 -904699,2020 -154277,2020 -154246,2020 -154247,2020 -191445,2020 -904555,2020 -904618,2020 -212403,2020 -244421,2020 -251253,2020 -251299,2020 -251746,2020 -251860,2020 -252203,2020 -904566,2020 -904727,2020 -192416,2020 -904608,2020 -904615,2020 -175190,2019 -193973,2019 -199272,2019 -215660,2019 -242795,2019 -247742,2019 -39290,2019 -97795,2019 -247820,2019 -248449,2019 -248665,2019 -249016,2019 -250537,2019 -250771,2019 -904593,2019 -904594,2019 -904719,2019 -904788,2019 -904451,2019 -904459,2019 -230793,2019 -904471,2019 -904325,2019 -904368,2019 -904445,2019 -904479,2019 -904500,2019 -904512,2019 -235223,2018 -224327,2018 -223089,2018 -197898,2018 -191929,2018 -172153,2018 -249006,2018 -250448,2018 -248685,2018 -248511,2018 -248837,2018 -248198,2018 -248119,2018 -247095,2018 -246668,2018 -242635,2018 -236124,2018 -183892,2018 -216261,2018 -158744,2018 -214227,2018 -216431,2018 -904633,2018 -904137,2018 -904329,2018 -904391,2018 -904433,2018 -904468,2018 -904536,2018 -904580,2018 -904101,2018 -904773,2018 -199802,2017 -160841,2017 -160839,2017 -170077,2017 -140482,2017 -22798,2017 -904393,2017 -904549,2017 -904675,2017 -904088,2017 -214227,2017 -903932,2017 -246862,2017 -246823,2017 -247262,2017 -246762,2017 -247342,2017 -246691,2017 -247743,2017 -246478,2017 -246430,2017 -246272,2017 -244887,2017 -247788,2017 -247208,2017 -248837,2017 -247015,2017 -247658,2016 -245687,2016 -255666,2016 -245797,2016 -243781,2016 -241142,2016 -220881,2016 -199903,2016 -244464,2016 -245891,2016 -246032,2016 -246503,2016 -247345,2016 -182108,2016 -212404,2016 -214227,2016 -62641,2016 -903732,2016 -904350,2016 -203150,2016 -212516,2016 -92172,2018 -200449,2021 -180912,2020 -147633,2020 -229886,2019 -147635,2019 -236473,2020 -200130,2019 -159651,2021 -159651,2020 -195758,2020 -203339,2018 -250336,2020 -192787,2019 -139446,2022 -257528,2022 -97378,2018 -225260,2022 -184025,2022 -33364,2021 -134414,2018 -163998,2018 -191533,2022 -151109,2022 -123976,2022 -40858,2020 -200216,2020 -255001,2022 -216555,2020 -240234,2022 -233859,2021 -232538,2018 -144893,2020 -6196,2017 -195030,2020 -160733,2021 -160462,2018 -222798,2018 -128408,2021 -104819,2022 -201778,2020 -70729,2018 -208597,2020 -19265,2020 -108729,2019 -131402,2022 -188167,2021 -135888,2021 -181503,2020 -233807,2020 -230853,2020 -202067,2021 -241501,2019 -204581,2021 -169234,2020 -246395,2020 -257022,2021 -248547,2022 -227083,2019 -242286,2021 -251249,2020 -237091,2020 -241618,2020 -73966,2022 -24133,2022 -251385,2020 -219756,2021 -43419,2020 -113129,2020 -231180,2021 -221722,2022 -230608,2019 -210294,2019 -131011,2020 -237731,2022 -174480,2021 -245059,2021 -231494,2021 -146766,2022 -195203,2020 -179942,2021 -186894,2020 -110766,2022 -225705,2022 -177909,2021 -181726,2021 -243181,2019 -250712,2022 -236351,2021 -203116,2020 -230697,2021 -170743,2020 -229536,2022 -170118,2020 -136561,2021 -37007,2018 -197793,2020 -234426,2021 -76857,2018 -110982,2021 -247762,2021 -190281,2020 -184556,2018 -251927,2022 -237943,2021 -154954,2021 -154944,2020 -244854,2020 -237943,2020 -215526,2020 -215047,2020 -188745,2020 -24463,2021 -24463,2020 -185706,2020 -202420,2018 -250153,2021 -187680,2021 -164198,2021 -164152,2021 -245397,2020 -42733,2022 -42733,2021 -239520,2021 -164900,2021 -208658,2021 -257447,2021 -254217,2021 -203814,2018 -253902,2021 -208658,2018 -164845,2021 -198160,2021 -164845,2020 -220858,2020 -150536,2020 -211102,2019 -232770,2020 -238919,2020 -26082,2017 -205353,2022 -230738,2022 -373,2021 -169938,2022 -169938,2021 -238459,2021 -211489,2021 -226427,2021 -220696,2022 -148751,2020 -248547,2021 -248547,2018 -248547,2020 -194697,2021 -229628,2020 -195438,2020 -181769,2020 -232265,2022 -228358,2018 -158282,2018 -227168,2018 -147662,2022 -147662,2021 -225245,2021 -184789,2020 -247164,2018 -247572,2018 -227076,2021 -181692,2022 -170586,2021 -170828,2021 -170828,2022 -242889,2021 -248772,2018 -241339,2017 -114257,2020 -252871,2022 -214841,2020 -130587,2019 -196979,2020 -220158,2022 -188759,2020 -179978,2021 -14123,2021 -175391,2022 -250290,2018 -250290,2017 -225466,2019 -114138,2022 -114138,2021 -114138,2020 -252236,2020 -82175,2022 -246769,2021 -105840,2020 -249121,2022 -250085,2020 -181024,2022 -191284,2022 -85483,2020 -242333,2022 -251203,2020 -259056,2021 -60442,2021 -257529,2022 -81410,2022 -256142,2022 -254775,2021 -252793,2021 -152295,2019 -174235,2021 -175289,2018 -255465,2022 -192840,2021 -192840,2020 -75117,2020 -198496,2019 -182926,2019 -138590,2017 -162392,2017 -138590,2018 -193698,2016 -138590,2016 -204066,2022 -259010,2022 -250980,2022 -132184,2022 -132234,2022 -132038,2022 -207269,2022 -149112,2022 -201400,2022 -221748,2022 -186845,2022 -221258,2022 -241217,2022 -132027,2022 -250646,2022 -129089,2022 -257730,2022 -155962,2022 -143705,2022 -258704,2022 -219545,2022 -186571,2022 -235543,2022 -258453,2022 -176503,2022 -258687,2022 -162699,2022 -194518,2022 -199286,2022 -206372,2022 -258071,2022 -257727,2022 -258078,2022 -188000,2022 -208001,2022 -214268,2022 -149236,2022 -234363,2022 -257535,2022 -190659,2022 -258833,2022 -138899,2022 -258076,2022 -258370,2022 -258820,2022 -211400,2022 -246054,2022 -183688,2022 -254036,2022 -240266,2022 -258418,2022 -258461,2022 -131924,2022 -202985,2022 -258557,2022 -250842,2022 -243260,2022 -257199,2022 -190140,2022 -199660,2022 -258898,2022 -198108,2022 -258755,2022 -201487,2022 -250623,2022 -240530,2022 -137066,2022 -211071,2022 -212437,2022 -172852,2022 -221301,2022 -203303,2022 -161576,2022 -205843,2022 -172208,2022 -200715,2022 -151230,2022 -225312,2022 -257315,2022 -257099,2022 -143646,2022 -188783,2022 -258009,2022 -178453,2022 -222579,2022 -175073,2022 -258996,2022 -251006,2022 -137166,2022 -196421,2022 -193815,2022 -202966,2022 -194096,2022 -248634,2022 -259135,2022 -259062,2022 -258455,2022 -218763,2022 -208429,2022 -136270,2022 -205911,2022 -131963,2022 -191922,2022 -258298,2022 -194443,2022 -258585,2022 -219968,2022 -145265,2022 -240994,2022 -248310,2022 -181475,2022 -252603,2022 -251063,2022 -258837,2022 -258360,2022 -259016,2022 -132072,2022 -148743,2022 -191240,2022 -206941,2022 -136926,2022 -259145,2022 -149886,2022 -210920,2022 -193361,2022 -234662,2022 -258646,2022 -208741,2022 -258560,2022 -258208,2022 -195915,2022 -257164,2022 -258561,2022 -250974,2022 -213361,2022 -224358,2022 -236558,2022 -200634,2022 -257925,2022 -241776,2022 -228509,2022 -190278,2022 -132288,2022 -211185,2022 -211455,2022 -234558,2022 -191238,2022 -152819,2022 -201282,2022 -258602,2022 -238979,2022 -259132,2022 -202423,2022 -204362,2022 -132042,2022 -136469,2022 -201545,2022 -129048,2022 -144837,2022 -238710,2022 -179798,2022 -176495,2022 -132179,2022 -250287,2022 -251068,2022 -200061,2022 -251872,2022 -194819,2022 -228680,2022 -195437,2022 -239269,2022 -144931,2022 -228517,2022 -219133,2022 -257233,2022 -204225,2022 -245233,2022 -194371,2022 -257993,2022 -144808,2022 -192631,2022 -139128,2022 -238653,2022 -247680,2022 -256295,2022 -231932,2022 -134801,2022 -213224,2022 -252166,2022 -258818,2022 -166191,2022 -147753,2022 -209819,2022 -217386,2022 -258412,2022 -238604,2022 -250766,2022 -225071,2022 -187139,2022 -188863,2022 -257610,2022 -250128,2022 -252204,2022 -228346,2022 -258543,2022 -198067,2022 -228903,2022 -243574,2022 -149411,2022 -258507,2022 -258583,2022 -211907,2022 -243081,2022 -199893,2022 -258113,2022 -143718,2022 -228361,2022 -216457,2022 -227928,2022 -176507,2022 -192673,2022 -201610,2022 -152817,2022 -257304,2022 -161643,2022 -258982,2022 -194082,2022 -222305,2022 -251069,2022 -250008,2022 -192884,2022 -203336,2022 -258359,2022 -248381,2022 -239869,2022 -258951,2022 -249629,2022 -170426,2022 -252758,2022 -248426,2022 -256467,2022 -212668,2022 -209387,2022 -216737,2022 -257920,2022 -137221,2022 -225092,2022 -250825,2022 -190169,2022 -258075,2022 -230797,2022 -248202,2022 -132168,2022 -258706,2022 -258423,2022 -257641,2022 -187205,2022 -202995,2022 -208871,2022 -250793,2022 -239050,2022 -213495,2022 -258956,2022 -179813,2022 -191840,2022 -256840,2022 -136340,2022 -238725,2022 -244655,2022 -250398,2022 -231240,2022 -225181,2022 -195127,2022 -237027,2022 -162674,2022 -175116,2022 -132025,2022 -250402,2022 -136475,2022 -258130,2022 -241102,2022 -199892,2022 -240729,2022 -258748,2022 -228246,2022 -258321,2022 -258658,2022 -149218,2022 -213558,2022 -202545,2022 -219387,2022 -250953,2022 -145911,2022 -212627,2022 -255986,2022 -244191,2022 -176425,2022 -192240,2022 -250022,2022 -208436,2022 -181477,2022 -214629,2022 -251931,2022 -209917,2022 -258121,2022 -257914,2022 -234296,2022 -231306,2022 -178441,2022 -220403,2022 -152806,2022 -162610,2022 -208032,2022 -258135,2022 -143655,2022 -258488,2022 -191598,2022 -904756,2022 -131916,2017 -171944,2017 \ No newline at end of file diff --git a/backend/tools/load_public_dissem_data/Dockerfile b/backend/tools/load_public_dissem_data/Dockerfile new file mode 100644 index 0000000000..53614673ba --- /dev/null +++ b/backend/tools/load_public_dissem_data/Dockerfile @@ -0,0 +1,27 @@ +FROM debian:bookworm + +RUN apt-get update && \ + apt-get -y upgrade +RUN apt-get install -y \ + postgresql-client \ + curl \ + unzip + +WORKDIR /app + +# Curl in `sling` +RUN curl -LO 'https://github.com/slingdata-io/sling-cli/releases/latest/download/sling_linux_amd64.tar.gz' \ + && tar xf sling_linux_amd64.tar.gz \ + && rm -f sling_linux_amd64.tar.gz \ + && chmod +x sling \ + && mv sling /bin/sling + +RUN curl -L -O https://github.com/GSA-TTS/fac-backup-utility/releases/download/v0.1.8/gov.gsa.fac.cgov-util-v0.1.8-linux-amd64.tar.gz \ + && tar xvzf gov.gsa.fac.cgov-util-v0.1.8-linux-amd64.tar.gz gov.gsa.fac.cgov-util \ + && chmod 755 gov.gsa.fac.cgov-util \ + && mv gov.gsa.fac.cgov-util /bin/gov.gsa.fac.cgov-util + +WORKDIR /root/.fac +RUN curl -L -O https://raw.githubusercontent.com/GSA-TTS/fac-backup-utility/refs/heads/main/config.json + +ENTRYPOINT [ "./load_public_data_locally.sh" ] diff --git a/backend/tools/load_public_dissem_data/README.md b/backend/tools/load_public_dissem_data/README.md new file mode 100644 index 0000000000..5dc9b32630 --- /dev/null +++ b/backend/tools/load_public_dissem_data/README.md @@ -0,0 +1,50 @@ +# loading public data + +This script loads public data. + +## Grab the ZIP + +You'll need to grab + +https://drive.google.com/drive/folders/1gUsqD31Pkd17CruE4PWwwPKJVUssYNnI + +which is cleaned/historic public data, fit for our dissem_* tables. + +Compressed, it is 330MB. Uncompressed, around 3GB. + +Put it in dissemination/tools/load_public_dissem_data (this directory) + +## Run the script + +``` +./load_public_data_locally.sh +``` + +Note you need to run it from within that directory. + +## Or, run the Dockerfile + +If you're on a Mac, you may want to run the docker container to do the load. + +First, build the container. + +``` +docker build -t facloaddata . +``` + +Then, with the zipfile of data in the current directory: + +``` +docker run -i --rm -v ${PWD}:/app \ + --network container:backend-web-1 \ + -t facloaddata +``` + +The `--network` flag tells Docker to run this container on the same network as your currently running stack. It assumes that you did a `docker compose up` on the FAC stack, and that the web container has the default name of `backend-web-1`. If this does not work, you will need to do a + +``` +docker ps +``` + +and discover the name of a container running in your FAC stack. Once you do this, it will be possible for the loader script to find the Postgres databases. + diff --git a/backend/tools/load_public_dissem_data/data/README.md b/backend/tools/load_public_dissem_data/data/README.md new file mode 100644 index 0000000000..6019b70931 --- /dev/null +++ b/backend/tools/load_public_dissem_data/data/README.md @@ -0,0 +1,3 @@ +The historic/public data dump ends up in this directory. + +The container will mount this directory in order to unzip and load the data into the local environment. diff --git a/backend/tools/load_public_dissem_data/load_public_data_locally.sh b/backend/tools/load_public_dissem_data/load_public_data_locally.sh new file mode 100755 index 0000000000..b8ee813e46 --- /dev/null +++ b/backend/tools/load_public_dissem_data/load_public_data_locally.sh @@ -0,0 +1,126 @@ +#!/bin/bash + +export CGOV_UTIL_VERSION=v0.1.8 +export FAC_DB_URI=postgresql://postgres@db:5432/postgres?sslmode=disable +export FAC_SNAPSHOT_URI=postgresql://postgres@db2:5432/postgres?sslmode=disable + + +function check_table_exists() { + local db_uri="$1" + local dbname="$2" + psql $db_uri -c "SELECT '$dbname'::regclass" >/dev/null 2>&1 + result=$? + return $result +} + +echo "This will unzip ~3.3GB of data, and load it into a local FAC." +echo "Make sure the FAC is running." +echo "Make sure you have disk space." +echo "Sleeping for 4 seconds..." +sleep 4 + +cat << EOM +First, we cleanup the local filesystem. +This removes any temporary files from any +previous data loads +EOM + +rm -f /app/data/db_dissem_dump +rm -rf /app/data/__MACOSX + +cat << EOM +Next, we drop the public_data schema. +This is because we want to make sure it is +regenerated fresh. +EOM + +psql $FAC_SNAPSHOT_URI -c "DROP SCHEMA IF EXISTS public_data_v1_0_0 CASCADE" + +cat << EOM +Now, the schema for the public_data is +created. This provies a place for the tables to +land when we run sling +EOM + +psql $FAC_SNAPSHOT_URI -c "CREATE SCHEMA IF NOT EXISTS public_data_v1_0_0" +psql $FAC_SNAPSHOT_URI -c "CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_combined START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE" + +echo </dev/null 2>&1 + result=$? + return $result +} + +echo "This will unzip ~3.3GB of data, and load it into a local FAC." +echo "Make sure the FAC is running." +echo "Make sure you have disk space." +echo "Sleeping for 4 seconds..." +sleep 4 + +echo < Date: Sun, 29 Sep 2024 16:49:57 -0400 Subject: [PATCH 04/89] Updating ignores... --- backend/tools/load_public_dissem_data/.gitignore | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 backend/tools/load_public_dissem_data/.gitignore diff --git a/backend/tools/load_public_dissem_data/.gitignore b/backend/tools/load_public_dissem_data/.gitignore new file mode 100644 index 0000000000..1afc10f9b1 --- /dev/null +++ b/backend/tools/load_public_dissem_data/.gitignore @@ -0,0 +1,6 @@ +*.csv +*.zip +*.tar +*.gz +*.json +db_dissem_dump* From 7e54577898420a1c3aaee5dae7a6d44bc340c031 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Mon, 30 Sep 2024 09:36:02 -0400 Subject: [PATCH 05/89] Runs a ffull sequence This runs a full install sequence. Ready for testing on preview. --- backend/Makefile | 40 +- backend/dissemination/api/api/drop_schema.sql | 10 - .../api/api_historic_v0_1_0_alpha/base.sql | 5 - .../create_functions.sql | 5 - .../create_schema.sql | 42 -- .../create_views.sql | 5 - .../api_historic_v0_1_0_alpha/drop_schema.sql | 5 - .../api_historic_v0_1_0_alpha/drop_views.sql | 5 - .../api/api_historic_v0_1_0_alpha/views.py | 49 -- backend/dissemination/api/api_v1_0_3/base.sql | 29 - .../api/api_v1_0_3/create_functions.sql | 24 - .../api/api_v1_0_3/create_schema.sql | 52 -- .../api/api_v1_0_3/create_views.sql | 335 ------------ .../api/api_v1_0_3/drop_schema.sql | 11 - .../api/api_v1_0_3/drop_views.sql | 17 - backend/dissemination/api/api_v1_1_0/base.sql | 29 - .../api/api_v1_1_0/drop_schema.sql | 11 - .../api/api_v1_1_0/drop_views.sql | 17 - .../dissemination/api/api_v1_1_0/teardown.sql | 1 - backend/dissemination/api/api_v1_1_1/base.sql | 29 - .../api/api_v1_1_1/create_functions.sql | 106 ---- .../api/api_v1_1_1/create_schema.sql | 48 -- .../api/api_v1_1_1/create_views.sql | 410 -------------- .../api/api_v1_1_1/drop_schema.sql | 11 - .../api/api_v1_1_1/drop_views.sql | 18 - .../sql/api/api_v1_0_3/standup.sql | 106 ++-- .../sql/api/api_v1_0_3/teardown.sql | 32 +- .../api/public_api_v2_0_0_alpha/public.rest | 27 + .../api/public_api_v2_0_0_alpha/standup.sql | 124 +---- .../api/public_api_v2_0_0_alpha/teardown.sql | 2 + .../fac-snapshot-db/additional_eins.sql | 8 + .../fac-snapshot-db/additional_ueis.sql | 8 + .../sql/indexes/fac-snapshot-db/combined.sql | 75 +++ .../corrective_action_plans.sql | 11 + .../fac-snapshot-db/federal_awards.sql | 29 + .../sql/indexes/fac-snapshot-db/findings.sql | 16 + .../indexes/fac-snapshot-db/findings_text.sql | 8 + .../sql/indexes/fac-snapshot-db/general.sql | 53 ++ .../indexes/fac-snapshot-db/notes_to_sefa.sql | 8 + .../indexes/fac-snapshot-db/passthrough.sql | 8 + .../fac-snapshot-db/secondary_auditors.sql | 8 + .../public_data_v1_0_0.yaml | 106 ++-- .../sql/sling/public_data_v1_0_0/standup.sql | 6 - .../sql/sling/public_data_v1_0_0/teardown.sql | 78 --- backend/docker-compose.yml | 25 +- backend/run.sh | 29 +- backend/tools/api_standup.sh | 42 +- backend/tools/api_teardown.sh | 68 ++- backend/tools/build_indexes.sh | 29 + .../tools/load_public_dissem_data/.gitignore | 2 +- .../tools/load_public_dissem_data/Dockerfile | 10 +- .../tools/load_public_dissem_data/Makefile | 8 + .../tools/load_public_dissem_data/README.md | 67 ++- .../load_public_dissem_data/data/config.json | 90 ++++ .../data/public_data_v1_0_0.yaml | 499 ++++++++++++++++++ .../load_public_dissem_data/data/sling.yaml | 499 ++++++++++++++++++ .../load_public_data_locally.sh | 94 +--- backend/tools/setup_local_env.sh | 2 +- backend/tools/variables.sh | 45 ++ 59 files changed, 1769 insertions(+), 1767 deletions(-) delete mode 100644 backend/dissemination/api/api/drop_schema.sql delete mode 100644 backend/dissemination/api/api_historic_v0_1_0_alpha/base.sql delete mode 100644 backend/dissemination/api/api_historic_v0_1_0_alpha/create_functions.sql delete mode 100644 backend/dissemination/api/api_historic_v0_1_0_alpha/create_schema.sql delete mode 100644 backend/dissemination/api/api_historic_v0_1_0_alpha/create_views.sql delete mode 100644 backend/dissemination/api/api_historic_v0_1_0_alpha/drop_schema.sql delete mode 100644 backend/dissemination/api/api_historic_v0_1_0_alpha/drop_views.sql delete mode 100644 backend/dissemination/api/api_historic_v0_1_0_alpha/views.py delete mode 100644 backend/dissemination/api/api_v1_0_3/base.sql delete mode 100644 backend/dissemination/api/api_v1_0_3/create_functions.sql delete mode 100644 backend/dissemination/api/api_v1_0_3/create_schema.sql delete mode 100644 backend/dissemination/api/api_v1_0_3/create_views.sql delete mode 100644 backend/dissemination/api/api_v1_0_3/drop_schema.sql delete mode 100644 backend/dissemination/api/api_v1_0_3/drop_views.sql delete mode 100644 backend/dissemination/api/api_v1_1_0/base.sql delete mode 100644 backend/dissemination/api/api_v1_1_0/drop_schema.sql delete mode 100644 backend/dissemination/api/api_v1_1_0/drop_views.sql delete mode 100644 backend/dissemination/api/api_v1_1_0/teardown.sql delete mode 100644 backend/dissemination/api/api_v1_1_1/base.sql delete mode 100644 backend/dissemination/api/api_v1_1_1/create_functions.sql delete mode 100644 backend/dissemination/api/api_v1_1_1/create_schema.sql delete mode 100644 backend/dissemination/api/api_v1_1_1/create_views.sql delete mode 100644 backend/dissemination/api/api_v1_1_1/drop_schema.sql delete mode 100644 backend/dissemination/api/api_v1_1_1/drop_views.sql create mode 100644 backend/dissemination/sql/api/public_api_v2_0_0_alpha/public.rest create mode 100644 backend/dissemination/sql/indexes/fac-snapshot-db/additional_eins.sql create mode 100644 backend/dissemination/sql/indexes/fac-snapshot-db/additional_ueis.sql create mode 100644 backend/dissemination/sql/indexes/fac-snapshot-db/combined.sql create mode 100644 backend/dissemination/sql/indexes/fac-snapshot-db/corrective_action_plans.sql create mode 100644 backend/dissemination/sql/indexes/fac-snapshot-db/federal_awards.sql create mode 100644 backend/dissemination/sql/indexes/fac-snapshot-db/findings.sql create mode 100644 backend/dissemination/sql/indexes/fac-snapshot-db/findings_text.sql create mode 100644 backend/dissemination/sql/indexes/fac-snapshot-db/general.sql create mode 100644 backend/dissemination/sql/indexes/fac-snapshot-db/notes_to_sefa.sql create mode 100644 backend/dissemination/sql/indexes/fac-snapshot-db/passthrough.sql create mode 100644 backend/dissemination/sql/indexes/fac-snapshot-db/secondary_auditors.sql delete mode 100644 backend/dissemination/sql/sling/public_data_v1_0_0/standup.sql delete mode 100644 backend/dissemination/sql/sling/public_data_v1_0_0/teardown.sql create mode 100644 backend/tools/build_indexes.sh create mode 100644 backend/tools/load_public_dissem_data/Makefile create mode 100644 backend/tools/load_public_dissem_data/data/config.json create mode 100644 backend/tools/load_public_dissem_data/data/public_data_v1_0_0.yaml create mode 100644 backend/tools/load_public_dissem_data/data/sling.yaml create mode 100644 backend/tools/variables.sh diff --git a/backend/Makefile b/backend/Makefile index 8e78d3466d..a2d795507e 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -88,9 +88,9 @@ docker-lint: docker compose run web bash -c 'flake8 && black --check . && bandit -c pyproject.toml -r . && mypy . && djlint .' ghcr-first-run: - docker compose -f docker-compose-web.yml run web python manage.py makemigrations - docker compose -f docker-compose-web.yml run web python manage.py migrate - docker compose -f docker-compose-web.yml run web python manage.py loaddata + -docker compose -f docker-compose-web.yml run web python manage.py makemigrations + -docker compose -f docker-compose-web.yml run web python manage.py migrate + -docker compose -f docker-compose-web.yml run web python manage.py loaddata # Run Django tests with docker ghcr-test: @@ -110,14 +110,32 @@ ghcr-lint: docker compose -f docker-compose-web.yml run web bash -c 'flake8 && black --check . && bandit -c pyproject.toml -r . && python -m pip install types-pytz && mypy . && djlint .' docker-clean: - docker compose down - docker rm -f $(shell docker ps -a -q) - docker volume rm $(shell docker volume ls -q) + -docker compose down + -docker rm -f $(shell docker ps -a -q) + -docker volume rm $(shell docker volume ls -q) + +remove-coverage: + -rm -f .coverage.* -docker-full-clean: - rm -f .coverage.* +compose-down: docker compose down + +remove-processes: docker rm -f $(docker ps -a -q) - docker volume rm $(docker volume ls -q) - docker system prune -f - docker volume prune -f + +remove-volumes: + -docker volume ls -q | docker volume rm + +remove-dangling-volumes: remove-volumes + -docker volume ls -q --filter dangling=true | docker volume rm + +system-prune: + -docker system prune -f + +volume-prune: + -docker volume prune -f + +# Proceed past errors. +# We want all of these commands to run, even if one fails. +# make -i docker-full-clean +docker-full-clean: remove-coverage compose-down remove-processes remove-dangling-volumes system-prune volume-prune diff --git a/backend/dissemination/api/api/drop_schema.sql b/backend/dissemination/api/api/drop_schema.sql deleted file mode 100644 index 4613e1c1e4..0000000000 --- a/backend/dissemination/api/api/drop_schema.sql +++ /dev/null @@ -1,10 +0,0 @@ - -begin; - -DROP SCHEMA IF EXISTS api CASCADE; - -commit; - -notify pgrst, - 'reload schema'; - diff --git a/backend/dissemination/api/api_historic_v0_1_0_alpha/base.sql b/backend/dissemination/api/api_historic_v0_1_0_alpha/base.sql deleted file mode 100644 index 37e9d7347f..0000000000 --- a/backend/dissemination/api/api_historic_v0_1_0_alpha/base.sql +++ /dev/null @@ -1,5 +0,0 @@ -begin; -select 1; -commit; - -notify pgrst, 'reload schema'; diff --git a/backend/dissemination/api/api_historic_v0_1_0_alpha/create_functions.sql b/backend/dissemination/api/api_historic_v0_1_0_alpha/create_functions.sql deleted file mode 100644 index 37e9d7347f..0000000000 --- a/backend/dissemination/api/api_historic_v0_1_0_alpha/create_functions.sql +++ /dev/null @@ -1,5 +0,0 @@ -begin; -select 1; -commit; - -notify pgrst, 'reload schema'; diff --git a/backend/dissemination/api/api_historic_v0_1_0_alpha/create_schema.sql b/backend/dissemination/api/api_historic_v0_1_0_alpha/create_schema.sql deleted file mode 100644 index 127b6b4319..0000000000 --- a/backend/dissemination/api/api_historic_v0_1_0_alpha/create_schema.sql +++ /dev/null @@ -1,42 +0,0 @@ --- This schema is handled external to the app. --- Why? --- It relies on static tables that are loaded before the app exists. --- Therefore, we assume those tables are loaded. Or, mostly assume. --- This grants permissions, nothing more. - -begin; - -do -$$ -begin - -- If it exists, grant permissions. - if exists (select schema_name from information_schema.schemata where schema_name = 'api_historic_v0_1_0_alpha') then - -- Grant access to tables and views - alter default privileges - in schema api_historic_v0_1_0_alpha - grant select - -- this includes views - on tables - to api_fac_gov; - - -- Grant access to sequences, if we have them - grant usage on schema api_historic_v0_1_0_alpha to api_fac_gov; - grant select, usage on all sequences in schema api_historic_v0_1_0_alpha to api_fac_gov; - alter default privileges - in schema api_historic_v0_1_0_alpha - grant select, usage - on sequences - to api_fac_gov; - - GRANT SELECT ON ALL TABLES IN SCHEMA api_historic_v0_1_0_alpha TO api_fac_gov; - end if; -end -$$ -; - -select 1; - -commit; - -notify pgrst, 'reload schema'; - diff --git a/backend/dissemination/api/api_historic_v0_1_0_alpha/create_views.sql b/backend/dissemination/api/api_historic_v0_1_0_alpha/create_views.sql deleted file mode 100644 index 37e9d7347f..0000000000 --- a/backend/dissemination/api/api_historic_v0_1_0_alpha/create_views.sql +++ /dev/null @@ -1,5 +0,0 @@ -begin; -select 1; -commit; - -notify pgrst, 'reload schema'; diff --git a/backend/dissemination/api/api_historic_v0_1_0_alpha/drop_schema.sql b/backend/dissemination/api/api_historic_v0_1_0_alpha/drop_schema.sql deleted file mode 100644 index 37e9d7347f..0000000000 --- a/backend/dissemination/api/api_historic_v0_1_0_alpha/drop_schema.sql +++ /dev/null @@ -1,5 +0,0 @@ -begin; -select 1; -commit; - -notify pgrst, 'reload schema'; diff --git a/backend/dissemination/api/api_historic_v0_1_0_alpha/drop_views.sql b/backend/dissemination/api/api_historic_v0_1_0_alpha/drop_views.sql deleted file mode 100644 index 37e9d7347f..0000000000 --- a/backend/dissemination/api/api_historic_v0_1_0_alpha/drop_views.sql +++ /dev/null @@ -1,5 +0,0 @@ -begin; -select 1; -commit; - -notify pgrst, 'reload schema'; diff --git a/backend/dissemination/api/api_historic_v0_1_0_alpha/views.py b/backend/dissemination/api/api_historic_v0_1_0_alpha/views.py deleted file mode 100644 index 993dd3be76..0000000000 --- a/backend/dissemination/api/api_historic_v0_1_0_alpha/views.py +++ /dev/null @@ -1,49 +0,0 @@ -schema = "api_historic_v0_1_0_alpha" -prefix = "census_" - -tables = { - "agency": (16, 22), - "captext": (19, 22), - "captext_formatted": (19, 22), - "cfda": (16, 22), - "cpas": (16, 22), - "duns": (16, 22), - "eins": (16, 22), - "findings": (16, 22), - "findingstext": (19, 22), - "findingstext_formatted": (19, 22), - "gen": (16, 22), - "notes": (19, 22), - "passthrough": (16, 22), - "revisions": (19, 22), - "ueis": (22, 22), -} - - -def just_table_names(lot): - return list(tables.keys()) - - -def generate_views(tbs): - print("begin;\n") - for t, rng in tbs.items(): - # Range is exclusive on the second value - for v in range(rng[0], rng[1] + 1): - print(f"create view {schema}.{t}{v} as") - print("\tselect *") - print(f"\tfrom {prefix}{t}{v}") - print(f"\torder by {prefix}{t}{v}.id") - print(";\n") - print("commit;") - print("notify pgrst, 'reload schema';") - - -if __name__ in "__main__": - generate_views(tables) - -# (define (generate-drops lot) -# (printf "begin;~n~n") -# (for ([t lot]) -# (printf "drop table if exists ~a.~a;~n" schema t)) -# (printf "commit;~n") -# (printf "notify pgrst, 'reload schema';~n")) diff --git a/backend/dissemination/api/api_v1_0_3/base.sql b/backend/dissemination/api/api_v1_0_3/base.sql deleted file mode 100644 index dedabe0cb7..0000000000 --- a/backend/dissemination/api/api_v1_0_3/base.sql +++ /dev/null @@ -1,29 +0,0 @@ -DO -$do$ -BEGIN - IF EXISTS ( - SELECT FROM pg_catalog.pg_roles - WHERE rolname = 'authenticator') THEN - RAISE NOTICE 'Role "authenticator" already exists. Skipping.'; - ELSE - CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; - END IF; -END -$do$; - -DO -$do$ -BEGIN - IF EXISTS ( - SELECT FROM pg_catalog.pg_roles - WHERE rolname = 'api_fac_gov') THEN - RAISE NOTICE 'Role "api_fac_gov" already exists. Skipping.'; - ELSE - CREATE ROLE api_fac_gov NOLOGIN; - END IF; -END -$do$; - -GRANT api_fac_gov TO authenticator; - -NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/api/api_v1_0_3/create_functions.sql b/backend/dissemination/api/api_v1_0_3/create_functions.sql deleted file mode 100644 index 62d2b400e7..0000000000 --- a/backend/dissemination/api/api_v1_0_3/create_functions.sql +++ /dev/null @@ -1,24 +0,0 @@ --- WARNING --- Under PostgreSQL 12, the functions below work. --- Under PostgreSQL 14, these will break. --- --- Note the differences: --- --- raise info 'Works under PostgreSQL 12'; --- raise info 'request.header.x-magic %', (SELECT current_setting('request.header.x-magic', true)); --- raise info 'request.jwt.claim.expires %', (SELECT current_setting('request.jwt.claim.expires', true)); --- raise info 'Works under PostgreSQL 14'; --- raise info 'request.headers::json->>x-magic %', (SELECT current_setting('request.headers', true)::json->>'x-magic'); --- raise info 'request.jwt.claims::json->expires %', (SELECT current_setting('request.jwt.claims', true)::json->>'expires'); --- --- To quote the work of Dav Pilkey, "remember this now." - --- We don't grant tribal access (yet) -create or replace function api_v1_0_3_functions.has_tribal_data_access() returns boolean -as $has_tribal_data_access$ -BEGIN - RETURN 0::BOOLEAN; -END; -$has_tribal_data_access$ LANGUAGE plpgsql; - -NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/api/api_v1_0_3/create_schema.sql b/backend/dissemination/api/api_v1_0_3/create_schema.sql deleted file mode 100644 index 089e746f2f..0000000000 --- a/backend/dissemination/api/api_v1_0_3/create_schema.sql +++ /dev/null @@ -1,52 +0,0 @@ -begin; - -do -$$ -begin - DROP SCHEMA IF EXISTS api_v1_0_3 CASCADE; - DROP SCHEMA IF EXISTS api_v1_0_3_functions CASCADE; - - if not exists (select schema_name from information_schema.schemata where schema_name = 'api_v1_0_3') then - create schema api_v1_0_3; - create schema api_v1_0_3_functions; - - grant usage on schema api_v1_0_3_functions to api_fac_gov; - - -- Grant access to tables and views - alter default privileges - in schema api_v1_0_3 - grant select - -- this includes views - on tables - to api_fac_gov; - - -- Grant access to sequences, if we have them - grant usage on schema api_v1_0_3 to api_fac_gov; - grant select, usage on all sequences in schema api_v1_0_3 to api_fac_gov; - alter default privileges - in schema api_v1_0_3 - grant select, usage - on sequences - to api_fac_gov; - end if; -end -$$ -; - --- This is the description -COMMENT ON SCHEMA api_v1_0_3 IS - 'The FAC dissemation API version 1.0.3.' -; - --- https://postgrest.org/en/stable/references/api/openapi.html --- This is the title -COMMENT ON SCHEMA api_v1_0_3 IS -$$v1.0.3 - -A RESTful API that serves data from the SF-SAC.$$; - -commit; - -notify pgrst, - 'reload schema'; - diff --git a/backend/dissemination/api/api_v1_0_3/create_views.sql b/backend/dissemination/api/api_v1_0_3/create_views.sql deleted file mode 100644 index cbaf47d169..0000000000 --- a/backend/dissemination/api/api_v1_0_3/create_views.sql +++ /dev/null @@ -1,335 +0,0 @@ -begin; - ---------------------------------------- --- finding_text ---------------------------------------- -create view api_v1_0_3.findings_text as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - ft.finding_ref_number, - ft.contains_chart_or_table, - ft.finding_text - from - dissemination_findingtext ft, - dissemination_general gen - where - ft.report_id = gen.report_id - and - (gen.is_public = true - or - (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) - order by ft.id -; - ---------------------------------------- --- additional_ueis ---------------------------------------- -create view api_v1_0_3.additional_ueis as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - uei.additional_uei - from - dissemination_general gen, - dissemination_additionaluei uei - where - gen.report_id = uei.report_id - and - (gen.is_public = true - or - (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) - order by uei.id -; - ---------------------------------------- --- finding ---------------------------------------- -create view api_v1_0_3.findings as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - finding.award_reference, - finding.reference_number, - finding.is_material_weakness, - finding.is_modified_opinion, - finding.is_other_findings, - finding.is_other_matters, - finding.prior_finding_ref_numbers, - finding.is_questioned_costs, - finding.is_repeat_finding, - finding.is_significant_deficiency, - finding.type_requirement - from - dissemination_finding finding, - dissemination_general gen - where - finding.report_id = gen.report_id - and - (gen.is_public = true - or - (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) - order by finding.id -; - ---------------------------------------- --- federal award ---------------------------------------- -create view api_v1_0_3.federal_awards as - select - award.report_id, - gen.auditee_uei, - gen.audit_year, - --- - award.award_reference, - award.federal_agency_prefix, - award.federal_award_extension, - award.additional_award_identification, - award.federal_program_name, - award.amount_expended, - award.cluster_name, - award.other_cluster_name, - award.state_cluster_name, - award.cluster_total, - award.federal_program_total, - award.is_major, - award.is_loan, - award.loan_balance, - award.is_direct, - award.audit_report_type, - award.findings_count, - award.is_passthrough_award, - award.passthrough_amount - from - dissemination_federalaward award, - dissemination_general gen - where - award.report_id = gen.report_id - and - (gen.is_public = true - or - (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) - order by award.id -; - - ---------------------------------------- --- corrective_action_plan ---------------------------------------- -create view api_v1_0_3.corrective_action_plans as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - ct.finding_ref_number, - ct.contains_chart_or_table, - ct.planned_action - from - dissemination_CAPText ct, - dissemination_General gen - where - ct.report_id = gen.report_id - and - (gen.is_public = true - or - (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) - order by ct.id -; - ---------------------------------------- --- notes_to_sefa ---------------------------------------- -create view api_v1_0_3.notes_to_sefa as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - note.note_title as title, - note.accounting_policies, - note.is_minimis_rate_used, - note.rate_explained, - note.content, - note.contains_chart_or_table - from - dissemination_general gen, - dissemination_note note - where - note.report_id = gen.report_id - and - (gen.is_public = true - or - (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) - order by note.id -; - ---------------------------------------- --- passthrough ---------------------------------------- -create view api_v1_0_3.passthrough as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - pass.award_reference, - pass.passthrough_id, - pass.passthrough_name - from - dissemination_general as gen, - dissemination_passthrough as pass - where - gen.report_id = pass.report_id - and - (gen.is_public = true - or - (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) - order by pass.id -; - - ---------------------------------------- --- general ---------------------------------------- -create view api_v1_0_3.general as - select - -- every table starts with report_id, UEI, and year - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - gen.auditee_certify_name, - gen.auditee_certify_title, - gen.auditee_contact_name, - gen.auditee_email, - gen.auditee_name, - gen.auditee_phone, - gen.auditee_contact_title, - gen.auditee_address_line_1, - gen.auditee_city, - gen.auditee_state, - gen.auditee_ein, - gen.auditee_zip, - -- auditor - gen.auditor_phone, - gen.auditor_state, - gen.auditor_city, - gen.auditor_contact_title, - gen.auditor_address_line_1, - gen.auditor_zip, - gen.auditor_country, - gen.auditor_contact_name, - gen.auditor_email, - gen.auditor_firm_name, - gen.auditor_foreign_address, - gen.auditor_ein, - -- agency - gen.cognizant_agency, - gen.oversight_agency, - -- dates - gen.date_created, - gen.ready_for_certification_date, - gen.auditor_certified_date, - gen.auditee_certified_date, - gen.submitted_date, - gen.fac_accepted_date, - gen.fy_end_date, - gen.fy_start_date, - gen.audit_type, - gen.gaap_results, - gen.sp_framework_basis, - gen.is_sp_framework_required, - gen.sp_framework_opinions, - gen.is_going_concern_included, - gen.is_internal_control_deficiency_disclosed, - gen.is_internal_control_material_weakness_disclosed, - gen.is_material_noncompliance_disclosed, - gen.dollar_threshold, - gen.is_low_risk_auditee, - gen.agencies_with_prior_findings, - gen.entity_type, - gen.number_months, - gen.audit_period_covered, - gen.total_amount_expended, - gen.type_audit_code, - gen.is_public, - gen.data_source, - gen.is_aicpa_audit_guide_included, - gen.is_additional_ueis, - CASE EXISTS(SELECT ein.report_id FROM dissemination_additionalein ein WHERE ein.report_id = gen.report_id) - WHEN FALSE THEN 'No' - ELSE 'Yes' - END AS is_multiple_eins, - CASE EXISTS(SELECT aud.report_id FROM dissemination_secondaryauditor aud WHERE aud.report_id = gen.report_id) - WHEN FALSE THEN 'No' - ELSE 'Yes' - END AS is_secondary_auditors - from - dissemination_general gen - where - gen.is_public = true - or - (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access()) - order by gen.id -; - ---------------------------------------- --- auditor (secondary auditor) ---------------------------------------- -create view api_v1_0_3.secondary_auditors as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - sa.auditor_ein, - sa.auditor_name, - sa.contact_name, - sa.contact_title, - sa.contact_email, - sa.contact_phone, - sa.address_street, - sa.address_city, - sa.address_state, - sa.address_zipcode - from - dissemination_General gen, - dissemination_SecondaryAuditor sa - where - sa.report_id = gen.report_id - and - (gen.is_public=True - or - (gen.is_public=false and api_v1_0_3_functions.has_tribal_data_access())) - order by sa.id -; - -create view api_v1_0_3.additional_eins as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - ein.additional_ein - from - dissemination_general gen, - dissemination_additionalein ein - where - gen.report_id = ein.report_id - and - (gen.is_public = true - or - (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) - order by ein.id -; - -commit; - -notify pgrst, - 'reload schema'; - diff --git a/backend/dissemination/api/api_v1_0_3/drop_schema.sql b/backend/dissemination/api/api_v1_0_3/drop_schema.sql deleted file mode 100644 index cf1aca6d91..0000000000 --- a/backend/dissemination/api/api_v1_0_3/drop_schema.sql +++ /dev/null @@ -1,11 +0,0 @@ - -begin; - -DROP SCHEMA IF EXISTS api_v1_0_3 CASCADE; --- DROP ROLE IF EXISTS authenticator; --- DROP ROLE IF EXISTS api_fac_gov; - -commit; - -notify pgrst, - 'reload schema'; diff --git a/backend/dissemination/api/api_v1_0_3/drop_views.sql b/backend/dissemination/api/api_v1_0_3/drop_views.sql deleted file mode 100644 index 2775087977..0000000000 --- a/backend/dissemination/api/api_v1_0_3/drop_views.sql +++ /dev/null @@ -1,17 +0,0 @@ -begin; - drop table if exists api_v1_0_3.metadata; - drop view if exists api_v1_0_3.general; - drop view if exists api_v1_0_3.auditor; - drop view if exists api_v1_0_3.federal_awards; - drop view if exists api_v1_0_3.findings; - drop view if exists api_v1_0_3.findings_text; - drop view if exists api_v1_0_3.corrective_action_plans; - drop view if exists api_v1_0_3.additional_ueis; - drop view if exists api_v1_0_3.notes_to_sefa; - drop view if exists api_v1_0_3.passthrough; - drop view if exists api_v1_0_3.secondary_auditors; - drop view if exists api_v1_0_3.additional_eins; -commit; - -notify pgrst, - 'reload schema'; diff --git a/backend/dissemination/api/api_v1_1_0/base.sql b/backend/dissemination/api/api_v1_1_0/base.sql deleted file mode 100644 index dedabe0cb7..0000000000 --- a/backend/dissemination/api/api_v1_1_0/base.sql +++ /dev/null @@ -1,29 +0,0 @@ -DO -$do$ -BEGIN - IF EXISTS ( - SELECT FROM pg_catalog.pg_roles - WHERE rolname = 'authenticator') THEN - RAISE NOTICE 'Role "authenticator" already exists. Skipping.'; - ELSE - CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; - END IF; -END -$do$; - -DO -$do$ -BEGIN - IF EXISTS ( - SELECT FROM pg_catalog.pg_roles - WHERE rolname = 'api_fac_gov') THEN - RAISE NOTICE 'Role "api_fac_gov" already exists. Skipping.'; - ELSE - CREATE ROLE api_fac_gov NOLOGIN; - END IF; -END -$do$; - -GRANT api_fac_gov TO authenticator; - -NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/api/api_v1_1_0/drop_schema.sql b/backend/dissemination/api/api_v1_1_0/drop_schema.sql deleted file mode 100644 index e32038ee46..0000000000 --- a/backend/dissemination/api/api_v1_1_0/drop_schema.sql +++ /dev/null @@ -1,11 +0,0 @@ - -begin; - -DROP SCHEMA IF EXISTS api_v1_1_0 CASCADE; --- DROP ROLE IF EXISTS authenticator; --- DROP ROLE IF EXISTS api_fac_gov; - -commit; - -notify pgrst, - 'reload schema'; diff --git a/backend/dissemination/api/api_v1_1_0/drop_views.sql b/backend/dissemination/api/api_v1_1_0/drop_views.sql deleted file mode 100644 index 877d5920c6..0000000000 --- a/backend/dissemination/api/api_v1_1_0/drop_views.sql +++ /dev/null @@ -1,17 +0,0 @@ -begin; - drop table if exists api_v1_1_0.metadata; - drop view if exists api_v1_1_0.general; - drop view if exists api_v1_1_0.auditor; - drop view if exists api_v1_1_0.federal_awards; - drop view if exists api_v1_1_0.findings; - drop view if exists api_v1_1_0.findings_text; - drop view if exists api_v1_1_0.corrective_action_plans; - drop view if exists api_v1_1_0.additional_ueis; - drop view if exists api_v1_1_0.notes_to_sefa; - drop view if exists api_v1_1_0.passthrough; - drop view if exists api_v1_1_0.secondary_auditors; - drop view if exists api_v1_1_0.additional_eins; -commit; - -notify pgrst, - 'reload schema'; diff --git a/backend/dissemination/api/api_v1_1_0/teardown.sql b/backend/dissemination/api/api_v1_1_0/teardown.sql deleted file mode 100644 index 99b1d27eb9..0000000000 --- a/backend/dissemination/api/api_v1_1_0/teardown.sql +++ /dev/null @@ -1 +0,0 @@ -\i drop_schema.sql diff --git a/backend/dissemination/api/api_v1_1_1/base.sql b/backend/dissemination/api/api_v1_1_1/base.sql deleted file mode 100644 index dedabe0cb7..0000000000 --- a/backend/dissemination/api/api_v1_1_1/base.sql +++ /dev/null @@ -1,29 +0,0 @@ -DO -$do$ -BEGIN - IF EXISTS ( - SELECT FROM pg_catalog.pg_roles - WHERE rolname = 'authenticator') THEN - RAISE NOTICE 'Role "authenticator" already exists. Skipping.'; - ELSE - CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; - END IF; -END -$do$; - -DO -$do$ -BEGIN - IF EXISTS ( - SELECT FROM pg_catalog.pg_roles - WHERE rolname = 'api_fac_gov') THEN - RAISE NOTICE 'Role "api_fac_gov" already exists. Skipping.'; - ELSE - CREATE ROLE api_fac_gov NOLOGIN; - END IF; -END -$do$; - -GRANT api_fac_gov TO authenticator; - -NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/api/api_v1_1_1/create_functions.sql b/backend/dissemination/api/api_v1_1_1/create_functions.sql deleted file mode 100644 index 87114cd660..0000000000 --- a/backend/dissemination/api/api_v1_1_1/create_functions.sql +++ /dev/null @@ -1,106 +0,0 @@ --- WARNING --- Under PostgreSQL 12, the functions below work. --- Under PostgreSQL 14, these will break. --- --- Note the differences: --- --- raise info 'Works under PostgreSQL 12'; --- raise info 'request.header.x-magic %', (SELECT current_setting('request.header.x-magic', true)); --- raise info 'request.jwt.claim.expires %', (SELECT current_setting('request.jwt.claim.expires', true)); --- raise info 'Works under PostgreSQL 14'; --- raise info 'request.headers::json->>x-magic %', (SELECT current_setting('request.headers', true)::json->>'x-magic'); --- raise info 'request.jwt.claims::json->expires %', (SELECT current_setting('request.jwt.claims', true)::json->>'expires'); --- --- To quote the work of Dav Pilkey, "remember this now." - - -CREATE OR REPLACE FUNCTION api_v1_1_1_functions.get_header(item text) RETURNS text - AS $get_header$ - declare res text; - begin - SELECT (current_setting('request.headers', true)::json)->>item into res; - return res; - end; -$get_header$ LANGUAGE plpgsql; - -create or replace function api_v1_1_1_functions.get_api_key_uuid() returns TEXT -as $gaku$ -declare uuid text; -begin - select api_v1_1_1_functions.get_header('x-api-user-id') into uuid; - return uuid; -end; -$gaku$ LANGUAGE plpgsql; - -create or replace function api_v1_1_1_functions.has_tribal_data_access() -returns boolean -as $has_tribal_data_access$ -DECLARE - uuid_header UUID; - key_exists boolean; -BEGIN - - SELECT api_v1_1_1_functions.get_api_key_uuid() INTO uuid_header; - SELECT - CASE WHEN EXISTS ( - SELECT key_id - FROM public.dissemination_TribalApiAccessKeyIds taaki - WHERE taaki.key_id = uuid_header::TEXT) - THEN 1::BOOLEAN - ELSE 0::BOOLEAN - END - INTO key_exists; - RAISE INFO 'api_v1_1_1 has_tribal % %', uuid_header, key_exists; - RETURN key_exists; -END; -$has_tribal_data_access$ LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION api_v1_1_1.request_file_access( - report_id TEXT -) RETURNS JSON LANGUAGE plpgsql AS -$$ -DECLARE - v_uuid_header TEXT; - v_access_uuid VARCHAR(200); - v_key_exists BOOLEAN; - v_key_added_date DATE; -BEGIN - - SELECT api_v1_1_1_functions.get_api_key_uuid() INTO v_uuid_header; - - -- Check if the provided API key exists in public.dissemination_TribalApiAccessKeyIds - SELECT - EXISTS( - SELECT 1 - FROM public.dissemination_TribalApiAccessKeyIds - WHERE key_id = v_uuid_header - ) INTO v_key_exists; - - - -- Get the added date of the key from public.dissemination_TribalApiAccessKeyIds - SELECT date_added - INTO v_key_added_date - FROM public.dissemination_TribalApiAccessKeyIds - WHERE key_id = v_uuid_header; - - - -- Check if the key is less than 6 months old - IF v_uuid_header IS NOT NULL AND v_key_exists AND v_key_added_date >= CURRENT_DATE - INTERVAL '6 months' THEN - -- Generate UUID (using PostgreSQL's gen_random_uuid function) - SELECT gen_random_uuid() INTO v_access_uuid; - - -- Inserting data into the one_time_access table - INSERT INTO public.dissemination_onetimeaccess (uuid, api_key_id, timestamp, report_id) - VALUES (v_access_uuid::UUID, v_uuid_header, CURRENT_TIMESTAMP, report_id); - - -- Return the UUID to the user - RETURN json_build_object('access_uuid', v_access_uuid); - ELSE - -- Return an error for unauthorized access - RETURN json_build_object('error', 'Unauthorized access or key older than 6 months')::JSON; - END IF; -END; -$$; - -NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/api/api_v1_1_1/create_schema.sql b/backend/dissemination/api/api_v1_1_1/create_schema.sql deleted file mode 100644 index 2b6a58d553..0000000000 --- a/backend/dissemination/api/api_v1_1_1/create_schema.sql +++ /dev/null @@ -1,48 +0,0 @@ -begin; - -do -$$ -begin - DROP SCHEMA IF EXISTS api_v1_1_1 CASCADE; - DROP SCHEMA IF EXISTS api_v1_1_1_functions CASCADE; - - if not exists (select schema_name from information_schema.schemata where schema_name = 'api_v1_1_1') then - create schema api_v1_1_1; - create schema api_v1_1_1_functions; - - grant usage on schema api_v1_1_1_functions to api_fac_gov; - - -- Grant access to tables and views - alter default privileges - in schema api_v1_1_1 - grant select - -- this includes views - on tables - to api_fac_gov; - - -- Grant access to sequences, if we have them - grant usage on schema api_v1_1_1 to api_fac_gov; - grant select, usage on all sequences in schema api_v1_1_1 to api_fac_gov; - alter default privileges - in schema api_v1_1_1 - grant select, usage - on sequences - to api_fac_gov; - end if; -end -$$ -; - --- https://postgrest.org/en/stable/references/api/openapi.html --- This is the title (version number) and description (text). -COMMENT ON SCHEMA api_v1_1_1 IS -$$v1.1.1 - -A RESTful API that serves data from the SF-SAC.$$; - - -commit; - -notify pgrst, - 'reload schema'; - diff --git a/backend/dissemination/api/api_v1_1_1/create_views.sql b/backend/dissemination/api/api_v1_1_1/create_views.sql deleted file mode 100644 index 7fdb73e109..0000000000 --- a/backend/dissemination/api/api_v1_1_1/create_views.sql +++ /dev/null @@ -1,410 +0,0 @@ -begin; - ---------------------------------------- --- finding_text ---------------------------------------- -create view api_v1_1_1.findings_text as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - ft.finding_ref_number, - ft.contains_chart_or_table, - ft.finding_text - from - dissemination_findingtext ft, - dissemination_general gen - where - ft.report_id = gen.report_id - and - (gen.is_public = true - or (gen.is_public = false and api_v1_1_1_functions.has_tribal_data_access())) - order by ft.id -; - ---------------------------------------- --- additional_ueis ---------------------------------------- -create view api_v1_1_1.additional_ueis as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - uei.additional_uei - from - dissemination_general gen, - dissemination_additionaluei uei - where - gen.report_id = uei.report_id - order by uei.id -; - ---------------------------------------- --- finding ---------------------------------------- -create view api_v1_1_1.findings as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - finding.award_reference, - finding.reference_number, - finding.is_material_weakness, - finding.is_modified_opinion, - finding.is_other_findings, - finding.is_other_matters, - finding.prior_finding_ref_numbers, - finding.is_questioned_costs, - finding.is_repeat_finding, - finding.is_significant_deficiency, - finding.type_requirement - from - dissemination_finding finding, - dissemination_general gen - where - finding.report_id = gen.report_id - order by finding.id -; - ---------------------------------------- --- federal award ---------------------------------------- -create view api_v1_1_1.federal_awards as - select - award.report_id, - gen.auditee_uei, - gen.audit_year, - --- - award.award_reference, - award.federal_agency_prefix, - award.federal_award_extension, - award.additional_award_identification, - award.federal_program_name, - award.amount_expended, - award.cluster_name, - award.other_cluster_name, - award.state_cluster_name, - award.cluster_total, - award.federal_program_total, - award.is_major, - award.is_loan, - award.loan_balance, - award.is_direct, - award.audit_report_type, - award.findings_count, - award.is_passthrough_award, - award.passthrough_amount - from - dissemination_federalaward award, - dissemination_general gen - where - award.report_id = gen.report_id - order by award.id -; - - ---------------------------------------- --- corrective_action_plan ---------------------------------------- -create view api_v1_1_1.corrective_action_plans as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - ct.finding_ref_number, - ct.contains_chart_or_table, - ct.planned_action - from - dissemination_CAPText ct, - dissemination_General gen - where - ct.report_id = gen.report_id - and - (gen.is_public = true - or (gen.is_public = false and api_v1_1_1_functions.has_tribal_data_access())) - order by ct.id -; - ---------------------------------------- --- notes_to_sefa ---------------------------------------- -create view api_v1_1_1.notes_to_sefa as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - note.note_title as title, - note.accounting_policies, - note.is_minimis_rate_used, - note.rate_explained, - note.content, - note.contains_chart_or_table - from - dissemination_general gen, - dissemination_note note - where - note.report_id = gen.report_id - and - (gen.is_public = true - or (gen.is_public = false and api_v1_1_1_functions.has_tribal_data_access())) - order by note.id -; - ---------------------------------------- --- passthrough ---------------------------------------- -create view api_v1_1_1.passthrough as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - pass.award_reference, - pass.passthrough_id, - pass.passthrough_name - from - dissemination_general as gen, - dissemination_passthrough as pass - where - gen.report_id = pass.report_id - order by pass.id -; - - ---------------------------------------- --- general ---------------------------------------- -create view api_v1_1_1.general as - select - -- every table starts with report_id, UEI, and year - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - gen.auditee_certify_name, - gen.auditee_certify_title, - gen.auditee_contact_name, - gen.auditee_email, - gen.auditee_name, - gen.auditee_phone, - gen.auditee_contact_title, - gen.auditee_address_line_1, - gen.auditee_city, - gen.auditee_state, - gen.auditee_ein, - gen.auditee_zip, - -- auditor - gen.auditor_certify_name, - gen.auditor_certify_title, - gen.auditor_phone, - gen.auditor_state, - gen.auditor_city, - gen.auditor_contact_title, - gen.auditor_address_line_1, - gen.auditor_zip, - gen.auditor_country, - gen.auditor_contact_name, - gen.auditor_email, - gen.auditor_firm_name, - gen.auditor_foreign_address, - gen.auditor_ein, - -- agency - gen.cognizant_agency, - gen.oversight_agency, - -- dates - gen.date_created, - gen.ready_for_certification_date, - gen.auditor_certified_date, - gen.auditee_certified_date, - gen.submitted_date, - gen.fac_accepted_date, - gen.fy_end_date, - gen.fy_start_date, - gen.audit_type, - gen.gaap_results, - gen.sp_framework_basis, - gen.is_sp_framework_required, - gen.sp_framework_opinions, - gen.is_going_concern_included, - gen.is_internal_control_deficiency_disclosed, - gen.is_internal_control_material_weakness_disclosed, - gen.is_material_noncompliance_disclosed, - gen.dollar_threshold, - gen.is_low_risk_auditee, - gen.agencies_with_prior_findings, - gen.entity_type, - gen.number_months, - gen.audit_period_covered, - gen.total_amount_expended, - gen.type_audit_code, - gen.is_public, - gen.data_source, - gen.is_aicpa_audit_guide_included, - gen.is_additional_ueis, - CASE EXISTS(SELECT ein.report_id FROM dissemination_additionalein ein WHERE ein.report_id = gen.report_id) - WHEN FALSE THEN 'No' - ELSE 'Yes' - END AS is_multiple_eins, - CASE EXISTS(SELECT aud.report_id FROM dissemination_secondaryauditor aud WHERE aud.report_id = gen.report_id) - WHEN FALSE THEN 'No' - ELSE 'Yes' - END AS is_secondary_auditors - from - dissemination_general gen - order by gen.id -; - ---------------------------------------- --- auditor (secondary auditor) ---------------------------------------- -create view api_v1_1_1.secondary_auditors as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - sa.auditor_ein, - sa.auditor_name, - sa.contact_name, - sa.contact_title, - sa.contact_email, - sa.contact_phone, - sa.address_street, - sa.address_city, - sa.address_state, - sa.address_zipcode - from - dissemination_General gen, - dissemination_SecondaryAuditor sa - where - sa.report_id = gen.report_id - order by sa.id -; - -create view api_v1_1_1.additional_eins as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - ein.additional_ein - from - dissemination_general gen, - dissemination_additionalein ein - where - gen.report_id = ein.report_id - order by ein.id -; - --- Specify every field in dissemination_combined, omitting the id. --- Generated fields like ALN are done in the creation of the table, not here. -create view api_v1_1_1.combined as - select - combined.report_id, - combined.award_reference, - combined.reference_number, - combined.aln, - combined.agencies_with_prior_findings, - combined.audit_period_covered, - combined.audit_type, - combined.audit_year, - combined.auditee_address_line_1, - combined.auditee_certified_date, - combined.auditee_certify_name, - combined.auditee_certify_title, - combined.auditee_city, - combined.auditee_contact_name, - combined.auditee_contact_title, - combined.auditee_ein, - combined.auditee_email, - combined.auditee_name, - combined.auditee_phone, - combined.auditee_state, - combined.auditee_uei, - combined.auditee_zip, - combined.auditor_address_line_1, - combined.auditor_certified_date, - combined.auditor_certify_name, - combined.auditor_certify_title, - combined.auditor_city, - combined.auditor_contact_name, - combined.auditor_contact_title, - combined.auditor_country, - combined.auditor_ein, - combined.auditor_email, - combined.auditor_firm_name, - combined.auditor_foreign_address, - combined.auditor_phone, - combined.auditor_state, - combined.auditor_zip, - combined.cognizant_agency, - combined.data_source, - combined.date_created, - combined.dollar_threshold, - combined.entity_type, - combined.fac_accepted_date, - combined.fy_end_date, - combined.fy_start_date, - combined.gaap_results, - combined.is_additional_ueis, - combined.is_aicpa_audit_guide_included, - combined.is_going_concern_included, - combined.is_internal_control_deficiency_disclosed, - combined.is_internal_control_material_weakness_disclosed, - combined.is_low_risk_auditee, - combined.is_material_noncompliance_disclosed, - combined.is_public, - combined.is_sp_framework_required, - combined.number_months, - combined.oversight_agency, - combined.ready_for_certification_date, - combined.sp_framework_basis, - combined.sp_framework_opinions, - combined.submitted_date, - combined.total_amount_expended, - combined.type_audit_code, - combined.additional_award_identification, - combined.amount_expended, - combined.cluster_name, - combined.cluster_total, - combined.federal_agency_prefix, - combined.federal_award_extension, - combined.federal_program_name, - combined.federal_program_total, - combined.findings_count, - combined.is_direct, - combined.is_loan, - combined.is_major, - combined.is_passthrough_award, - combined.loan_balance, - combined.audit_report_type, - combined.other_cluster_name, - combined.passthrough_amount, - combined.state_cluster_name, - combined.is_material_weakness, - combined.is_modified_opinion, - combined.is_other_findings, - combined.is_other_matters, - combined.is_questioned_costs, - combined.is_repeat_finding, - combined.is_significant_deficiency, - combined.prior_finding_ref_numbers, - combined.type_requirement, - combined.passthrough_name, - combined.passthrough_id - from - dissemination_combined combined - where - (combined.is_public = true - or (combined.is_public = false and api_v1_1_1_functions.has_tribal_data_access())) - order by combined.id -; - -commit; - -notify pgrst, - 'reload schema'; - diff --git a/backend/dissemination/api/api_v1_1_1/drop_schema.sql b/backend/dissemination/api/api_v1_1_1/drop_schema.sql deleted file mode 100644 index 705153e878..0000000000 --- a/backend/dissemination/api/api_v1_1_1/drop_schema.sql +++ /dev/null @@ -1,11 +0,0 @@ - -begin; - -DROP SCHEMA IF EXISTS api_v1_1_1 CASCADE; --- DROP ROLE IF EXISTS authenticator; --- DROP ROLE IF EXISTS api_fac_gov; - -commit; - -notify pgrst, - 'reload schema'; diff --git a/backend/dissemination/api/api_v1_1_1/drop_views.sql b/backend/dissemination/api/api_v1_1_1/drop_views.sql deleted file mode 100644 index c95c0fc139..0000000000 --- a/backend/dissemination/api/api_v1_1_1/drop_views.sql +++ /dev/null @@ -1,18 +0,0 @@ -begin; - drop table if exists api_v1_1_1.metadata; - drop view if exists api_v1_1_1.general; - drop view if exists api_v1_1_1.auditor; - drop view if exists api_v1_1_1.federal_awards; - drop view if exists api_v1_1_1.findings; - drop view if exists api_v1_1_1.findings_text; - drop view if exists api_v1_1_1.corrective_action_plans; - drop view if exists api_v1_1_1.additional_ueis; - drop view if exists api_v1_1_1.notes_to_sefa; - drop view if exists api_v1_1_1.passthrough; - drop view if exists api_v1_1_1.secondary_auditors; - drop view if exists api_v1_1_1.additional_eins; - drop view if exists api_v1_1_1.combined; -commit; - -notify pgrst, - 'reload schema'; diff --git a/backend/dissemination/sql/api/api_v1_0_3/standup.sql b/backend/dissemination/sql/api/api_v1_0_3/standup.sql index 18e99e91d1..aa91f37887 100644 --- a/backend/dissemination/sql/api/api_v1_0_3/standup.sql +++ b/backend/dissemination/sql/api/api_v1_0_3/standup.sql @@ -1,3 +1,6 @@ +------------------------------------------------------------------ +-- authenticator role +------------------------------------------------------------------ DO $do$ BEGIN @@ -11,6 +14,9 @@ BEGIN END $do$; +------------------------------------------------------------------ +-- api_fac_gov role +------------------------------------------------------------------ DO $do$ BEGIN @@ -27,74 +33,49 @@ $do$; GRANT api_fac_gov TO authenticator; NOTIFY pgrst, 'reload schema'; -begin; -do -$$ -begin - DROP SCHEMA IF EXISTS api_v1_0_3 CASCADE; - DROP SCHEMA IF EXISTS api_v1_0_3_functions CASCADE; - if not exists (select schema_name from information_schema.schemata where schema_name = 'api_v1_0_3') then - create schema api_v1_0_3; - create schema api_v1_0_3_functions; - - grant usage on schema api_v1_0_3_functions to api_fac_gov; +BEGIN; - -- Grant access to tables and views - alter default privileges - in schema api_v1_0_3 - grant select - -- this includes views - on tables - to api_fac_gov; - - -- Grant access to sequences, if we have them - grant usage on schema api_v1_0_3 to api_fac_gov; - grant select, usage on all sequences in schema api_v1_0_3 to api_fac_gov; - alter default privileges - in schema api_v1_0_3 - grant select, usage - on sequences - to api_fac_gov; - end if; -end -$$ -; - --- This is the description -COMMENT ON SCHEMA api_v1_0_3 IS - 'The FAC dissemation API version 1.0.3.' -; +DO +$APIV103$ + BEGIN + DROP SCHEMA IF EXISTS api_v1_0_3 CASCADE; + DROP SCHEMA IF EXISTS api_v1_0_3_functions CASCADE; --- https://postgrest.org/en/stable/references/api/openapi.html --- This is the title -COMMENT ON SCHEMA api_v1_0_3 IS -$$v1.0.3 + IF NOT EXISTS (select schema_name from information_schema.schemata where schema_name = 'api_v1_0_3') then + CREATE SCHEMA api_v1_0_3; + CREATE SCHEMA api_v1_0_3_functions; + + GRANT USAGE ON SCHEMA api_v1_0_3_functions to api_fac_gov; -A RESTful API that serves data from the SF-SAC.$$; + -- Grant access to tables and views + alter default privileges + in schema api_v1_0_3 + grant select + -- this includes views + on tables + to api_fac_gov; -commit; - -notify pgrst, - 'reload schema'; + -- Grant access to sequences, if we have them + grant usage on schema api_v1_0_3 to api_fac_gov; + grant select, usage on all sequences in schema api_v1_0_3 to api_fac_gov; + alter default privileges + in schema api_v1_0_3 + grant select, usage + on sequences + to api_fac_gov; + end if; + END +$APIV103$ +; --- WARNING --- Under PostgreSQL 12, the functions below work. --- Under PostgreSQL 14, these will break. --- --- Note the differences: --- --- raise info 'Works under PostgreSQL 12'; --- raise info 'request.header.x-magic %', (SELECT current_setting('request.header.x-magic', true)); --- raise info 'request.jwt.claim.expires %', (SELECT current_setting('request.jwt.claim.expires', true)); --- raise info 'Works under PostgreSQL 14'; --- raise info 'request.headers::json->>x-magic %', (SELECT current_setting('request.headers', true)::json->>'x-magic'); --- raise info 'request.jwt.claims::json->expires %', (SELECT current_setting('request.jwt.claims', true)::json->>'expires'); --- --- To quote the work of Dav Pilkey, "remember this now." +COMMIT; +NOTIFY pgrst, 'reload schema'; --- We don't grant tribal access (yet) +------------------------------------------------------------------ +-- functions +------------------------------------------------------------------ create or replace function api_v1_0_3_functions.has_tribal_data_access() returns boolean as $has_tribal_data_access$ BEGIN @@ -103,8 +84,11 @@ END; $has_tribal_data_access$ LANGUAGE plpgsql; NOTIFY pgrst, 'reload schema'; -begin; +------------------------------------------------------------------ +-- VIEWs +------------------------------------------------------------------ +BEGIN; --------------------------------------- -- finding_text --------------------------------------- diff --git a/backend/dissemination/sql/api/api_v1_0_3/teardown.sql b/backend/dissemination/sql/api/api_v1_0_3/teardown.sql index d29d530cbc..7e1b118b45 100644 --- a/backend/dissemination/sql/api/api_v1_0_3/teardown.sql +++ b/backend/dissemination/sql/api/api_v1_0_3/teardown.sql @@ -1,27 +1,7 @@ -begin; +BEGIN; + -- This wipes out the schema and all attached objects, + -- including all of our views. + DROP SCHEMA IF EXISTS api_v1_0_3 CASCADE; +COMMIT; -DROP SCHEMA IF EXISTS api_v1_0_3 CASCADE; --- DROP ROLE IF EXISTS authenticator; --- DROP ROLE IF EXISTS api_fac_gov; - -commit; - -notify pgrst, - 'reload schema'; -begin; - drop table if exists api_v1_0_3.metadata; - drop view if exists api_v1_0_3.general; - drop view if exists api_v1_0_3.auditor; - drop view if exists api_v1_0_3.federal_awards; - drop view if exists api_v1_0_3.findings; - drop view if exists api_v1_0_3.findings_text; - drop view if exists api_v1_0_3.corrective_action_plans; - drop view if exists api_v1_0_3.additional_ueis; - drop view if exists api_v1_0_3.notes_to_sefa; - drop view if exists api_v1_0_3.passthrough; - drop view if exists api_v1_0_3.secondary_auditors; - drop view if exists api_v1_0_3.additional_eins; -commit; - -notify pgrst, - 'reload schema'; +NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/public.rest b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/public.rest new file mode 100644 index 0000000000..e70df70e64 --- /dev/null +++ b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/public.rest @@ -0,0 +1,27 @@ +### +GET {{scheme}}://{{apiUrl}}/general?limit=20000&offset=100000 +authorization: {{authorization}} +x-api-user-id: {{xApiUserId}} +accept-profile: {{acceptProfile}} +Accept: application/vnd.pgrst.plan + +### +GET {{scheme}}://{{apiUrl}}/general?report_id=eq.2021-12-CENSUS-0000250449 +authorization: {{authorization}} +x-api-user-id: {{xApiUserId}} +accept-profile: api_v1_1_0 +Accept: application/vnd.pgrst.plan + +### +GET {{scheme}}://{{apiUrl}}/general?report_id=eq.2021-12-CENSUS-0000250449 +authorization: {{authorization}} +x-api-user-id: {{xApiUserId}} +accept-profile: public_api_v2_0_0_alpha +Accept: application/vnd.pgrst.plan + +### +GET {{scheme}}://{{apiUrl}}/combined?report_id=eq.2021-12-CENSUS-0000250449 +authorization: {{authorization}} +x-api-user-id: {{xApiUserId}} +accept-profile: public_api_v2_0_0_alpha +Accept: application/vnd.pgrst.plan diff --git a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/standup.sql b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/standup.sql index 16e76dab85..e0a36042de 100644 --- a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/standup.sql +++ b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/standup.sql @@ -26,13 +26,6 @@ $do$; GRANT api_fac_gov TO authenticator; -CREATE SEQUENCE IF NOT EXISTS public_data_general - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE; - - NOTIFY pgrst, 'reload schema'; begin; @@ -62,11 +55,6 @@ BEGIN IN SCHEMA public_api_v2_0_0_alpha TO api_fac_gov; - -- ALTER DEFAULT PRIVILEGES - -- IN SCHEMA public_api_v2_0_0_alpha - -- GRANT SELECT, USAGE - -- ON sequences - -- TO api_fac_gov; END $$ ; @@ -150,7 +138,6 @@ CREATE VIEW public_api_v2_0_0_alpha.passthrough AS ORDER BY p.id ; - --------------------------------------- -- auditor (secondary auditor) --------------------------------------- @@ -159,109 +146,14 @@ CREATE VIEW public_api_v2_0_0_alpha.secondary_auditors AS ORDER BY sa.id ; --- Specify every field in dissemination_combined, omitting the id. --- Generated fields like ALN are done in the creation of the table, not here. --- create view public_api_v2_0_0_alpha.combined as --- select --- combined.report_id, --- combined.award_reference, --- combined.reference_number, --- combined.aln, --- combined.agencies_with_prior_findings, --- combined.audit_period_covered, --- combined.audit_type, --- combined.audit_year, --- combined.auditee_address_line_1, --- combined.auditee_certified_date, --- combined.auditee_certify_name, --- combined.auditee_certify_title, --- combined.auditee_city, --- combined.auditee_contact_name, --- combined.auditee_contact_title, --- combined.auditee_ein, --- combined.auditee_email, --- combined.auditee_name, --- combined.auditee_phone, --- combined.auditee_state, --- combined.auditee_uei, --- combined.auditee_zip, --- combined.auditor_address_line_1, --- combined.auditor_certified_date, --- combined.auditor_certify_name, --- combined.auditor_certify_title, --- combined.auditor_city, --- combined.auditor_contact_name, --- combined.auditor_contact_title, --- combined.auditor_country, --- combined.auditor_ein, --- combined.auditor_email, --- combined.auditor_firm_name, --- combined.auditor_foreign_address, --- combined.auditor_phone, --- combined.auditor_state, --- combined.auditor_zip, --- combined.cognizant_agency, --- combined.data_source, --- combined.date_created, --- combined.dollar_threshold, --- combined.entity_type, --- combined.fac_accepted_date, --- combined.fy_end_date, --- combined.fy_start_date, --- combined.gaap_results, --- combined.is_additional_ueis, --- combined.is_aicpa_audit_guide_included, --- combined.is_going_concern_included, --- combined.is_internal_control_deficiency_disclosed, --- combined.is_internal_control_material_weakness_disclosed, --- combined.is_low_risk_auditee, --- combined.is_material_noncompliance_disclosed, --- combined.is_public, --- combined.is_sp_framework_required, --- combined.number_months, --- combined.oversight_agency, --- combined.ready_for_certification_date, --- combined.sp_framework_basis, --- combined.sp_framework_opinions, --- combined.submitted_date, --- combined.total_amount_expended, --- combined.type_audit_code, --- combined.additional_award_identification, --- combined.amount_expended, --- combined.cluster_name, --- combined.cluster_total, --- combined.federal_agency_prefix, --- combined.federal_award_extension, --- combined.federal_program_name, --- combined.federal_program_total, --- combined.findings_count, --- combined.is_direct, --- combined.is_loan, --- combined.is_major, --- combined.is_passthrough_award, --- combined.loan_balance, --- combined.audit_report_type, --- combined.other_cluster_name, --- combined.passthrough_amount, --- combined.state_cluster_name, --- combined.is_material_weakness, --- combined.is_modified_opinion, --- combined.is_other_findings, --- combined.is_other_matters, --- combined.is_questioned_costs, --- combined.is_repeat_finding, --- combined.is_significant_deficiency, --- combined.prior_finding_ref_numbers, --- combined.type_requirement, --- combined.passthrough_name, --- combined.passthrough_id --- from --- dissemination_combined combined --- where --- (combined.is_public = true --- or (combined.is_public = false and public_api_v2_0_0_alpha_functions.has_tribal_data_access())) --- order by combined.id --- ; +--------------------------------------- +-- combined +--------------------------------------- +CREATE VIEW public_api_v2_0_0_alpha.combined AS + SELECT * FROM public_data_v1_0_0.combined comb + ; + + COMMIT; diff --git a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/teardown.sql b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/teardown.sql index 718504fa32..e4570c81bf 100644 --- a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/teardown.sql +++ b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/teardown.sql @@ -3,6 +3,8 @@ BEGIN; -- This drops all the views, too. --- DROP SCHEMA IF EXISTS public_api_v2_0_0_alpha CASCADE; + DROP SCHEMA IF EXISTS public_api_v2_0_0_alpha_functions CASCADE; + COMMIT; notify pgrst, diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/additional_eins.sql b/backend/dissemination/sql/indexes/fac-snapshot-db/additional_eins.sql new file mode 100644 index 0000000000..09169f0c48 --- /dev/null +++ b/backend/dissemination/sql/indexes/fac-snapshot-db/additional_eins.sql @@ -0,0 +1,8 @@ +--------------------------------------- +-- INDEXES on additional_eins +--------------------------------------- +CREATE INDEX IF NOT EXISTS fac_snapshot_db_additional_eins_report_id_idx + on public_data_v1_0_0.additional_eins (report_id); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_additional_eins_report_id_fad + on public_data_v1_0_0.additional_eins (report_id, fac_accepted_date); diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/additional_ueis.sql b/backend/dissemination/sql/indexes/fac-snapshot-db/additional_ueis.sql new file mode 100644 index 0000000000..5835b7f049 --- /dev/null +++ b/backend/dissemination/sql/indexes/fac-snapshot-db/additional_ueis.sql @@ -0,0 +1,8 @@ +--------------------------------------- +-- INDEXES on additional_ueis +--------------------------------------- +CREATE INDEX IF NOT EXISTS fac_snapshot_db_additional_ueis_report_id_idx + on public_data_v1_0_0.additional_ueis (report_id); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_additional_ueis_report_id_fad + on public_data_v1_0_0.additional_ueis (report_id, fac_accepted_date); diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/combined.sql b/backend/dissemination/sql/indexes/fac-snapshot-db/combined.sql new file mode 100644 index 0000000000..21b9668c8c --- /dev/null +++ b/backend/dissemination/sql/indexes/fac-snapshot-db/combined.sql @@ -0,0 +1,75 @@ + +--------------------------------------- +-- INDEXES on combined +--------------------------------------- +CREATE INDEX IF NOT EXISTS fac_snapshot_db_combined_report_id_idx + on public_data_v1_0_0.combined (report_id); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_combined_report_id_fad + on public_data_v1_0_0.combined (report_id, fac_accepted_date); + +-- Some of these may be redundant? Not sure how indexes overlap (or don't). +CREATE INDEX IF NOT EXISTS fac_snapshot_db_combined_fad_aln + on public_data_v1_0_0.combined (fac_accepted_date, aln); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_combined_fad_agency + on public_data_v1_0_0.combined (fac_accepted_date, federal_agency_prefix); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_combined_fad_agency_ext + on public_data_v1_0_0.combined (fac_accepted_date, federal_agency_prefix, federal_award_extension); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_combined_report_id_fad_aln + on public_data_v1_0_0.combined (report_id, fac_accepted_date, aln); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_combined_report_id_fad_agency_ext + on public_data_v1_0_0.combined (report_id, federal_agency_prefix, federal_award_extension); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_combined_auditee_certify_name_idx + ON public_data_v1_0_0.combined + ((lower(auditee_certify_name))); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_combined_auditee_name_idx + ON public_data_v1_0_0.combined + ((lower(auditee_name))); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_combined_auditor_certify_name_idx + ON public_data_v1_0_0.combined + ((lower(auditor_certify_name))); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_combined_auditor_contact_name_idx + ON public_data_v1_0_0.combined + ((lower(auditor_contact_name))); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_combined_auditor_firm_name_idx + ON public_data_v1_0_0.combined + ((lower(auditor_firm_name))); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_combined_auditee_email_idx + on public_data_v1_0_0.combined ((lower(auditee_email))); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_combined_auditor_email_idx + on public_data_v1_0_0.combined ((lower(auditor_email))); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_combined_start_date_idx + ON public_data_v1_0_0.combined (fy_start_date); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_combined_end_date_idx + ON public_data_v1_0_0.combined (fy_end_date); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_combined_auditee_uei_idx + ON public_data_v1_0_0.combined (auditee_uei); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_combined_auditee_ein_idx + ON public_data_v1_0_0.combined (auditee_ein); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_combined_federal_agency_prefix_idx + on public_data_v1_0_0.combined (federal_agency_prefix); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_combined_federal_award_extension_idx + on public_data_v1_0_0.combined (federal_award_extension); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_combined_audit_year_idx + on public_data_v1_0_0.combined (audit_year); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_combined_aln_idx + on public_data_v1_0_0.combined (aln); diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/corrective_action_plans.sql b/backend/dissemination/sql/indexes/fac-snapshot-db/corrective_action_plans.sql new file mode 100644 index 0000000000..5dba259c88 --- /dev/null +++ b/backend/dissemination/sql/indexes/fac-snapshot-db/corrective_action_plans.sql @@ -0,0 +1,11 @@ +--------------------------------------- +-- INDEXES on corrective_action_plans +--------------------------------------- +CREATE INDEX IF NOT EXISTS fac_snapshot_db_corrective_action_plans_report_id_idx + on public_data_v1_0_0.corrective_action_plans (report_id); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_corrective_action_plans_report_id_fad + on public_data_v1_0_0.corrective_action_plans (report_id, fac_accepted_date); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_corrective_action_plans_report_id_fad_frn + on public_data_v1_0_0.corrective_action_plans (report_id, fac_accepted_date, finding_ref_number); diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/federal_awards.sql b/backend/dissemination/sql/indexes/fac-snapshot-db/federal_awards.sql new file mode 100644 index 0000000000..5a6998f169 --- /dev/null +++ b/backend/dissemination/sql/indexes/fac-snapshot-db/federal_awards.sql @@ -0,0 +1,29 @@ +--------------------------------------- +-- INDEXES on federal_awards +--------------------------------------- +CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_awards_report_id_idx + on public_data_v1_0_0.federal_awards (report_id); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_awards_report_id_fad + on public_data_v1_0_0.federal_awards (report_id, fac_accepted_date); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_awards_rid_fap + ON public_data_v1_0_0.federal_awards (report_id, federal_agency_prefix); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_awards_rid_fap_fext + ON public_data_v1_0_0.federal_awards (report_id, federal_agency_prefix, federal_award_extension); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_awards_rid_fap_fext_fad + ON public_data_v1_0_0.federal_awards (report_id, federal_agency_prefix, federal_award_extension, fac_accepted_date); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_awards_fap_fext_cnt + ON public_data_v1_0_0.federal_awards (federal_agency_prefix,federal_award_extension,findings_count); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_awards_rid_aln + ON public_data_v1_0_0.federal_awards (report_id, aln); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_awards_rid_aln_fad + ON public_data_v1_0_0.federal_awards (report_id, aln, fac_accepted_date); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_awards_aln_cnt + ON public_data_v1_0_0.federal_awards (aln,findings_count); diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/findings.sql b/backend/dissemination/sql/indexes/fac-snapshot-db/findings.sql new file mode 100644 index 0000000000..ea83d06316 --- /dev/null +++ b/backend/dissemination/sql/indexes/fac-snapshot-db/findings.sql @@ -0,0 +1,16 @@ +--------------------------------------- +-- INDEXES on findings +--------------------------------------- +CREATE INDEX IF NOT EXISTS fac_snapshot_db_findings_report_id_idx + on public_data_v1_0_0.findings (report_id); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_findings_report_id_fad + on public_data_v1_0_0.findings (report_id, fac_accepted_date); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_findings_rid_aref + on public_data_v1_0_0.findings (report_id, award_reference); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_findings_rid_aref_fad + on public_data_v1_0_0.findings (report_id, award_reference, fac_accepted_date); + + diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/findings_text.sql b/backend/dissemination/sql/indexes/fac-snapshot-db/findings_text.sql new file mode 100644 index 0000000000..4efee2f99a --- /dev/null +++ b/backend/dissemination/sql/indexes/fac-snapshot-db/findings_text.sql @@ -0,0 +1,8 @@ +--------------------------------------- +-- INDEXES on findings_text +--------------------------------------- +CREATE INDEX IF NOT EXISTS fac_snapshot_db_findings_text_report_id_idx + on public_data_v1_0_0.findings_text (report_id); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_findings_text_report_id_fad + on public_data_v1_0_0.findings_text (report_id, fac_accepted_date); diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/general.sql b/backend/dissemination/sql/indexes/fac-snapshot-db/general.sql new file mode 100644 index 0000000000..f590653112 --- /dev/null +++ b/backend/dissemination/sql/indexes/fac-snapshot-db/general.sql @@ -0,0 +1,53 @@ + +--------------------------------------- +-- INDEXES on general +--------------------------------------- +CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_report_id_idx + on public_data_v1_0_0.general (report_id); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_report_id_fad + on public_data_v1_0_0.general (report_id, fac_accepted_date); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_report_id_fad + on public_data_v1_0_0.general (report_id, fac_accepted_date); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditee_certify_name_idx + ON public_data_v1_0_0.general + ((lower(auditee_certify_name))); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditee_name_idx + ON public_data_v1_0_0.general + ((lower(auditee_name))); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditor_certify_name_idx + ON public_data_v1_0_0.general + ((lower(auditor_certify_name))); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditor_contact_name_idx + ON public_data_v1_0_0.general + ((lower(auditor_contact_name))); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditor_firm_name_idx + ON public_data_v1_0_0.general + ((lower(auditor_firm_name))); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditee_email_idx + on public_data_v1_0_0.general ((lower(auditee_email))); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditor_email_idx + on public_data_v1_0_0.general ((lower(auditor_email))); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_start_date_idx + ON public_data_v1_0_0.general (fy_start_date); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_end_date_idx + ON public_data_v1_0_0.general (fy_end_date); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditee_uei_idx + ON public_data_v1_0_0.general (auditee_uei); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditee_ein_idx + ON public_data_v1_0_0.general (auditee_ein); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_audit_year_idx + on public_data_v1_0_0.general (audit_year); diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/notes_to_sefa.sql b/backend/dissemination/sql/indexes/fac-snapshot-db/notes_to_sefa.sql new file mode 100644 index 0000000000..00acf4209b --- /dev/null +++ b/backend/dissemination/sql/indexes/fac-snapshot-db/notes_to_sefa.sql @@ -0,0 +1,8 @@ +--------------------------------------- +-- INDEXES on notes_to_sefa +--------------------------------------- +CREATE INDEX IF NOT EXISTS fac_snapshot_db_notes_to_sefa_report_id_idx + on public_data_v1_0_0.notes_to_sefa (report_id); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_notes_to_sefa_report_id_fad + on public_data_v1_0_0.notes_to_sefa (report_id, fac_accepted_date); diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/passthrough.sql b/backend/dissemination/sql/indexes/fac-snapshot-db/passthrough.sql new file mode 100644 index 0000000000..673209b62a --- /dev/null +++ b/backend/dissemination/sql/indexes/fac-snapshot-db/passthrough.sql @@ -0,0 +1,8 @@ +--------------------------------------- +-- INDEXES on passthrough +--------------------------------------- +CREATE INDEX IF NOT EXISTS fac_snapshot_db_passthrough_report_id_idx + on public_data_v1_0_0.passthrough (report_id); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_passthrough_report_id_fad + on public_data_v1_0_0.passthrough (report_id, fac_accepted_date); diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/secondary_auditors.sql b/backend/dissemination/sql/indexes/fac-snapshot-db/secondary_auditors.sql new file mode 100644 index 0000000000..3680e04bc9 --- /dev/null +++ b/backend/dissemination/sql/indexes/fac-snapshot-db/secondary_auditors.sql @@ -0,0 +1,8 @@ +--------------------------------------- +-- INDEXES on secondary_auditors +--------------------------------------- +CREATE INDEX IF NOT EXISTS fac_snapshot_db_secondary_auditors_report_id_idx + on public_data_v1_0_0.secondary_auditors (report_id); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_secondary_auditors_report_id_fad + on public_data_v1_0_0.secondary_auditors (report_id, fac_accepted_date); diff --git a/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml b/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml index ab9af6f700..093de28a6b 100644 --- a/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml +++ b/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml @@ -1,6 +1,6 @@ # This uses an env var called PG to # set the connection string. -source: FAC_DB_URI +source: FAC_SNAPSHOT_URI target: FAC_SNAPSHOT_URI env: SLING_ALLOW_EMPTY: true @@ -16,12 +16,13 @@ streams: # The first table we create is the general table. All of the # general table is public data, so we copy it wholesale. public.dissemination_general: - mode: full-refresh + mode: incremental + update_key: id object: public_data_v1_0_0.general sql: | -- All of the general table is public data. SELECT - nextval('public_data_v1_0_0.seq_general') AS id, + gen.id, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -108,15 +109,17 @@ streams: gen.type_audit_code FROM public.dissemination_general gen + WHERE {incremental_where_cond} ORDER BY gen.id # All additional EINs are public. public.additional_eins: - mode: full-refresh + mode: incremental + update_key: ein.id object: public_data_v1_0_0.additional_eins sql: | SELECT - nextval('public_data_v1_0_0.seq_additional_eins') AS id, + ein.id, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -128,15 +131,18 @@ streams: public.dissemination_additionalein ein WHERE gen.report_id = ein.report_id + AND + {incremental_where_cond} ORDER BY ein.id # All of the additional UEI info is public info. public.dissemination_additional_ueis: - mode: full-refresh + mode: incremental + update_key: uei.id object: public_data_v1_0_0.additional_ueis sql: | SELECT - nextval('public_data_v1_0_0.seq_additional_ueis') AS id, + uei.id, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -148,44 +154,51 @@ streams: public.dissemination_additionaluei uei WHERE gen.report_id = uei.report_id + AND + {incremental_where_cond} ORDER BY uei.id # Corrective action plans are NOT always public. public.dissemination_corrective_action_plan: - mode: full-refresh + mode: incremental + update_key: cap.id object: public_data_v1_0_0.corrective_action_plans sql: | SELECT - nextval('public_data_v1_0_0.seq_corrective_action_plans') AS id, + cap.id, gen.report_id, gen.auditee_uei, gen.audit_year, gen.fac_accepted_date, --- - ct.contains_chart_or_table, - ct.finding_ref_number, - ct.planned_action + cap.contains_chart_or_table, + cap.finding_ref_number, + cap.planned_action FROM - public.dissemination_captext ct, - public_data_v1_0_0.general gen + public_data_v1_0_0.general gen, + public.dissemination_captext cap WHERE - ct.report_id = gen.report_id + cap.report_id = gen.report_id AND -- Only include the public corrective action plans. gen.is_public = true - ORDER BY ct.id + AND + {incremental_where_cond} + ORDER BY cap.id # All Federal award data is public. public.dissemination_federalaward: - mode: full-refresh + mode: incremental + update_key: award.id object: public_data_v1_0_0.federal_awards sql: | - SELECT - nextval('public_data_v1_0_0.seq_federal_awards') AS id, + SELECT + award.id, dg.report_id, dg.auditee_uei, dg.audit_year, dg.fac_accepted_date, + concat(award.federal_agency_prefix,'.',award.federal_award_extension) as aln, --- award.additional_award_identification, award.amount_expended, @@ -207,19 +220,22 @@ streams: award.passthrough_amount, award.state_cluster_name FROM - public.dissemination_federalaward award, - public_data_v1_0_0.general dg + public_data_v1_0_0.general dg, + public.dissemination_federalaward award WHERE award.report_id = dg.report_id + AND + {incremental_where_cond} ORDER BY award.id # The findings table is public. public.dissemination_findings: - mode: full-refresh + mode: incremental + update_key: finding.id object: public_data_v1_0_0.findings sql: | SELECT - nextval('public_data_v1_0_0.seq_findings') AS id, + finding.id, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -237,19 +253,22 @@ streams: finding.reference_number, finding.type_requirement FROM - public.dissemination_finding finding, - public_data_v1_0_0.general gen + public_data_v1_0_0.general gen, + public.dissemination_finding finding WHERE finding.report_id = gen.report_id + AND + {incremental_where_cond} ORDER BY finding.id # Findings text is NOT always public. public.dissemination_findingstext: - mode: full-refresh + mode: incremental + update_key: ft.id object: public_data_v1_0_0.findings_text sql: | SELECT - nextval('public_data_v1_0_0.seq_findings_text') AS id, + ft.id, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -258,21 +277,24 @@ streams: ft.contains_chart_or_table, ft.finding_text FROM - public.dissemination_findingtext ft, - public_data_v1_0_0.general gen + public_data_v1_0_0.general gen, + public.dissemination_findingtext ft WHERE ft.report_id = gen.report_id AND gen.is_public = true + AND + {incremental_where_cond} ORDER BY ft.id # The notes to SEFA are NOT all public. public.dissemination_notes: - mode: full-refresh + mode: incremental + update_key: note.id object: public_data_v1_0_0.notes_to_sefa sql: | SELECT - nextval('public_data_v1_0_0.seq_notes_to_sefa') AS id, + note.id, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -292,15 +314,18 @@ streams: AND -- Some notes are not public. gen.is_public = true + AND + {incremental_where_cond} ORDER BY note.id # All passthrough information is public. public.dissemination_passthrough: - mode: full-refresh + mode: incremental + update_key: pass.id object: public_data_v1_0_0.passthrough sql: | SELECT - nextval('public_data_v1_0_0.seq_passthrough') AS id, + pass.id, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -314,15 +339,18 @@ streams: public.dissemination_passthrough pass WHERE gen.report_id = pass.report_id + AND + {incremental_where_cond} ORDER BY pass.id # All secondary auditor info is public. public.secondary_auditors: - mode: full-refresh + mode: incremental + update_key: sa.id object: public_data_v1_0_0.secondary_auditors sql: | SELECT - nextval('public_data_v1_0_0.seq_secondary_auditors') AS id, + sa.id, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -343,6 +371,8 @@ streams: public.dissemination_secondaryauditor sa WHERE sa.report_id = gen.report_id + AND + {incremental_where_cond} ORDER BY sa.id public.combined: @@ -458,12 +488,12 @@ streams: dp.passthrough_id, dp.passthrough_name FROM - public_data_v1_0_0.federal_awards dfa + public.dissemination_federalaward dfa LEFT JOIN public_data_v1_0_0.general dg ON dfa.report_id = dg.report_id - LEFT JOIN public_data_v1_0_0.findings df + LEFT JOIN public.dissemination_finding df ON dfa.report_id = df.report_id AND dfa.award_reference = df.award_reference - LEFT JOIN public_data_v1_0_0.passthrough dp + LEFT JOIN public.dissemination_passthrough dp ON dfa.report_id = dp.report_id AND dfa.award_reference = dp.award_reference diff --git a/backend/dissemination/sql/sling/public_data_v1_0_0/standup.sql b/backend/dissemination/sql/sling/public_data_v1_0_0/standup.sql deleted file mode 100644 index f1280eab4e..0000000000 --- a/backend/dissemination/sql/sling/public_data_v1_0_0/standup.sql +++ /dev/null @@ -1,6 +0,0 @@ --- WE DO NEED THE TABLES, THOUGH. --- In theory, `sling` will be creating these tables. But, if they don't exist, --- then the views will crash. So, lets go ahead and create the tables if they don't exist. --- This means we have to stay in sync with the sling YAML. - -CREATE TABLE IF NOT EXISTS public_data_v1_0_0 diff --git a/backend/dissemination/sql/sling/public_data_v1_0_0/teardown.sql b/backend/dissemination/sql/sling/public_data_v1_0_0/teardown.sql deleted file mode 100644 index 022a8b409a..0000000000 --- a/backend/dissemination/sql/sling/public_data_v1_0_0/teardown.sql +++ /dev/null @@ -1,78 +0,0 @@ - -BEGIN; - - -- - -- We always tear everything down. Every time we create the public data, - -- it is a complete refresh. So, rip out the schemas. - -- - DROP SCHEMA IF EXISTS public_data_v1_0_0 CASCADE; - - -- - -- The teardown has to do some standup. Why? In this case, we're about to run - -- `sling`. It will need to access some sequences to create the new tables with unique - -- row IDs. Those sequences need to exist before sling runs. - -- - CREATE SCHEMA IF NOT EXISTS public_data_v1_0_0; - - -- - -- Sequences for the public tables. - CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_general - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE; - CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_additional_eins - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE; - CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_additional_ueis - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE; - CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_corrective_action_plans - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE; - CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_federal_awards - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE; - CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_findings - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE; - CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_findings_text - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE; - CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_notes_to_sefa - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE; - CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_passthrough - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE; - CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_secondary_auditors - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE; - CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_combined - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE; - -COMMIT; - -notify pgrst, - 'reload schema'; diff --git a/backend/docker-compose.yml b/backend/docker-compose.yml index afa155a863..46c4fa0ee2 100644 --- a/backend/docker-compose.yml +++ b/backend/docker-compose.yml @@ -1,5 +1,3 @@ -version: "3.7" - services: #--------------------------------------------- # Postgres DB @@ -34,13 +32,13 @@ services: #--------------------------------------------- # Historic data #--------------------------------------------- - historic-data: - image: ghcr.io/gsa-tts/fac-historic-public-csvs/load-historic-public-data:20230912 - depends_on: - db: - condition: service_healthy - environment: - DATABASE_URL: postgres://postgres@db/postgres + # historic-data: + # image: ghcr.io/gsa-tts/fac-historic-public-csvs/load-historic-public-data:20230912 + # depends_on: + # db: + # condition: service_healthy + # environment: + # DATABASE_URL: postgres://postgres@db/postgres #--------------------------------------------- # Django app @@ -122,17 +120,18 @@ services: expose: - "3000" environment: - PGRST_DB_URI: postgres://postgres@db:5432/postgres - PGRST2_DB_URI: postgres://postgres@db:5431/postgres + # This now requires us to `sling` data to db2 for local API testing. + PGRST_DB_URI: postgres://postgres@db2:5432/postgres + # PGRST2_DB_URI: postgres://postgres@db:5431/postgres PGRST_OPENAPI_SERVER_PROXY_URI: http://127.0.0.1:3000 PGRST_DB_ANON_ROLE: anon # See https://postgrest.org/en/stable/references/api/schemas.html#multiple-schemas for multiple schemas - PGRST_DB_SCHEMAS: "api_v1_0_3, api_v1_1_0, admin_api_v1_1_0" + PGRST_DB_SCHEMAS: "api_v1_0_3,api_v1_1_0,admin_api_v1_1_0,public_api_v2_0_0_alpha" PGRST_JWT_SECRET: ${PGRST_JWT_SECRET:-32_chars_fallback_secret_testing} # Fallback value for testing environments # Enable this to inspect the DB plans for queries via EXPLAIN PGRST_DB_PLAN_ENABLED: 1 depends_on: - db: + db2: condition: service_healthy volumes: postgres-data: diff --git a/backend/run.sh b/backend/run.sh index 30e71f1093..eb20562efd 100755 --- a/backend/run.sh +++ b/backend/run.sh @@ -10,7 +10,8 @@ source tools/util_startup.sh # for local envs (LOCAL or TESTING) and cloud.gov source tools/setup_env.sh source tools/api_teardown.sh -source tools/sling_public_tables.sh +source tools/setup_public_table_placeholders.sh +source tools/build_indexes.sh source tools/migrate_app_tables.sh source tools/api_standup.sh source tools/seed_cog_baseline.sh @@ -27,27 +28,9 @@ gonogo "setup_env" api_teardown gonogo "api_teardown" -##### -# PUBLIC TABLES -# We only want to do this if the sling tables don't exist. -# Because they should run nightly, we'll generally skip this -# during startup. -check_table_exists $FAC_DB_URI 'public_data_v1_0_0.general' -if [ $? -ne 0 ]; -then - # This takes time. We'll need to watch our deploy timeout. - # And, it will interrupt search/API while the tables are being - # recreated. So, we generally don't want to do this on deploy, only - # as part of nightly batch jobs. - sling_public_tables - gonogo "sling_public_tables" -else - startup_log "RUN" "Skipping sling" -fi - ##### # MIGRATE APP TABLES -migrate_app_tables +# migrate_app_tables gonogo "migrate_app_tables" ##### @@ -56,6 +39,12 @@ gonogo "migrate_app_tables" api_standup gonogo "api_standup" +##### +# API STANDUP +# Standup the API, which may depend on migration changes +build_indexes +gonogo "build_indexes" + ##### # SEED COG/OVER TABLES # Setup tables for cog/over assignments diff --git a/backend/tools/api_standup.sh b/backend/tools/api_standup.sh index 1c75a0ea06..f8fbfa669a 100644 --- a/backend/tools/api_standup.sh +++ b/backend/tools/api_standup.sh @@ -3,27 +3,31 @@ source tools/util_startup.sh function api_standup { startup_log "API_STANDUP" "BEGIN" - - run_sql_for_public_apis 'api' 'standup.sql' - - # First create non-managed tables - #startup_log "CREATE_API_ACCESS_TABLES" "BEGIN" - # python manage.py create_api_access_tables - #run_sql_for_public_apis 'api' 'create_access_tables.sql' - #startup_log "CREATE_API_ACCESS_TABLES" "END" + local base_path='dissemination/sql' + local location='api' - # Bring the API back, possibly installing a new API - # startup_log "CREATE_API_SCHEMA" "BEGIN" - # # python manage.py create_api_schema - # run_sql_for_public_apis 'api' 'base.sql' - # run_sql_for_public_apis 'api' 'create_schema.sql' - # startup_log "CREATE_API_SCHEMA" "END" + # This loops by index, so we can use two arrays. + # public_api_versions and public_api_required_tables + for index in "${!api_versions[@]}" + do + local api_version="${api_versions[index]}" + local required_table="${api_required_tables[index]}" + # If the file to stand up the API exists... + if [ -f ${base_path}/${location}/${api_version}/standup.sql ]; then + check_table_exists $FAC_SNAPSHOT_URI $required_table + local result=$? + echo "check_table_exists $required_table $result" + if [ $result -eq 0 ]; then + run_sql $FAC_SNAPSHOT_URI $base_path $location $api_version 'standup.sql' + gonogo "$api_version teardown.sql" + else + echo "API TABLE NOT FOUND/SKIPPED $required_table not found for $api_version" + fi + else + echo "API FILE NOT FOUND/SKIPPED dissemination/sql/api/${api_version}/standup.sql" + fi + done - # startup_log "CREATE_API_VIEWS" "BEGIN" - # # python manage.py create_api_views && - # run_sql_for_public_apis 'api' 'create_functions.sql' - # run_sql_for_public_apis 'api' 'create_views.sql' - # startup_log "CREATE_API_VIEWS" "END" startup_log "API_STANDUP" "END" return 0 diff --git a/backend/tools/api_teardown.sh b/backend/tools/api_teardown.sh index a8892e9c15..ded6ebbcbb 100644 --- a/backend/tools/api_teardown.sh +++ b/backend/tools/api_teardown.sh @@ -1,23 +1,59 @@ source tools/util_startup.sh +function teardown_public_tables () { + run_sql $FAC_DB_URI 'dissemination/sql' 'sling' 'public_data_v1_0_0' 'teardown.sql' + return 0 +} + +function standup_public_table_placeholders () { + # run_sql has its own gonogo built in. + # $SLING_EXE run -r dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml + run_sql $FAC_DB_URI 'dissemination/sql' 'sling' 'public_data_v1_0_0' 'standup.sql' + return 0 +} + +function run_sql_for_public_apis () { + local location="$1" + local sql_file="$2" + local base_path='dissemination/sql' + local location='api' + + for api_version in "${public_api_versions[@]}" + do + if [ -f ${base_path}/${location}/${api_version}/${sql_file} ]; then + # $PSQL_EXE $FAC_DB_URI < ${base_path}/${location}/${api_version}/${sql_file} + run_sql $FAC_DB_URI $base_path $location $api_version $sql_file + else + echo "API FILE NOT FOUND/SKIPPED ${location}/${api_version}/${sql_file}" + fi + done + +} + + + function api_teardown { startup_log "API_TEARDOWN" "BEGIN" - - # startup_log "DROP_DEPRECATED_API_SCHEMA_AND_VIEWS" "BEGIN" - # # python manage.py drop_deprecated_api_schema_and_views - # local d1=$? - # startup_log "DROP_DEPRECATED_API_SCHEMA_AND_VIEWS" "END" - # startup_log "DROP_API_SCHEMA" "BEGIN" - # python manage.py drop_api_schema - - # for api_version in "${public_api_versions[@]}" - # do - # echo "VERSION $api_version" - # $PSQL_EXE $FAC_SNAPSHOT_URI < dissemination/sql/api/${api_version}/drop_schema.sql - # gonogo "$api_version teardown.sql" - # done - - run_sql_for_public_apis 'api' 'teardown.sql' + local base_path='dissemination/sql' + local location='api' + local sql_file='teardown.sql' + + # The API now relies on actual tables. + # We can tear down the API safely, because we're tearing down + # just the permissions at this point. That is, we're tearing down + # the API portion. The data will remain in the DB. + + #run_sql_for_public_apis 'api' 'teardown.sql' + for api_version in "${public_api_versions[@]}" + do + if [ -f ${base_path}/${location}/${api_version}/${sql_file} ]; then + # $PSQL_EXE $FAC_DB_URI < ${base_path}/${location}/${api_version}/${sql_file} + run_sql $FAC_DB_URI $base_path $location $api_version $sql_file + gonogo "$api_version teardown.sql" + else + echo "API FILE NOT FOUND/SKIPPED ${location}/${api_version}/${sql_file}" + fi + done startup_log "API_TEARDOWN" "END" diff --git a/backend/tools/build_indexes.sh b/backend/tools/build_indexes.sh new file mode 100644 index 0000000000..18452b393f --- /dev/null +++ b/backend/tools/build_indexes.sh @@ -0,0 +1,29 @@ +source tools/util_startup.sh + +function build_indexes { + startup_log "BUILD INDEXES" "BEGIN" + local base_path='dissemination/sql' + local location='indexes' + local which_db='fac-snapshot-db' + # This loops by index, so we can use two arrays. + for index in "${!db2_indexes[@]}" + do + local dbindex="${db2_indexes[index]}" + local required_table="${db2_indexes_required_tables[index]}" + # If the file to stand up the API exists... + if [ -f ${base_path}/${location}/$which_db/${dbindex}.sql ]; then + check_table_exists $FAC_SNAPSHOT_URI $required_table + local result=$? + echo "check_table_exists $required_table $result" + if [ $result -eq 0 ]; then + run_sql $FAC_SNAPSHOT_URI $base_path $location $which_db ${dbindex}.sql + gonogo "${dbindex}" + else + echo "API TABLE NOT FOUND/SKIPPED $required_table not found for $dbindex" + fi + else + echo "API FILE NOT FOUND/SKIPPED ${base_path}/${location}/$which_db/${dbindex}.sql" + fi + done + startup_log "BUILD INDEXES" "END" +} diff --git a/backend/tools/load_public_dissem_data/.gitignore b/backend/tools/load_public_dissem_data/.gitignore index 1afc10f9b1..b5f6e4ed48 100644 --- a/backend/tools/load_public_dissem_data/.gitignore +++ b/backend/tools/load_public_dissem_data/.gitignore @@ -2,5 +2,5 @@ *.zip *.tar *.gz -*.json db_dissem_dump* +__MACOSX diff --git a/backend/tools/load_public_dissem_data/Dockerfile b/backend/tools/load_public_dissem_data/Dockerfile index 53614673ba..473c5cf2a1 100644 --- a/backend/tools/load_public_dissem_data/Dockerfile +++ b/backend/tools/load_public_dissem_data/Dockerfile @@ -7,7 +7,9 @@ RUN apt-get install -y \ curl \ unzip -WORKDIR /app +WORKDIR /layered +COPY data/sling.yaml . +COPY data/config.json . # Curl in `sling` RUN curl -LO 'https://github.com/slingdata-io/sling-cli/releases/latest/download/sling_linux_amd64.tar.gz' \ @@ -19,9 +21,13 @@ RUN curl -LO 'https://github.com/slingdata-io/sling-cli/releases/latest/download RUN curl -L -O https://github.com/GSA-TTS/fac-backup-utility/releases/download/v0.1.8/gov.gsa.fac.cgov-util-v0.1.8-linux-amd64.tar.gz \ && tar xvzf gov.gsa.fac.cgov-util-v0.1.8-linux-amd64.tar.gz gov.gsa.fac.cgov-util \ && chmod 755 gov.gsa.fac.cgov-util \ - && mv gov.gsa.fac.cgov-util /bin/gov.gsa.fac.cgov-util + && mv gov.gsa.fac.cgov-util /layered/cgov-util +# This grabs a config file from the repo +# that is intended for local devleopment. Therefore, it has all the right values +# for talking to our stack. WORKDIR /root/.fac RUN curl -L -O https://raw.githubusercontent.com/GSA-TTS/fac-backup-utility/refs/heads/main/config.json +WORKDIR /app ENTRYPOINT [ "./load_public_data_locally.sh" ] diff --git a/backend/tools/load_public_dissem_data/Makefile b/backend/tools/load_public_dissem_data/Makefile new file mode 100644 index 0000000000..0d4bdba335 --- /dev/null +++ b/backend/tools/load_public_dissem_data/Makefile @@ -0,0 +1,8 @@ +NETWORK?=backend-web-1 + +build: + cp ../../dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml data/sling.yaml + docker build -t facloaddata . + +run: + docker run -i --rm -v ${PWD}:/app --network container:${NETWORK} -t facloaddata diff --git a/backend/tools/load_public_dissem_data/README.md b/backend/tools/load_public_dissem_data/README.md index 5dc9b32630..3134d062a0 100644 --- a/backend/tools/load_public_dissem_data/README.md +++ b/backend/tools/load_public_dissem_data/README.md @@ -1,6 +1,34 @@ # loading public data -This script loads public data. +This provides a containerized data loading process that sets up your local FAC in a manner that duplicates the live/production app. + +The data we are using is public, historic data. It can be replaced, at a later point, with data that is more current. + +## Full clean + +You might want a completely clean local stack to start. It is not strictly necessary. If you get key conflicts, it means you already have some of this historic data loaded. + +### Wipe the stack + +From the backend folder + +``` +make -i docker-full-clean +``` + +Note the `-i` flag. This means `make` should ignore errors. You want it to, so it will keep going and wipe everything. + +``` +make docker-first-run +``` + +and then + +``` +docker compose up +``` + +We need the stack running for this whole process. ## Grab the ZIP @@ -12,39 +40,38 @@ which is cleaned/historic public data, fit for our dissem_* tables. Compressed, it is 330MB. Uncompressed, around 3GB. -Put it in dissemination/tools/load_public_dissem_data (this directory) +Put it in dissemination/tools/load_public_dissem_data/data (a child of this directory) + +## Build the container -## Run the script +This is containerized, so it should work on all platforms. To build the container, run ``` -./load_public_data_locally.sh +make build ``` -Note you need to run it from within that directory. +Then, to run the container, -## Or, run the Dockerfile +``` +make run +``` -If you're on a Mac, you may want to run the docker container to do the load. +You need to run this from the current directory. -First, build the container. +NOTE: The docker command in the Makefile uses the `--network` flag. The `--network` flag tells Docker to run this container on the same network as your currently running stack. It assumes that you did a `docker compose up` on the FAC stack, and that the web container has the default name of `backend-web-1`. If this does not work, you will need to... ``` -docker build -t facloaddata . +make NETWORK= run ``` -Then, with the zipfile of data in the current directory: +where `` is the name of your web container. This should allow this container to correctly talk to our databases. -``` -docker run -i --rm -v ${PWD}:/app \ - --network container:backend-web-1 \ - -t facloaddata -``` +## When to rebuild this container -The `--network` flag tells Docker to run this container on the same network as your currently running stack. It assumes that you did a `docker compose up` on the FAC stack, and that the web container has the default name of `backend-web-1`. If this does not work, you will need to do a +Note this is pinned to v0.1.8 of the cgov-util. -``` -docker ps -``` +https://github.com/GSA-TTS/fac-backup-utility -and discover the name of a container running in your FAC stack. Once you do this, it will be possible for the loader script to find the Postgres databases. +If that gets updated, you'll need to update the dockerfile. +It also copies in the YAML for sling from `dissemination/sql/sling`. If that changes, you'll want to diff --git a/backend/tools/load_public_dissem_data/data/config.json b/backend/tools/load_public_dissem_data/data/config.json new file mode 100644 index 0000000000..4551b86b62 --- /dev/null +++ b/backend/tools/load_public_dissem_data/data/config.json @@ -0,0 +1,90 @@ +{ + "user-provided": [ + { + "label": "mc", + "name": "backups", + "tags": [], + "instance_guid": "UUIDALPHA1", + "instance_name": "backups", + "binding_guid": "UUIDALPHA2", + "binding_name": null, + "credentials": { + "access_key_id": "longtest", + "secret_access_key": "longtest", + "bucket": "backups", + "endpoint": "http://127.0.0.1:9001", + "admin_username": "minioadmin", + "admin_password": "minioadmin" + } + }, + { + "label": "mc", + "name": "fac-private-s3", + "tags": [], + "instance_guid": "UUIDALPHA1", + "instance_name": "fac-private-s3", + "binding_guid": "UUIDALPHA2", + "binding_name": null, + "credentials": { + "access_key_id": "longtest", + "secret_access_key": "longtest", + "bucket": "fac-private-s3", + "endpoint": "http://127.0.0.1:9001", + "admin_username": "minioadmin", + "admin_password": "minioadmin" + } + } + ], + "aws-rds": [ + { + "label": "fac-db", + "provider": null, + "plan": null, + "name": "fac-db", + "tags": [ + "database", + "docker" + ], + "instance_guid": "UUIDINDIA1", + "instance_name": "db", + "binding_guid": "UUIDINDIA2", + "binding_name": null, + "credentials": { + "db_name": "postgres", + "host": "db", + "name": "postgres", + "password": "", + "port": "5432", + "uri": "postgres://postgres@db:5432/postgres?sslmode=disable", + "username": "postgres" + }, + "syslog_drain_url": null, + "volume_mounts": [] + }, + { + "label": "fac-snapshot-db", + "provider": null, + "plan": null, + "name": "fac-snapshot-db", + "tags": [ + "database", + "docker" + ], + "instance_guid": "UUIDJULIET1", + "instance_name": "db", + "binding_guid": "UUIDJULIET2", + "binding_name": null, + "credentials": { + "db_name": "postgres", + "host": "db2", + "name": "postgres", + "password": "", + "port": "5432", + "uri": "postgres://postgres@db2:5432/postgres?sslmode=disable", + "username": "postgres" + }, + "syslog_drain_url": null, + "volume_mounts": [] + } + ] +} diff --git a/backend/tools/load_public_dissem_data/data/public_data_v1_0_0.yaml b/backend/tools/load_public_dissem_data/data/public_data_v1_0_0.yaml new file mode 100644 index 0000000000..093de28a6b --- /dev/null +++ b/backend/tools/load_public_dissem_data/data/public_data_v1_0_0.yaml @@ -0,0 +1,499 @@ +# This uses an env var called PG to +# set the connection string. +source: FAC_SNAPSHOT_URI +target: FAC_SNAPSHOT_URI +env: + SLING_ALLOW_EMPTY: true + +# Handy +# SELECT column_name +# FROM information_schema.columns +# WHERE table_schema = 'public' +# AND table_name = 'dissemination_general' +# AND column_name NOT IN ('id'); + +streams: + # The first table we create is the general table. All of the + # general table is public data, so we copy it wholesale. + public.dissemination_general: + mode: incremental + update_key: id + object: public_data_v1_0_0.general + sql: | + -- All of the general table is public data. + SELECT + gen.id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.agencies_with_prior_findings, + gen.audit_period_covered, + gen.audit_type, + gen.auditee_address_line_1, + gen.auditee_certified_date, + gen.auditee_certify_name, + gen.auditee_certify_title, + gen.auditee_city, + gen.auditee_contact_name, + gen.auditee_contact_title, + gen.auditee_ein, + gen.auditee_email, + gen.auditee_name, + gen.auditee_phone, + gen.auditee_state, + gen.auditee_zip, + gen.auditor_address_line_1, + gen.auditor_certified_date, + gen.auditor_certify_name, + gen.auditor_certify_title, + gen.auditor_city, + gen.auditor_contact_name, + gen.auditor_contact_title, + gen.auditor_country, + gen.auditor_ein, + gen.auditor_email, + gen.auditor_firm_name, + gen.auditor_foreign_address, + gen.auditor_phone, + gen.auditor_state, + gen.auditor_zip, + gen.cognizant_agency, + gen.data_source, + gen.date_created, + gen.dollar_threshold, + gen.entity_type, + gen.fac_accepted_date, + gen.fy_end_date, + gen.fy_start_date, + gen.gaap_results, + gen.is_additional_ueis, + gen.is_aicpa_audit_guide_included, + gen.is_going_concern_included, + gen.is_internal_control_deficiency_disclosed, + gen.is_internal_control_material_weakness_disclosed, + gen.is_low_risk_auditee, + gen.is_material_noncompliance_disclosed, + CASE EXISTS + ( + SELECT + ein.report_id + FROM + dissemination_additionalein ein + WHERE + ein.report_id = gen.report_id + ) + WHEN FALSE THEN 'No' + ELSE 'Yes' + END AS is_multiple_eins, + gen.is_public, + CASE EXISTS + ( + SELECT + aud.report_id + FROM + dissemination_secondaryauditor aud + WHERE + aud.report_id = gen.report_id + ) + WHEN FALSE THEN 'No' + ELSE 'Yes' + END AS is_secondary_auditors, + gen.is_sp_framework_required, + gen.number_months, + gen.oversight_agency, + gen.ready_for_certification_date, + gen.sp_framework_basis, + gen.sp_framework_opinions, + gen.submitted_date, + gen.total_amount_expended, + gen.type_audit_code + FROM + public.dissemination_general gen + WHERE {incremental_where_cond} + ORDER BY gen.id + + # All additional EINs are public. + public.additional_eins: + mode: incremental + update_key: ein.id + object: public_data_v1_0_0.additional_eins + sql: | + SELECT + ein.id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + ein.additional_ein + FROM + public_data_v1_0_0.general gen, + public.dissemination_additionalein ein + WHERE + gen.report_id = ein.report_id + AND + {incremental_where_cond} + ORDER BY ein.id + + # All of the additional UEI info is public info. + public.dissemination_additional_ueis: + mode: incremental + update_key: uei.id + object: public_data_v1_0_0.additional_ueis + sql: | + SELECT + uei.id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + uei.additional_uei + FROM + public_data_v1_0_0.general gen, + public.dissemination_additionaluei uei + WHERE + gen.report_id = uei.report_id + AND + {incremental_where_cond} + ORDER BY uei.id + + # Corrective action plans are NOT always public. + public.dissemination_corrective_action_plan: + mode: incremental + update_key: cap.id + object: public_data_v1_0_0.corrective_action_plans + sql: | + SELECT + cap.id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + cap.contains_chart_or_table, + cap.finding_ref_number, + cap.planned_action + FROM + public_data_v1_0_0.general gen, + public.dissemination_captext cap + WHERE + cap.report_id = gen.report_id + AND + -- Only include the public corrective action plans. + gen.is_public = true + AND + {incremental_where_cond} + ORDER BY cap.id + + # All Federal award data is public. + public.dissemination_federalaward: + mode: incremental + update_key: award.id + object: public_data_v1_0_0.federal_awards + sql: | + SELECT + award.id, + dg.report_id, + dg.auditee_uei, + dg.audit_year, + dg.fac_accepted_date, + concat(award.federal_agency_prefix,'.',award.federal_award_extension) as aln, + --- + award.additional_award_identification, + award.amount_expended, + award.audit_report_type, + award.award_reference, + award.cluster_name, + award.cluster_total, + award.federal_agency_prefix, + award.federal_award_extension, + award.federal_program_name, + award.federal_program_total, + award.findings_count, + award.is_direct, + award.is_loan, + award.is_major, + award.is_passthrough_award, + award.loan_balance, + award.other_cluster_name, + award.passthrough_amount, + award.state_cluster_name + FROM + public_data_v1_0_0.general dg, + public.dissemination_federalaward award + WHERE + award.report_id = dg.report_id + AND + {incremental_where_cond} + ORDER BY award.id + + # The findings table is public. + public.dissemination_findings: + mode: incremental + update_key: finding.id + object: public_data_v1_0_0.findings + sql: | + SELECT + finding.id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + finding.award_reference, + finding.is_material_weakness, + finding.is_modified_opinion, + finding.is_other_findings, + finding.is_other_matters, + finding.is_questioned_costs, + finding.is_repeat_finding, + finding.is_significant_deficiency, + finding.prior_finding_ref_numbers, + finding.reference_number, + finding.type_requirement + FROM + public_data_v1_0_0.general gen, + public.dissemination_finding finding + WHERE + finding.report_id = gen.report_id + AND + {incremental_where_cond} + ORDER BY finding.id + + # Findings text is NOT always public. + public.dissemination_findingstext: + mode: incremental + update_key: ft.id + object: public_data_v1_0_0.findings_text + sql: | + SELECT + ft.id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + ft.finding_ref_number, + ft.contains_chart_or_table, + ft.finding_text + FROM + public_data_v1_0_0.general gen, + public.dissemination_findingtext ft + WHERE + ft.report_id = gen.report_id + AND + gen.is_public = true + AND + {incremental_where_cond} + ORDER BY ft.id + + # The notes to SEFA are NOT all public. + public.dissemination_notes: + mode: incremental + update_key: note.id + object: public_data_v1_0_0.notes_to_sefa + sql: | + SELECT + note.id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + note.accounting_policies, + note.contains_chart_or_table, + note.content, + note.is_minimis_rate_used, + note.note_title as title, + note.rate_explained + FROM + public_data_v1_0_0.general gen, + public.dissemination_note note + WHERE + note.report_id = gen.report_id + AND + -- Some notes are not public. + gen.is_public = true + AND + {incremental_where_cond} + ORDER BY note.id + + # All passthrough information is public. + public.dissemination_passthrough: + mode: incremental + update_key: pass.id + object: public_data_v1_0_0.passthrough + sql: | + SELECT + pass.id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + pass.award_reference, + pass.passthrough_id, + pass.passthrough_name + FROM + public_data_v1_0_0.general gen, + public.dissemination_passthrough pass + WHERE + gen.report_id = pass.report_id + AND + {incremental_where_cond} + ORDER BY pass.id + + # All secondary auditor info is public. + public.secondary_auditors: + mode: incremental + update_key: sa.id + object: public_data_v1_0_0.secondary_auditors + sql: | + SELECT + sa.id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + sa.address_city, + sa.address_state, + sa.address_street, + sa.address_zipcode, + sa.auditor_ein, + sa.auditor_name, + sa.contact_email, + sa.contact_name, + sa.contact_phone, + sa.contact_title + FROM + public_data_v1_0_0.general gen, + public.dissemination_secondaryauditor sa + WHERE + sa.report_id = gen.report_id + AND + {incremental_where_cond} + ORDER BY sa.id + + public.combined: + mode: full-refresh + object: public_data_v1_0_0.combined + sql: | + SELECT + nextval('public_data_v1_0_0.seq_combined') AS id, + dg.report_id, + dfa.award_reference, + df.reference_number, + concat(dfa.federal_agency_prefix,'.',dfa.federal_award_extension) as aln, + -- + -- general + -- + dg.auditee_uei, + dg.audit_year, + dg.agencies_with_prior_findings, + dg.audit_period_covered, + dg.audit_type, + dg.auditee_address_line_1, + dg.auditee_certified_date, + dg.auditee_certify_name, + dg.auditee_certify_title, + dg.auditee_city, + dg.auditee_contact_name, + dg.auditee_contact_title, + dg.auditee_ein, + dg.auditee_email, + dg.auditee_name, + dg.auditee_phone, + dg.auditee_state, + dg.auditee_zip, + dg.auditor_address_line_1, + dg.auditor_certified_date, + dg.auditor_certify_name, + dg.auditor_certify_title, + dg.auditor_city, + dg.auditor_contact_name, + dg.auditor_contact_title, + dg.auditor_country, + dg.auditor_ein, + dg.auditor_email, + dg.auditor_firm_name, + dg.auditor_foreign_address, + dg.auditor_phone, + dg.auditor_state, + dg.auditor_zip, + dg.cognizant_agency, + dg.data_source, + dg.date_created, + dg.dollar_threshold, + dg.entity_type, + dg.fac_accepted_date, + dg.fy_end_date, + dg.fy_start_date, + dg.gaap_results, + dg.is_additional_ueis, + dg.is_aicpa_audit_guide_included, + dg.is_going_concern_included, + dg.is_internal_control_deficiency_disclosed, + dg.is_internal_control_material_weakness_disclosed, + dg.is_low_risk_auditee, + dg.is_material_noncompliance_disclosed, + dg.is_multiple_eins, + dg.is_public, + dg.is_secondary_auditors, + dg.is_sp_framework_required, + dg.number_months, + dg.oversight_agency, + dg.ready_for_certification_date, + dg.sp_framework_basis, + dg.sp_framework_opinions, + dg.submitted_date, + dg.total_amount_expended, + dg.type_audit_code, + -- + -- federal_award + -- + dfa.additional_award_identification, + dfa.amount_expended, + dfa.audit_report_type, + dfa.cluster_name, + dfa.cluster_total, + dfa.federal_agency_prefix, + dfa.federal_award_extension, + dfa.federal_program_name, + dfa.federal_program_total, + dfa.findings_count, + dfa.is_direct, + dfa.is_loan, + dfa.is_major, + dfa.is_passthrough_award, + dfa.loan_balance, + dfa.other_cluster_name, + dfa.passthrough_amount, + dfa.state_cluster_name, + -- + -- finding + -- + df.is_material_weakness, + df.is_modified_opinion, + df.is_other_findings, + df.is_other_matters, + df.is_questioned_costs, + df.is_repeat_finding, + df.is_significant_deficiency, + df.prior_finding_ref_numbers, + df.type_requirement, + -- + -- passthrough + -- + dp.passthrough_id, + dp.passthrough_name + FROM + public.dissemination_federalaward dfa + LEFT JOIN public_data_v1_0_0.general dg + ON dfa.report_id = dg.report_id + LEFT JOIN public.dissemination_finding df + ON dfa.report_id = df.report_id + AND dfa.award_reference = df.award_reference + LEFT JOIN public.dissemination_passthrough dp + ON dfa.report_id = dp.report_id + AND dfa.award_reference = dp.award_reference diff --git a/backend/tools/load_public_dissem_data/data/sling.yaml b/backend/tools/load_public_dissem_data/data/sling.yaml new file mode 100644 index 0000000000..093de28a6b --- /dev/null +++ b/backend/tools/load_public_dissem_data/data/sling.yaml @@ -0,0 +1,499 @@ +# This uses an env var called PG to +# set the connection string. +source: FAC_SNAPSHOT_URI +target: FAC_SNAPSHOT_URI +env: + SLING_ALLOW_EMPTY: true + +# Handy +# SELECT column_name +# FROM information_schema.columns +# WHERE table_schema = 'public' +# AND table_name = 'dissemination_general' +# AND column_name NOT IN ('id'); + +streams: + # The first table we create is the general table. All of the + # general table is public data, so we copy it wholesale. + public.dissemination_general: + mode: incremental + update_key: id + object: public_data_v1_0_0.general + sql: | + -- All of the general table is public data. + SELECT + gen.id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.agencies_with_prior_findings, + gen.audit_period_covered, + gen.audit_type, + gen.auditee_address_line_1, + gen.auditee_certified_date, + gen.auditee_certify_name, + gen.auditee_certify_title, + gen.auditee_city, + gen.auditee_contact_name, + gen.auditee_contact_title, + gen.auditee_ein, + gen.auditee_email, + gen.auditee_name, + gen.auditee_phone, + gen.auditee_state, + gen.auditee_zip, + gen.auditor_address_line_1, + gen.auditor_certified_date, + gen.auditor_certify_name, + gen.auditor_certify_title, + gen.auditor_city, + gen.auditor_contact_name, + gen.auditor_contact_title, + gen.auditor_country, + gen.auditor_ein, + gen.auditor_email, + gen.auditor_firm_name, + gen.auditor_foreign_address, + gen.auditor_phone, + gen.auditor_state, + gen.auditor_zip, + gen.cognizant_agency, + gen.data_source, + gen.date_created, + gen.dollar_threshold, + gen.entity_type, + gen.fac_accepted_date, + gen.fy_end_date, + gen.fy_start_date, + gen.gaap_results, + gen.is_additional_ueis, + gen.is_aicpa_audit_guide_included, + gen.is_going_concern_included, + gen.is_internal_control_deficiency_disclosed, + gen.is_internal_control_material_weakness_disclosed, + gen.is_low_risk_auditee, + gen.is_material_noncompliance_disclosed, + CASE EXISTS + ( + SELECT + ein.report_id + FROM + dissemination_additionalein ein + WHERE + ein.report_id = gen.report_id + ) + WHEN FALSE THEN 'No' + ELSE 'Yes' + END AS is_multiple_eins, + gen.is_public, + CASE EXISTS + ( + SELECT + aud.report_id + FROM + dissemination_secondaryauditor aud + WHERE + aud.report_id = gen.report_id + ) + WHEN FALSE THEN 'No' + ELSE 'Yes' + END AS is_secondary_auditors, + gen.is_sp_framework_required, + gen.number_months, + gen.oversight_agency, + gen.ready_for_certification_date, + gen.sp_framework_basis, + gen.sp_framework_opinions, + gen.submitted_date, + gen.total_amount_expended, + gen.type_audit_code + FROM + public.dissemination_general gen + WHERE {incremental_where_cond} + ORDER BY gen.id + + # All additional EINs are public. + public.additional_eins: + mode: incremental + update_key: ein.id + object: public_data_v1_0_0.additional_eins + sql: | + SELECT + ein.id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + ein.additional_ein + FROM + public_data_v1_0_0.general gen, + public.dissemination_additionalein ein + WHERE + gen.report_id = ein.report_id + AND + {incremental_where_cond} + ORDER BY ein.id + + # All of the additional UEI info is public info. + public.dissemination_additional_ueis: + mode: incremental + update_key: uei.id + object: public_data_v1_0_0.additional_ueis + sql: | + SELECT + uei.id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + uei.additional_uei + FROM + public_data_v1_0_0.general gen, + public.dissemination_additionaluei uei + WHERE + gen.report_id = uei.report_id + AND + {incremental_where_cond} + ORDER BY uei.id + + # Corrective action plans are NOT always public. + public.dissemination_corrective_action_plan: + mode: incremental + update_key: cap.id + object: public_data_v1_0_0.corrective_action_plans + sql: | + SELECT + cap.id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + cap.contains_chart_or_table, + cap.finding_ref_number, + cap.planned_action + FROM + public_data_v1_0_0.general gen, + public.dissemination_captext cap + WHERE + cap.report_id = gen.report_id + AND + -- Only include the public corrective action plans. + gen.is_public = true + AND + {incremental_where_cond} + ORDER BY cap.id + + # All Federal award data is public. + public.dissemination_federalaward: + mode: incremental + update_key: award.id + object: public_data_v1_0_0.federal_awards + sql: | + SELECT + award.id, + dg.report_id, + dg.auditee_uei, + dg.audit_year, + dg.fac_accepted_date, + concat(award.federal_agency_prefix,'.',award.federal_award_extension) as aln, + --- + award.additional_award_identification, + award.amount_expended, + award.audit_report_type, + award.award_reference, + award.cluster_name, + award.cluster_total, + award.federal_agency_prefix, + award.federal_award_extension, + award.federal_program_name, + award.federal_program_total, + award.findings_count, + award.is_direct, + award.is_loan, + award.is_major, + award.is_passthrough_award, + award.loan_balance, + award.other_cluster_name, + award.passthrough_amount, + award.state_cluster_name + FROM + public_data_v1_0_0.general dg, + public.dissemination_federalaward award + WHERE + award.report_id = dg.report_id + AND + {incremental_where_cond} + ORDER BY award.id + + # The findings table is public. + public.dissemination_findings: + mode: incremental + update_key: finding.id + object: public_data_v1_0_0.findings + sql: | + SELECT + finding.id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + finding.award_reference, + finding.is_material_weakness, + finding.is_modified_opinion, + finding.is_other_findings, + finding.is_other_matters, + finding.is_questioned_costs, + finding.is_repeat_finding, + finding.is_significant_deficiency, + finding.prior_finding_ref_numbers, + finding.reference_number, + finding.type_requirement + FROM + public_data_v1_0_0.general gen, + public.dissemination_finding finding + WHERE + finding.report_id = gen.report_id + AND + {incremental_where_cond} + ORDER BY finding.id + + # Findings text is NOT always public. + public.dissemination_findingstext: + mode: incremental + update_key: ft.id + object: public_data_v1_0_0.findings_text + sql: | + SELECT + ft.id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + ft.finding_ref_number, + ft.contains_chart_or_table, + ft.finding_text + FROM + public_data_v1_0_0.general gen, + public.dissemination_findingtext ft + WHERE + ft.report_id = gen.report_id + AND + gen.is_public = true + AND + {incremental_where_cond} + ORDER BY ft.id + + # The notes to SEFA are NOT all public. + public.dissemination_notes: + mode: incremental + update_key: note.id + object: public_data_v1_0_0.notes_to_sefa + sql: | + SELECT + note.id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + note.accounting_policies, + note.contains_chart_or_table, + note.content, + note.is_minimis_rate_used, + note.note_title as title, + note.rate_explained + FROM + public_data_v1_0_0.general gen, + public.dissemination_note note + WHERE + note.report_id = gen.report_id + AND + -- Some notes are not public. + gen.is_public = true + AND + {incremental_where_cond} + ORDER BY note.id + + # All passthrough information is public. + public.dissemination_passthrough: + mode: incremental + update_key: pass.id + object: public_data_v1_0_0.passthrough + sql: | + SELECT + pass.id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + pass.award_reference, + pass.passthrough_id, + pass.passthrough_name + FROM + public_data_v1_0_0.general gen, + public.dissemination_passthrough pass + WHERE + gen.report_id = pass.report_id + AND + {incremental_where_cond} + ORDER BY pass.id + + # All secondary auditor info is public. + public.secondary_auditors: + mode: incremental + update_key: sa.id + object: public_data_v1_0_0.secondary_auditors + sql: | + SELECT + sa.id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + sa.address_city, + sa.address_state, + sa.address_street, + sa.address_zipcode, + sa.auditor_ein, + sa.auditor_name, + sa.contact_email, + sa.contact_name, + sa.contact_phone, + sa.contact_title + FROM + public_data_v1_0_0.general gen, + public.dissemination_secondaryauditor sa + WHERE + sa.report_id = gen.report_id + AND + {incremental_where_cond} + ORDER BY sa.id + + public.combined: + mode: full-refresh + object: public_data_v1_0_0.combined + sql: | + SELECT + nextval('public_data_v1_0_0.seq_combined') AS id, + dg.report_id, + dfa.award_reference, + df.reference_number, + concat(dfa.federal_agency_prefix,'.',dfa.federal_award_extension) as aln, + -- + -- general + -- + dg.auditee_uei, + dg.audit_year, + dg.agencies_with_prior_findings, + dg.audit_period_covered, + dg.audit_type, + dg.auditee_address_line_1, + dg.auditee_certified_date, + dg.auditee_certify_name, + dg.auditee_certify_title, + dg.auditee_city, + dg.auditee_contact_name, + dg.auditee_contact_title, + dg.auditee_ein, + dg.auditee_email, + dg.auditee_name, + dg.auditee_phone, + dg.auditee_state, + dg.auditee_zip, + dg.auditor_address_line_1, + dg.auditor_certified_date, + dg.auditor_certify_name, + dg.auditor_certify_title, + dg.auditor_city, + dg.auditor_contact_name, + dg.auditor_contact_title, + dg.auditor_country, + dg.auditor_ein, + dg.auditor_email, + dg.auditor_firm_name, + dg.auditor_foreign_address, + dg.auditor_phone, + dg.auditor_state, + dg.auditor_zip, + dg.cognizant_agency, + dg.data_source, + dg.date_created, + dg.dollar_threshold, + dg.entity_type, + dg.fac_accepted_date, + dg.fy_end_date, + dg.fy_start_date, + dg.gaap_results, + dg.is_additional_ueis, + dg.is_aicpa_audit_guide_included, + dg.is_going_concern_included, + dg.is_internal_control_deficiency_disclosed, + dg.is_internal_control_material_weakness_disclosed, + dg.is_low_risk_auditee, + dg.is_material_noncompliance_disclosed, + dg.is_multiple_eins, + dg.is_public, + dg.is_secondary_auditors, + dg.is_sp_framework_required, + dg.number_months, + dg.oversight_agency, + dg.ready_for_certification_date, + dg.sp_framework_basis, + dg.sp_framework_opinions, + dg.submitted_date, + dg.total_amount_expended, + dg.type_audit_code, + -- + -- federal_award + -- + dfa.additional_award_identification, + dfa.amount_expended, + dfa.audit_report_type, + dfa.cluster_name, + dfa.cluster_total, + dfa.federal_agency_prefix, + dfa.federal_award_extension, + dfa.federal_program_name, + dfa.federal_program_total, + dfa.findings_count, + dfa.is_direct, + dfa.is_loan, + dfa.is_major, + dfa.is_passthrough_award, + dfa.loan_balance, + dfa.other_cluster_name, + dfa.passthrough_amount, + dfa.state_cluster_name, + -- + -- finding + -- + df.is_material_weakness, + df.is_modified_opinion, + df.is_other_findings, + df.is_other_matters, + df.is_questioned_costs, + df.is_repeat_finding, + df.is_significant_deficiency, + df.prior_finding_ref_numbers, + df.type_requirement, + -- + -- passthrough + -- + dp.passthrough_id, + dp.passthrough_name + FROM + public.dissemination_federalaward dfa + LEFT JOIN public_data_v1_0_0.general dg + ON dfa.report_id = dg.report_id + LEFT JOIN public.dissemination_finding df + ON dfa.report_id = df.report_id + AND dfa.award_reference = df.award_reference + LEFT JOIN public.dissemination_passthrough dp + ON dfa.report_id = dp.report_id + AND dfa.award_reference = dp.award_reference diff --git a/backend/tools/load_public_dissem_data/load_public_data_locally.sh b/backend/tools/load_public_dissem_data/load_public_data_locally.sh index b8ee813e46..881988a720 100755 --- a/backend/tools/load_public_dissem_data/load_public_data_locally.sh +++ b/backend/tools/load_public_dissem_data/load_public_data_locally.sh @@ -19,57 +19,38 @@ echo "Make sure you have disk space." echo "Sleeping for 4 seconds..." sleep 4 -cat << EOM -First, we cleanup the local filesystem. -This removes any temporary files from any -previous data loads -EOM - +# First, we cleanup the local filesystem. +# This removes any temporary files from any +# previous data loads rm -f /app/data/db_dissem_dump rm -rf /app/data/__MACOSX -cat << EOM -Next, we drop the public_data schema. -This is because we want to make sure it is -regenerated fresh. -EOM - +# Next, we drop the public_data schema. +# This is because we want to make sure it is +# regenerated fresh. psql $FAC_SNAPSHOT_URI -c "DROP SCHEMA IF EXISTS public_data_v1_0_0 CASCADE" -cat << EOM -Now, the schema for the public_data is -created. This provies a place for the tables to -land when we run sling -EOM - +# Now, the schema for the public_data is +# created. This provies a place for the tables to +# land when we run sling psql $FAC_SNAPSHOT_URI -c "CREATE SCHEMA IF NOT EXISTS public_data_v1_0_0" psql $FAC_SNAPSHOT_URI -c "CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_combined START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE" -echo < Date: Mon, 30 Sep 2024 09:37:34 -0400 Subject: [PATCH 06/89] Cleaining up --- backend/tools/sling_public_tables.sh | 9 -- backend/tools/util_startup.sh | 49 ++------ tools/load_public_dissem_data/.gitignore | 5 - .../load_public_data_locally.sh | 109 ------------------ 4 files changed, 12 insertions(+), 160 deletions(-) delete mode 100644 backend/tools/sling_public_tables.sh delete mode 100644 tools/load_public_dissem_data/.gitignore delete mode 100755 tools/load_public_dissem_data/load_public_data_locally.sh diff --git a/backend/tools/sling_public_tables.sh b/backend/tools/sling_public_tables.sh deleted file mode 100644 index fc4a6b27b4..0000000000 --- a/backend/tools/sling_public_tables.sh +++ /dev/null @@ -1,9 +0,0 @@ -source tools/util_startup.sh - -function sling_public_tables () { - # run_sql has its own gonogo built in. - run_sql $FAC_DB_URI 'dissemination/sql' 'sling' 'public_data_v1_0_0' 'teardown.sql' - $SLING_EXE run -r dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml - local result=$? - return $result -} diff --git a/backend/tools/util_startup.sh b/backend/tools/util_startup.sh index 923f52893e..357cb3df7f 100644 --- a/backend/tools/util_startup.sh +++ b/backend/tools/util_startup.sh @@ -1,13 +1,4 @@ -declare -a public_api_versions=( - "api_v1_0_3" - "api_v1_1_0" - "public_api_v2_0_0_alpha" - ) - -declare -a admin_api_versions=( - "admin_api_v1_1_0" - "admin_api_v1_1_1" - ) +source tools/variables.sh function startup_log { local tag="$1" @@ -34,6 +25,17 @@ function check_table_exists() { return $result } +function check_schema_exists () { + local db_uri="$1" + local schema_name="$2" + local result=$(psql $db_uri -qtAX -c "SELECT EXISTS(SELECT 1 FROM information_schema.schemata WHERE schema_name = '$schema_name');") + if [ "$result" = "t" ]; then + return 0; + else + return 1; + fi +} + function run_sql () { local db_uri="$1" local base_path="$2" @@ -44,30 +46,3 @@ function run_sql () { $PSQL_EXE $db_uri < ${base_path}/${location}/${api_version}/${sql_file} gonogo "run_sql < ${base_path}/${location}/${api_version}/${sql_file}" } - -function run_sql_for_public_apis () { - local location="$1" - local sql_file="$2" - local base_path='dissemination/sql' - - for api_version in "${public_api_versions[@]}" - do - if [ -f ${base_path}/${location}/${api_version}/${sql_file} ]; then - # $PSQL_EXE $FAC_DB_URI < ${base_path}/${location}/${api_version}/${sql_file} - run_sql $FAC_DB_URI $base_path $location $api_version $sql_file - else - echo "API FILE NOT FOUND/SKIPPED ${location}/${api_version}/${sql_file}" - fi - done - - # for api_version in "${admin_api_versions[@]}" - # do - # echo "VERSION $api_version" - # if [ -f ${base_path}/${location}/${api_version}/${sql_file} ]; then - # $PSQL_EXE $FAC_DB_URI < ${base_path}/${location}/${api_version}/${sql_file} - # gonogo "psql < ${location}/${api_version}/${sql_file}" - # else - # echo "API FILE NOT FOUND/SKIPPED ${location}/${api_version}/${sql_file}" - # fi - # done -} diff --git a/tools/load_public_dissem_data/.gitignore b/tools/load_public_dissem_data/.gitignore deleted file mode 100644 index b675415e50..0000000000 --- a/tools/load_public_dissem_data/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -*.csv -*.zip -*.tar -*.gz -*.json diff --git a/tools/load_public_dissem_data/load_public_data_locally.sh b/tools/load_public_dissem_data/load_public_data_locally.sh deleted file mode 100755 index 10a7d9287b..0000000000 --- a/tools/load_public_dissem_data/load_public_data_locally.sh +++ /dev/null @@ -1,109 +0,0 @@ -#!/bin/bash - -export CGOV_UTIL_VERSION=v0.1.8 -export FAC_DB_URI=postgresql://postgres@localhost:5432/postgres?sslmode=disable -export FAC_SNAPSHOT_URI=postgresql://postgres@localhost:5431/postgres?sslmode=disable - - -function check_table_exists() { - local db_uri="$1" - local dbname="$2" - psql $db_uri -c "SELECT '$dbname'::regclass" >/dev/null 2>&1 - result=$? - return $result -} - -echo "This will unzip ~3.3GB of data, and load it into a local FAC." -echo "Make sure the FAC is running." -echo "Make sure you have disk space." -echo "Sleeping for 4 seconds..." -sleep 4 - -echo < Date: Mon, 30 Sep 2024 10:32:17 -0400 Subject: [PATCH 07/89] Updating .profile to match run.sh --- backend/.profile | 10 ++++++++-- backend/Makefile | 11 +++-------- backend/run.sh | 5 ++--- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/backend/.profile b/backend/.profile index 75952618ad..6c4c89023f 100644 --- a/backend/.profile +++ b/backend/.profile @@ -10,9 +10,9 @@ source tools/util_startup.sh # for local envs (LOCAL or TESTING) and cloud.gov source tools/setup_env.sh source tools/api_teardown.sh +source tools/build_indexes.sh source tools/migrate_app_tables.sh source tools/api_standup.sh -source tools/run_collectstatic.sh source tools/seed_cog_baseline.sh ##### @@ -40,6 +40,12 @@ if [[ "$CF_INSTANCE_INDEX" == 0 ]]; then api_standup gonogo "api_standup" + ##### + # BUILD INDEXES + # Builds indexes on the API tables in fac-snapshot-db + build_indexes + gonogo "build_indexes" + ##### # COLLECT STATIC # Do Django things with static files. @@ -54,4 +60,4 @@ if [[ "$CF_INSTANCE_INDEX" == 0 ]]; then fi # Make psql usable by scripts, for debugging, etc. -alias psql='/home/vcap/deps/0/apt/usr/lib/postgresql/*/bin/psql' \ No newline at end of file +alias psql='/home/vcap/deps/0/apt/usr/lib/postgresql/*/bin/psql' diff --git a/backend/Makefile b/backend/Makefile index a2d795507e..97d62c621a 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -109,11 +109,6 @@ ghcr-nctest: ghcr-lint: docker compose -f docker-compose-web.yml run web bash -c 'flake8 && black --check . && bandit -c pyproject.toml -r . && python -m pip install types-pytz && mypy . && djlint .' -docker-clean: - -docker compose down - -docker rm -f $(shell docker ps -a -q) - -docker volume rm $(shell docker volume ls -q) - remove-coverage: -rm -f .coverage.* @@ -121,13 +116,13 @@ compose-down: docker compose down remove-processes: - docker rm -f $(docker ps -a -q) + docker rm -f $$(docker ps -a -q) remove-volumes: - -docker volume ls -q | docker volume rm + -docker volume rm $$(docker volume ls -q) remove-dangling-volumes: remove-volumes - -docker volume ls -q --filter dangling=true | docker volume rm + -docker volume rm $$(docker volume ls -q --filter dangling=true) system-prune: -docker system prune -f diff --git a/backend/run.sh b/backend/run.sh index eb20562efd..0024e08a82 100755 --- a/backend/run.sh +++ b/backend/run.sh @@ -10,7 +10,6 @@ source tools/util_startup.sh # for local envs (LOCAL or TESTING) and cloud.gov source tools/setup_env.sh source tools/api_teardown.sh -source tools/setup_public_table_placeholders.sh source tools/build_indexes.sh source tools/migrate_app_tables.sh source tools/api_standup.sh @@ -40,8 +39,8 @@ api_standup gonogo "api_standup" ##### -# API STANDUP -# Standup the API, which may depend on migration changes +# BUILD INDEXES +# Builds indexes on the API tables in fac-snapshot-db build_indexes gonogo "build_indexes" From a31eea0a3e319e89597284f30d8358b9080679d9 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Wed, 2 Oct 2024 07:13:38 -0400 Subject: [PATCH 08/89] Bringup for the Admin API Adds the access tables to its own startup. Admin API relies on it, or it won't start. --- .../standup.sql} | 0 .../api/admin_api_access_tables/teardown.sql | 2 + .../sql/api/admin_api_v1_1_1/base.sql | 29 --- .../api/admin_api_v1_1_1/create_schema.sql | 56 ------ .../sql/api/admin_api_v1_1_1/create_views.sql | 92 --------- .../sql/api/admin_api_v1_1_1/drop_schema.sql | 9 - .../sql/api/admin_api_v1_1_1/drop_views.sql | 8 - .../{create_functions.sql => standup.sql} | 190 ++++++++++++++++-- .../sql/api/admin_api_v1_1_1/teardown.sql | 8 + .../sql/api/api_v1_1_0/standup.sql | 6 + 10 files changed, 192 insertions(+), 208 deletions(-) rename backend/dissemination/sql/api/{admin_api_v1_1_1/create_access_tables.sql => admin_api_access_tables/standup.sql} (100%) create mode 100644 backend/dissemination/sql/api/admin_api_access_tables/teardown.sql delete mode 100644 backend/dissemination/sql/api/admin_api_v1_1_1/base.sql delete mode 100644 backend/dissemination/sql/api/admin_api_v1_1_1/create_schema.sql delete mode 100644 backend/dissemination/sql/api/admin_api_v1_1_1/create_views.sql delete mode 100644 backend/dissemination/sql/api/admin_api_v1_1_1/drop_schema.sql delete mode 100644 backend/dissemination/sql/api/admin_api_v1_1_1/drop_views.sql rename backend/dissemination/sql/api/admin_api_v1_1_1/{create_functions.sql => standup.sql} (74%) create mode 100644 backend/dissemination/sql/api/admin_api_v1_1_1/teardown.sql diff --git a/backend/dissemination/sql/api/admin_api_v1_1_1/create_access_tables.sql b/backend/dissemination/sql/api/admin_api_access_tables/standup.sql similarity index 100% rename from backend/dissemination/sql/api/admin_api_v1_1_1/create_access_tables.sql rename to backend/dissemination/sql/api/admin_api_access_tables/standup.sql diff --git a/backend/dissemination/sql/api/admin_api_access_tables/teardown.sql b/backend/dissemination/sql/api/admin_api_access_tables/teardown.sql new file mode 100644 index 0000000000..e911176bd9 --- /dev/null +++ b/backend/dissemination/sql/api/admin_api_access_tables/teardown.sql @@ -0,0 +1,2 @@ + +DROP TABLE IF EXISTS support_administrative_key_uuids; diff --git a/backend/dissemination/sql/api/admin_api_v1_1_1/base.sql b/backend/dissemination/sql/api/admin_api_v1_1_1/base.sql deleted file mode 100644 index dedabe0cb7..0000000000 --- a/backend/dissemination/sql/api/admin_api_v1_1_1/base.sql +++ /dev/null @@ -1,29 +0,0 @@ -DO -$do$ -BEGIN - IF EXISTS ( - SELECT FROM pg_catalog.pg_roles - WHERE rolname = 'authenticator') THEN - RAISE NOTICE 'Role "authenticator" already exists. Skipping.'; - ELSE - CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; - END IF; -END -$do$; - -DO -$do$ -BEGIN - IF EXISTS ( - SELECT FROM pg_catalog.pg_roles - WHERE rolname = 'api_fac_gov') THEN - RAISE NOTICE 'Role "api_fac_gov" already exists. Skipping.'; - ELSE - CREATE ROLE api_fac_gov NOLOGIN; - END IF; -END -$do$; - -GRANT api_fac_gov TO authenticator; - -NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/api/admin_api_v1_1_1/create_schema.sql b/backend/dissemination/sql/api/admin_api_v1_1_1/create_schema.sql deleted file mode 100644 index 9eff082bfe..0000000000 --- a/backend/dissemination/sql/api/admin_api_v1_1_1/create_schema.sql +++ /dev/null @@ -1,56 +0,0 @@ -begin; - -do -$$ -begin - DROP SCHEMA IF EXISTS admin_api_v1_1_1 CASCADE; - DROP SCHEMA IF EXISTS admin_api_v1_1_1_functions CASCADE; - - if not exists (select schema_name from information_schema.schemata where schema_name = 'admin_api_v1_1_1') then - create schema admin_api_v1_1_1; - create schema admin_api_v1_1_1_functions; - - grant usage on schema admin_api_v1_1_1_functions to api_fac_gov; - - -- Grant access to tables and views - alter default privileges - in schema admin_api_v1_1_1 - grant select - -- this includes views - on tables - to api_fac_gov; - - -- Grant access to sequences, if we have them - grant usage on schema admin_api_v1_1_1 to api_fac_gov; - grant select, usage on all sequences in schema admin_api_v1_1_1 to api_fac_gov; - alter default privileges - in schema admin_api_v1_1_1 - grant select, usage - on sequences - to api_fac_gov; - - -- The admin API needs to be able to write user permissions. - -- This is so we can add and remove people who will have tribal data access - -- via the administrative API. - GRANT INSERT, SELECT, DELETE on public.users_userpermission to api_fac_gov; - -- We need to be able to look up slugs and turn them into permission IDs. - GRANT SELECT on public.users_permission to api_fac_gov; - -- It also needs to be able to log events. - GRANT INSERT on public.support_adminapievent to api_fac_gov; - -- And, it wants to read the UUIDs of administrative keys - GRANT SELECT ON public.support_administrative_key_uuids TO api_fac_gov; - -- We want to see data in flight as admins. - GRANT SELECT ON public.audit_singleauditchecklist TO api_fac_gov; - - GRANT INSERT, SELECT, DELETE on public.dissemination_tribalapiaccesskeyids to api_fac_gov; - GRANT INSERT on public.dissemination_onetimeaccess to api_fac_gov; - end if; -end -$$ -; - -commit; - -notify pgrst, - 'reload schema'; - diff --git a/backend/dissemination/sql/api/admin_api_v1_1_1/create_views.sql b/backend/dissemination/sql/api/admin_api_v1_1_1/create_views.sql deleted file mode 100644 index 6c84abf4d8..0000000000 --- a/backend/dissemination/sql/api/admin_api_v1_1_1/create_views.sql +++ /dev/null @@ -1,92 +0,0 @@ - -begin; - - ---------------------------------------- --- accesses ---------------------------------------- --- public.audit_access definition - --- Drop table - --- DROP TABLE public.audit_access; - -CREATE OR REPLACE VIEW admin_api_v1_1_1.audit_access AS - SELECT - aa.role, - aa.fullname, - aa.email, - aa.sac_id, - aa.user_id - FROM - public.audit_access aa - WHERE - admin_api_v1_1_1_functions.has_admin_data_access('READ') - ORDER BY aa.id -; - -CREATE OR REPLACE VIEW admin_api_v1_1_1.singleauditchecklist AS - SELECT - sac.id, - sac.date_created, - sac.submission_status, - sac.data_source, - sac.transition_name, - sac.transition_date, - sac.report_id, - sac.audit_type, - sac.general_information, - sac.audit_information, - sac.federal_awards, - sac.corrective_action_plan, - sac.findings_text, - sac.findings_uniform_guidance, - sac.additional_ueis, - sac.additional_eins, - sac.secondary_auditors, - sac.notes_to_sefa, - sac.auditor_certification, - sac.auditee_certification, - sac.tribal_data_consent, - sac.cognizant_agency, - sac.oversight_agency, - sac.submitted_by_id - from - public.audit_singleauditchecklist sac - where - admin_api_v1_1_1_functions.has_admin_data_access('READ') - order by sac.id -; - -CREATE OR REPLACE VIEW admin_api_v1_1_1.tribal_access AS - SELECT - uup.email, - up.slug as permission - FROM - users_userpermission uup, - users_permission up - WHERE - (uup.permission_id = up.id) - AND (up.slug = 'read-tribal') - AND admin_api_v1_1_1_functions.has_admin_data_access('READ') - ORDER BY uup.id -; - -CREATE OR REPLACE VIEW admin_api_v1_1_1.admin_api_events AS - SELECT - ae.timestamp, - ae.api_key_uuid, - ae.event, - ae.event_data - FROM - public.support_adminapievent ae - WHERE - admin_api_v1_1_1_functions.has_admin_data_access('READ') - ORDER BY ae.id -; - - -commit; - -notify pgrst, - 'reload schema'; diff --git a/backend/dissemination/sql/api/admin_api_v1_1_1/drop_schema.sql b/backend/dissemination/sql/api/admin_api_v1_1_1/drop_schema.sql deleted file mode 100644 index 0f53c84a81..0000000000 --- a/backend/dissemination/sql/api/admin_api_v1_1_1/drop_schema.sql +++ /dev/null @@ -1,9 +0,0 @@ - -begin; - -DROP SCHEMA IF EXISTS admin_api_v1_1_1 CASCADE; - -commit; - -notify pgrst, - 'reload schema'; diff --git a/backend/dissemination/sql/api/admin_api_v1_1_1/drop_views.sql b/backend/dissemination/sql/api/admin_api_v1_1_1/drop_views.sql deleted file mode 100644 index 6185d9b83d..0000000000 --- a/backend/dissemination/sql/api/admin_api_v1_1_1/drop_views.sql +++ /dev/null @@ -1,8 +0,0 @@ -begin; - - drop view if exists admin_api_v1_1_1.audit_access; - -commit; - -notify pgrst, - 'reload schema'; diff --git a/backend/dissemination/sql/api/admin_api_v1_1_1/create_functions.sql b/backend/dissemination/sql/api/admin_api_v1_1_1/standup.sql similarity index 74% rename from backend/dissemination/sql/api/admin_api_v1_1_1/create_functions.sql rename to backend/dissemination/sql/api/admin_api_v1_1_1/standup.sql index 77b2b2a303..aca364f30e 100644 --- a/backend/dissemination/sql/api/admin_api_v1_1_1/create_functions.sql +++ b/backend/dissemination/sql/api/admin_api_v1_1_1/standup.sql @@ -1,17 +1,87 @@ --- WARNING --- Under PostgreSQL 12, the functions below work. --- Under PostgreSQL 14, these will break. --- --- Note the differences: --- --- raise info 'Works under PostgreSQL 12'; --- raise info 'request.header.x-magic %', (SELECT current_setting('request.header.x-magic', true)); --- raise info 'request.jwt.claim.expires %', (SELECT current_setting('request.jwt.claim.expires', true)); --- raise info 'Works under PostgreSQL 14'; --- raise info 'request.headers::json->>x-magic %', (SELECT current_setting('request.headers', true)::json->>'x-magic'); --- raise info 'request.jwt.claims::json->expires %', (SELECT current_setting('request.jwt.claims', true)::json->>'expires'); --- --- To quote the work of Dav Pilkey, "remember this now." +DO +$do$ +BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = 'authenticator') THEN + RAISE NOTICE 'Role "authenticator" already exists. Skipping.'; + ELSE + CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; + END IF; +END +$do$; + +DO +$do$ +BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = 'api_fac_gov') THEN + RAISE NOTICE 'Role "api_fac_gov" already exists. Skipping.'; + ELSE + CREATE ROLE api_fac_gov NOLOGIN; + END IF; +END +$do$; + +GRANT api_fac_gov TO authenticator; + +NOTIFY pgrst, 'reload schema'; +begin; + +do +$$ +begin + DROP SCHEMA IF EXISTS admin_api_v1_1_1 CASCADE; + DROP SCHEMA IF EXISTS admin_api_v1_1_1_functions CASCADE; + + if not exists (select schema_name from information_schema.schemata where schema_name = 'admin_api_v1_1_1') then + create schema admin_api_v1_1_1; + create schema admin_api_v1_1_1_functions; + + grant usage on schema admin_api_v1_1_1_functions to api_fac_gov; + + -- Grant access to tables and views + alter default privileges + in schema admin_api_v1_1_1 + grant select + -- this includes views + on tables + to api_fac_gov; + + -- Grant access to sequences, if we have them + grant usage on schema admin_api_v1_1_1 to api_fac_gov; + grant select, usage on all sequences in schema admin_api_v1_1_1 to api_fac_gov; + alter default privileges + in schema admin_api_v1_1_1 + grant select, usage + on sequences + to api_fac_gov; + + -- The admin API needs to be able to write user permissions. + -- This is so we can add and remove people who will have tribal data access + -- via the administrative API. + GRANT INSERT, SELECT, DELETE on public.users_userpermission to api_fac_gov; + -- We need to be able to look up slugs and turn them into permission IDs. + GRANT SELECT on public.users_permission to api_fac_gov; + -- It also needs to be able to log events. + GRANT INSERT on public.support_adminapievent to api_fac_gov; + -- And, it wants to read the UUIDs of administrative keys + GRANT SELECT ON public.support_administrative_key_uuids TO api_fac_gov; + -- We want to see data in flight as admins. + GRANT SELECT ON public.audit_singleauditchecklist TO api_fac_gov; + + GRANT INSERT, SELECT, DELETE on public.dissemination_tribalapiaccesskeyids to api_fac_gov; + GRANT INSERT on public.dissemination_onetimeaccess to api_fac_gov; + end if; +end +$$ +; + +commit; + +notify pgrst, + 'reload schema'; begin; @@ -387,3 +457,95 @@ $remove_tribal_api_key_access$ LANGUAGE plpgsql; commit; NOTIFY pgrst, 'reload schema'; + +begin; + + +--------------------------------------- +-- accesses +--------------------------------------- +-- public.audit_access definition + +-- Drop table + +-- DROP TABLE public.audit_access; + +CREATE OR REPLACE VIEW admin_api_v1_1_1.audit_access AS + SELECT + aa.role, + aa.fullname, + aa.email, + aa.sac_id, + aa.user_id + FROM + public.audit_access aa + WHERE + admin_api_v1_1_1_functions.has_admin_data_access('READ') + ORDER BY aa.id +; + +CREATE OR REPLACE VIEW admin_api_v1_1_1.singleauditchecklist AS + SELECT + sac.id, + sac.date_created, + sac.submission_status, + sac.data_source, + sac.transition_name, + sac.transition_date, + sac.report_id, + sac.audit_type, + sac.general_information, + sac.audit_information, + sac.federal_awards, + sac.corrective_action_plan, + sac.findings_text, + sac.findings_uniform_guidance, + sac.additional_ueis, + sac.additional_eins, + sac.secondary_auditors, + sac.notes_to_sefa, + sac.auditor_certification, + sac.auditee_certification, + sac.tribal_data_consent, + sac.cognizant_agency, + sac.oversight_agency, + sac.submitted_by_id + from + public.audit_singleauditchecklist sac + where + admin_api_v1_1_1_functions.has_admin_data_access('READ') + order by sac.id +; + +CREATE OR REPLACE VIEW admin_api_v1_1_1.tribal_access AS + SELECT + uup.email, + up.slug as permission + FROM + users_userpermission uup, + users_permission up + WHERE + (uup.permission_id = up.id) + AND (up.slug = 'read-tribal') + AND admin_api_v1_1_1_functions.has_admin_data_access('READ') + ORDER BY uup.id +; + +CREATE OR REPLACE VIEW admin_api_v1_1_1.admin_api_events AS + SELECT + ae.timestamp, + ae.api_key_uuid, + ae.event, + ae.event_data + FROM + public.support_adminapievent ae + WHERE + admin_api_v1_1_1_functions.has_admin_data_access('READ') + ORDER BY ae.id +; + + +commit; + +notify pgrst, + 'reload schema'; diff --git a/backend/dissemination/sql/api/admin_api_v1_1_1/teardown.sql b/backend/dissemination/sql/api/admin_api_v1_1_1/teardown.sql new file mode 100644 index 0000000000..63138d39c9 --- /dev/null +++ b/backend/dissemination/sql/api/admin_api_v1_1_1/teardown.sql @@ -0,0 +1,8 @@ + +BEGIN; + +DROP SCHEMA IF EXISTS admin_api_v1_1_1 CASCADE; + +COMMIT; + +notify pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/api/api_v1_1_0/standup.sql b/backend/dissemination/sql/api/api_v1_1_0/standup.sql index 62ba940193..053d9288f1 100644 --- a/backend/dissemination/sql/api/api_v1_1_0/standup.sql +++ b/backend/dissemination/sql/api/api_v1_1_0/standup.sql @@ -102,6 +102,12 @@ notify pgrst, -- -- To quote the work of Dav Pilkey, "remember this now." +----------------------------------------------------- +-- FUNCTIONS +----------------------------------------------------- + + +notify pgrst, 'reload schema'; CREATE OR REPLACE FUNCTION api_v1_1_0_functions.get_header(item text) RETURNS text AS $get_header$ From b3e8523d36814cb4a7738fbc481d8f1055d1e8ff Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Wed, 2 Oct 2024 07:16:49 -0400 Subject: [PATCH 09/89] Cleanup, missed files. --- .../api/public_api_v2_0_0_alpha/standup.sql | 71 ++- .../public_data_v1_0_0.yaml | 47 ++ .../tools/load_public_dissem_data/.gitignore | 1 + .../load_public_dissem_data/data/sling.yaml | 499 ------------------ backend/tools/variables.sh | 8 +- 5 files changed, 120 insertions(+), 506 deletions(-) delete mode 100644 backend/tools/load_public_dissem_data/data/sling.yaml diff --git a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/standup.sql b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/standup.sql index e0a36042de..bbeed84011 100644 --- a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/standup.sql +++ b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/standup.sql @@ -1,3 +1,6 @@ +----------------------------------------------------- +-- ROLES +----------------------------------------------------- DO $do$ BEGIN @@ -29,6 +32,9 @@ GRANT api_fac_gov TO authenticator; NOTIFY pgrst, 'reload schema'; begin; +----------------------------------------------------- +-- PERMISSIONS +----------------------------------------------------- do $$ BEGIN @@ -61,13 +67,68 @@ $$ COMMIT; -notify pgrst, - 'reload schema'; +notify pgrst, 'reload schema'; + +----------------------------------------------------- +-- FUNCTIONS +----------------------------------------------------- +CREATE OR REPLACE FUNCTION public_api_v2_0_0_alpha.rows_per_batch() + RETURNS integer + LANGUAGE sql IMMUTABLE PARALLEL SAFE AS + 'SELECT 20000'; + +create or replace function public_api_v2_0_0_alpha_functions.batches (diss_table text) +returns integer +as $batches$ +declare count integer; +declare RPB integer; +begin + select public_api_v2_0_0_alpha.rows_per_batch() into RPB; + case + when diss_table = 'additional_eins' then + select div(count(*), RPB) into count + from public_data_v1_0_0.additional_eins; + when diss_table = 'additional_ueis' then + select div(count(*), RPB) into count + from public_data_v1_0_0.additional_ueis; + when diss_table = 'combined' then + select div(count(*), RPB) into count + from public_data_v1_0_0.combined; + when diss_table = 'corrective_action_plans' then + select div(count(*), RPB) into count + from public_data_v1_0_0.corrective_action_plans; + when diss_table = 'federal_awards' then + select div(count(*), RPB) into count + from public_data_v1_0_0.federal_awards; + when diss_table = 'findings_text' then + select div(count(*), RPB) into count + from public_data_v1_0_0.findings_text; + when diss_table = 'findings' then + select div(count(*), RPB) into count + from public_data_v1_0_0.findings; + when diss_table = 'general' then + select div(count(*), RPB) into count + from public_data_v1_0_0.general; + when diss_table = 'notes_to_sefa' then + select div(count(*), RPB) into count + from public_data_v1_0_0.notes_to_sefa; + when diss_table = 'passthrough' then + select div(count(*), RPB) into count + from public_data_v1_0_0.passthrough; + when diss_table = 'secondary_auditors' then + select div(count(*), RPB) into count + from public_data_v1_0_0.secondary_auditors; + else + count := 0; + end case; + RETURN count; +end +$batches$ +language plpgsql; + +notify pgrst, 'reload schema'; --- Under the new approach, we don't need --- any functions here. -NOTIFY pgrst, 'reload schema'; BEGIN; CREATE VIEW public_api_v2_0_0_alpha.additional_eins AS diff --git a/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml b/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml index 093de28a6b..48f6fe2e25 100644 --- a/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml +++ b/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml @@ -497,3 +497,50 @@ streams: LEFT JOIN public.dissemination_passthrough dp ON dfa.report_id = dp.report_id AND dfa.award_reference = dp.award_reference + + # This looks at the tables after they have been updated, and + # generates metadata that agencies use to determine if they + # have completely pulled all of the data in their systems. + public_data_v1_0_0.metadata: + mode: full-refresh + object: public_data_v1_0_0.metadata_v2_0_0_a + sql: | + SELECT 'additional_eins' + AS table, COUNT(*), public_api_v2_0_0_alpha_functions.batches('additional_eins') + FROM public_data_v1_0_0.additional_eins + UNION + SELECT 'additional_ueis' + AS table, COUNT(*), public_api_v2_0_0_alpha_functions.batches('additional_ueis') + FROM public_data_v1_0_0.additional_ueis + UNION + SELECT 'combined' + AS table, COUNT(*), public_api_v2_0_0_alpha_functions.batches('combined') + FROM public_data_v1_0_0.combined + UNION + SELECT 'federal_awards' + AS table, COUNT(*), public_api_v2_0_0_alpha_functions.batches('federal_awards') + FROM public_data_v1_0_0.federal_awards + UNION + SELECT 'findings_text' + AS table, COUNT(*), public_api_v2_0_0_alpha_functions.batches('findings_text') + FROM public_data_v1_0_0.findings_text + UNION + SELECT 'findings' + AS table, COUNT(*), public_api_v2_0_0_alpha_functions.batches('findings') + FROM public_data_v1_0_0.findings + UNION + SELECT 'general' + AS table, COUNT(*), public_api_v2_0_0_alpha_functions.batches('general') + FROM public_data_v1_0_0.general + UNION + SELECT 'notes_to_sefa' + AS table, COUNT(*), public_api_v2_0_0_alpha_functions.batches('notes_to_sefa') + FROM public_data_v1_0_0.notes_to_sefa + UNION + SELECT 'passthrough' + AS table, COUNT(*), public_api_v2_0_0_alpha_functions.batches('passthrough') + FROM public_data_v1_0_0.passthrough + UNION + SELECT 'secondary_auditors' + AS table, COUNT(*), public_api_v2_0_0_alpha_functions.batches('secondary_auditors') + FROM public_data_v1_0_0.secondary_auditors diff --git a/backend/tools/load_public_dissem_data/.gitignore b/backend/tools/load_public_dissem_data/.gitignore index b5f6e4ed48..6c05ad4054 100644 --- a/backend/tools/load_public_dissem_data/.gitignore +++ b/backend/tools/load_public_dissem_data/.gitignore @@ -4,3 +4,4 @@ *.gz db_dissem_dump* __MACOSX +data/sling.yaml diff --git a/backend/tools/load_public_dissem_data/data/sling.yaml b/backend/tools/load_public_dissem_data/data/sling.yaml deleted file mode 100644 index 093de28a6b..0000000000 --- a/backend/tools/load_public_dissem_data/data/sling.yaml +++ /dev/null @@ -1,499 +0,0 @@ -# This uses an env var called PG to -# set the connection string. -source: FAC_SNAPSHOT_URI -target: FAC_SNAPSHOT_URI -env: - SLING_ALLOW_EMPTY: true - -# Handy -# SELECT column_name -# FROM information_schema.columns -# WHERE table_schema = 'public' -# AND table_name = 'dissemination_general' -# AND column_name NOT IN ('id'); - -streams: - # The first table we create is the general table. All of the - # general table is public data, so we copy it wholesale. - public.dissemination_general: - mode: incremental - update_key: id - object: public_data_v1_0_0.general - sql: | - -- All of the general table is public data. - SELECT - gen.id, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.agencies_with_prior_findings, - gen.audit_period_covered, - gen.audit_type, - gen.auditee_address_line_1, - gen.auditee_certified_date, - gen.auditee_certify_name, - gen.auditee_certify_title, - gen.auditee_city, - gen.auditee_contact_name, - gen.auditee_contact_title, - gen.auditee_ein, - gen.auditee_email, - gen.auditee_name, - gen.auditee_phone, - gen.auditee_state, - gen.auditee_zip, - gen.auditor_address_line_1, - gen.auditor_certified_date, - gen.auditor_certify_name, - gen.auditor_certify_title, - gen.auditor_city, - gen.auditor_contact_name, - gen.auditor_contact_title, - gen.auditor_country, - gen.auditor_ein, - gen.auditor_email, - gen.auditor_firm_name, - gen.auditor_foreign_address, - gen.auditor_phone, - gen.auditor_state, - gen.auditor_zip, - gen.cognizant_agency, - gen.data_source, - gen.date_created, - gen.dollar_threshold, - gen.entity_type, - gen.fac_accepted_date, - gen.fy_end_date, - gen.fy_start_date, - gen.gaap_results, - gen.is_additional_ueis, - gen.is_aicpa_audit_guide_included, - gen.is_going_concern_included, - gen.is_internal_control_deficiency_disclosed, - gen.is_internal_control_material_weakness_disclosed, - gen.is_low_risk_auditee, - gen.is_material_noncompliance_disclosed, - CASE EXISTS - ( - SELECT - ein.report_id - FROM - dissemination_additionalein ein - WHERE - ein.report_id = gen.report_id - ) - WHEN FALSE THEN 'No' - ELSE 'Yes' - END AS is_multiple_eins, - gen.is_public, - CASE EXISTS - ( - SELECT - aud.report_id - FROM - dissemination_secondaryauditor aud - WHERE - aud.report_id = gen.report_id - ) - WHEN FALSE THEN 'No' - ELSE 'Yes' - END AS is_secondary_auditors, - gen.is_sp_framework_required, - gen.number_months, - gen.oversight_agency, - gen.ready_for_certification_date, - gen.sp_framework_basis, - gen.sp_framework_opinions, - gen.submitted_date, - gen.total_amount_expended, - gen.type_audit_code - FROM - public.dissemination_general gen - WHERE {incremental_where_cond} - ORDER BY gen.id - - # All additional EINs are public. - public.additional_eins: - mode: incremental - update_key: ein.id - object: public_data_v1_0_0.additional_eins - sql: | - SELECT - ein.id, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - ein.additional_ein - FROM - public_data_v1_0_0.general gen, - public.dissemination_additionalein ein - WHERE - gen.report_id = ein.report_id - AND - {incremental_where_cond} - ORDER BY ein.id - - # All of the additional UEI info is public info. - public.dissemination_additional_ueis: - mode: incremental - update_key: uei.id - object: public_data_v1_0_0.additional_ueis - sql: | - SELECT - uei.id, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - uei.additional_uei - FROM - public_data_v1_0_0.general gen, - public.dissemination_additionaluei uei - WHERE - gen.report_id = uei.report_id - AND - {incremental_where_cond} - ORDER BY uei.id - - # Corrective action plans are NOT always public. - public.dissemination_corrective_action_plan: - mode: incremental - update_key: cap.id - object: public_data_v1_0_0.corrective_action_plans - sql: | - SELECT - cap.id, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - cap.contains_chart_or_table, - cap.finding_ref_number, - cap.planned_action - FROM - public_data_v1_0_0.general gen, - public.dissemination_captext cap - WHERE - cap.report_id = gen.report_id - AND - -- Only include the public corrective action plans. - gen.is_public = true - AND - {incremental_where_cond} - ORDER BY cap.id - - # All Federal award data is public. - public.dissemination_federalaward: - mode: incremental - update_key: award.id - object: public_data_v1_0_0.federal_awards - sql: | - SELECT - award.id, - dg.report_id, - dg.auditee_uei, - dg.audit_year, - dg.fac_accepted_date, - concat(award.federal_agency_prefix,'.',award.federal_award_extension) as aln, - --- - award.additional_award_identification, - award.amount_expended, - award.audit_report_type, - award.award_reference, - award.cluster_name, - award.cluster_total, - award.federal_agency_prefix, - award.federal_award_extension, - award.federal_program_name, - award.federal_program_total, - award.findings_count, - award.is_direct, - award.is_loan, - award.is_major, - award.is_passthrough_award, - award.loan_balance, - award.other_cluster_name, - award.passthrough_amount, - award.state_cluster_name - FROM - public_data_v1_0_0.general dg, - public.dissemination_federalaward award - WHERE - award.report_id = dg.report_id - AND - {incremental_where_cond} - ORDER BY award.id - - # The findings table is public. - public.dissemination_findings: - mode: incremental - update_key: finding.id - object: public_data_v1_0_0.findings - sql: | - SELECT - finding.id, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - finding.award_reference, - finding.is_material_weakness, - finding.is_modified_opinion, - finding.is_other_findings, - finding.is_other_matters, - finding.is_questioned_costs, - finding.is_repeat_finding, - finding.is_significant_deficiency, - finding.prior_finding_ref_numbers, - finding.reference_number, - finding.type_requirement - FROM - public_data_v1_0_0.general gen, - public.dissemination_finding finding - WHERE - finding.report_id = gen.report_id - AND - {incremental_where_cond} - ORDER BY finding.id - - # Findings text is NOT always public. - public.dissemination_findingstext: - mode: incremental - update_key: ft.id - object: public_data_v1_0_0.findings_text - sql: | - SELECT - ft.id, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - ft.finding_ref_number, - ft.contains_chart_or_table, - ft.finding_text - FROM - public_data_v1_0_0.general gen, - public.dissemination_findingtext ft - WHERE - ft.report_id = gen.report_id - AND - gen.is_public = true - AND - {incremental_where_cond} - ORDER BY ft.id - - # The notes to SEFA are NOT all public. - public.dissemination_notes: - mode: incremental - update_key: note.id - object: public_data_v1_0_0.notes_to_sefa - sql: | - SELECT - note.id, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - note.accounting_policies, - note.contains_chart_or_table, - note.content, - note.is_minimis_rate_used, - note.note_title as title, - note.rate_explained - FROM - public_data_v1_0_0.general gen, - public.dissemination_note note - WHERE - note.report_id = gen.report_id - AND - -- Some notes are not public. - gen.is_public = true - AND - {incremental_where_cond} - ORDER BY note.id - - # All passthrough information is public. - public.dissemination_passthrough: - mode: incremental - update_key: pass.id - object: public_data_v1_0_0.passthrough - sql: | - SELECT - pass.id, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - pass.award_reference, - pass.passthrough_id, - pass.passthrough_name - FROM - public_data_v1_0_0.general gen, - public.dissemination_passthrough pass - WHERE - gen.report_id = pass.report_id - AND - {incremental_where_cond} - ORDER BY pass.id - - # All secondary auditor info is public. - public.secondary_auditors: - mode: incremental - update_key: sa.id - object: public_data_v1_0_0.secondary_auditors - sql: | - SELECT - sa.id, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - sa.address_city, - sa.address_state, - sa.address_street, - sa.address_zipcode, - sa.auditor_ein, - sa.auditor_name, - sa.contact_email, - sa.contact_name, - sa.contact_phone, - sa.contact_title - FROM - public_data_v1_0_0.general gen, - public.dissemination_secondaryauditor sa - WHERE - sa.report_id = gen.report_id - AND - {incremental_where_cond} - ORDER BY sa.id - - public.combined: - mode: full-refresh - object: public_data_v1_0_0.combined - sql: | - SELECT - nextval('public_data_v1_0_0.seq_combined') AS id, - dg.report_id, - dfa.award_reference, - df.reference_number, - concat(dfa.federal_agency_prefix,'.',dfa.federal_award_extension) as aln, - -- - -- general - -- - dg.auditee_uei, - dg.audit_year, - dg.agencies_with_prior_findings, - dg.audit_period_covered, - dg.audit_type, - dg.auditee_address_line_1, - dg.auditee_certified_date, - dg.auditee_certify_name, - dg.auditee_certify_title, - dg.auditee_city, - dg.auditee_contact_name, - dg.auditee_contact_title, - dg.auditee_ein, - dg.auditee_email, - dg.auditee_name, - dg.auditee_phone, - dg.auditee_state, - dg.auditee_zip, - dg.auditor_address_line_1, - dg.auditor_certified_date, - dg.auditor_certify_name, - dg.auditor_certify_title, - dg.auditor_city, - dg.auditor_contact_name, - dg.auditor_contact_title, - dg.auditor_country, - dg.auditor_ein, - dg.auditor_email, - dg.auditor_firm_name, - dg.auditor_foreign_address, - dg.auditor_phone, - dg.auditor_state, - dg.auditor_zip, - dg.cognizant_agency, - dg.data_source, - dg.date_created, - dg.dollar_threshold, - dg.entity_type, - dg.fac_accepted_date, - dg.fy_end_date, - dg.fy_start_date, - dg.gaap_results, - dg.is_additional_ueis, - dg.is_aicpa_audit_guide_included, - dg.is_going_concern_included, - dg.is_internal_control_deficiency_disclosed, - dg.is_internal_control_material_weakness_disclosed, - dg.is_low_risk_auditee, - dg.is_material_noncompliance_disclosed, - dg.is_multiple_eins, - dg.is_public, - dg.is_secondary_auditors, - dg.is_sp_framework_required, - dg.number_months, - dg.oversight_agency, - dg.ready_for_certification_date, - dg.sp_framework_basis, - dg.sp_framework_opinions, - dg.submitted_date, - dg.total_amount_expended, - dg.type_audit_code, - -- - -- federal_award - -- - dfa.additional_award_identification, - dfa.amount_expended, - dfa.audit_report_type, - dfa.cluster_name, - dfa.cluster_total, - dfa.federal_agency_prefix, - dfa.federal_award_extension, - dfa.federal_program_name, - dfa.federal_program_total, - dfa.findings_count, - dfa.is_direct, - dfa.is_loan, - dfa.is_major, - dfa.is_passthrough_award, - dfa.loan_balance, - dfa.other_cluster_name, - dfa.passthrough_amount, - dfa.state_cluster_name, - -- - -- finding - -- - df.is_material_weakness, - df.is_modified_opinion, - df.is_other_findings, - df.is_other_matters, - df.is_questioned_costs, - df.is_repeat_finding, - df.is_significant_deficiency, - df.prior_finding_ref_numbers, - df.type_requirement, - -- - -- passthrough - -- - dp.passthrough_id, - dp.passthrough_name - FROM - public.dissemination_federalaward dfa - LEFT JOIN public_data_v1_0_0.general dg - ON dfa.report_id = dg.report_id - LEFT JOIN public.dissemination_finding df - ON dfa.report_id = df.report_id - AND dfa.award_reference = df.award_reference - LEFT JOIN public.dissemination_passthrough dp - ON dfa.report_id = dp.report_id - AND dfa.award_reference = dp.award_reference diff --git a/backend/tools/variables.sh b/backend/tools/variables.sh index bd75e23689..e1aaf992e2 100644 --- a/backend/tools/variables.sh +++ b/backend/tools/variables.sh @@ -1,7 +1,10 @@ # These are our active APIs +# admin_api_access_tables must run before +# the admin API can be stood up. declare -a api_versions=( "api_v1_0_3" - "api_v1_1_0" + "api_v1_1_0" + "admin_api_access_tables" "admin_api_v1_1_0" "admin_api_v1_1_1" "public_api_v2_0_0_alpha" @@ -12,7 +15,8 @@ declare -a api_required_tables=( "public.dissemination_general" "public.dissemination_general" "public.support_adminapievent" - "public.support_adminapievent" + "public.support_administrative_key_uuids" + "public.support_administrative_key_uuids" "public_data_v1_0_0.general" ) From e4fa18ff929ef4b61b651d1cb9a9c7fafd8a9f88 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Wed, 2 Oct 2024 07:17:45 -0400 Subject: [PATCH 10/89] These are important... How did I miss those? --- .../sql/api/admin_api_v1_1_0/standup.sql | 615 ++++++++++++++++++ .../sql/api/admin_api_v1_1_0/teardown.sql | 17 + 2 files changed, 632 insertions(+) create mode 100644 backend/dissemination/sql/api/admin_api_v1_1_0/standup.sql create mode 100644 backend/dissemination/sql/api/admin_api_v1_1_0/teardown.sql diff --git a/backend/dissemination/sql/api/admin_api_v1_1_0/standup.sql b/backend/dissemination/sql/api/admin_api_v1_1_0/standup.sql new file mode 100644 index 0000000000..471069cd21 --- /dev/null +++ b/backend/dissemination/sql/api/admin_api_v1_1_0/standup.sql @@ -0,0 +1,615 @@ +DO +$do$ +BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = 'authenticator') THEN + RAISE NOTICE 'Role "authenticator" already exists. Skipping.'; + ELSE + CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; + END IF; +END +$do$; + +DO +$do$ +BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = 'api_fac_gov') THEN + RAISE NOTICE 'Role "api_fac_gov" already exists. Skipping.'; + ELSE + CREATE ROLE api_fac_gov NOLOGIN; + END IF; +END +$do$; + +GRANT api_fac_gov TO authenticator; + +NOTIFY pgrst, 'reload schema'; +begin; + +do +$$ +begin + DROP SCHEMA IF EXISTS admin_api_v1_1_0 CASCADE; + DROP SCHEMA IF EXISTS admin_api_v1_1_0_functions CASCADE; + + if not exists (select schema_name from information_schema.schemata where schema_name = 'admin_api_v1_1_0') then + create schema admin_api_v1_1_0; + create schema admin_api_v1_1_0_functions; + + grant usage on schema admin_api_v1_1_0_functions to api_fac_gov; + + -- Grant access to tables and views + alter default privileges + in schema admin_api_v1_1_0 + grant select + -- this includes views + on tables + to api_fac_gov; + + -- Grant access to sequences, if we have them + grant usage on schema admin_api_v1_1_0 to api_fac_gov; + grant select, usage on all sequences in schema admin_api_v1_1_0 to api_fac_gov; + alter default privileges + in schema admin_api_v1_1_0 + grant select, usage + on sequences + to api_fac_gov; + + -- The admin API needs to be able to write user permissions. + -- This is so we can add and remove people who will have tribal data access + -- via the administrative API. + GRANT INSERT, SELECT, DELETE on public.users_userpermission to api_fac_gov; + -- We need to be able to look up slugs and turn them into permission IDs. + GRANT SELECT on public.users_permission to api_fac_gov; + -- It also needs to be able to log events. + GRANT INSERT on public.support_adminapievent to api_fac_gov; + -- And, it wants to read the UUIDs of administrative keys + GRANT SELECT ON public.support_administrative_key_uuids TO api_fac_gov; + -- We want to see data in flight as admins. + GRANT SELECT ON public.audit_singleauditchecklist TO api_fac_gov; + + GRANT INSERT, SELECT, DELETE on public.dissemination_tribalapiaccesskeyids to api_fac_gov; + GRANT INSERT on public.dissemination_onetimeaccess to api_fac_gov; + end if; +end +$$ +; + +commit; + +notify pgrst, + 'reload schema'; + +-- WARNING +-- Under PostgreSQL 12, the functions below work. +-- Under PostgreSQL 14, these will break. +-- +-- Note the differences: +-- +-- raise info 'Works under PostgreSQL 12'; +-- raise info 'request.header.x-magic %', (SELECT current_setting('request.header.x-magic', true)); +-- raise info 'request.jwt.claim.expires %', (SELECT current_setting('request.jwt.claim.expires', true)); +-- raise info 'Works under PostgreSQL 14'; +-- raise info 'request.headers::json->>x-magic %', (SELECT current_setting('request.headers', true)::json->>'x-magic'); +-- raise info 'request.jwt.claims::json->expires %', (SELECT current_setting('request.jwt.claims', true)::json->>'expires'); +-- +-- To quote the work of Dav Pilkey, "remember this now." + +begin; + + +CREATE OR REPLACE FUNCTION admin_api_v1_1_0_functions.get_header(item text) RETURNS text + AS $get_header$ + declare res text; + begin + SELECT (current_setting('request.headers', true)::json)->>item into res; + return res; + end; +$get_header$ LANGUAGE plpgsql; + +create or replace function admin_api_v1_1_0_functions.get_api_key_uuid() returns TEXT +as $gaku$ +declare uuid text; +begin + select admin_api_v1_1_0_functions.get_header('x-api-user-id') into uuid; + return uuid; +end; +$gaku$ LANGUAGE plpgsql; + +-- log_api_event +-- Maintain an internal table of administrative API events. +-- Also RAISE INFO so that NR gets a copy. +create or replace function admin_api_v1_1_0_functions.log_admin_api_event(event TEXT, meta JSON) +returns boolean +as $log_admin_api_event$ +DECLARE + uuid_header text; +BEGIN + SELECT admin_api_v1_1_0_functions.get_api_key_uuid() INTO uuid_header; + + INSERT INTO public.support_adminapievent + (api_key_uuid, event, event_data, "timestamp") + VALUES (uuid_header, event, meta, NOW()); + + RAISE INFO 'ADMIN_API % % %', uuid_header, event, meta; + RETURN 1; +END; +$log_admin_api_event$ LANGUAGE plpgsql; + + +-- has_admin_data_access :: permission -> bool +-- The permissions (insert, select, delete) allow us to have users who can +-- read administrative data in addition to users who can (say) update +-- select tables like the tribal access lists. +create or replace function admin_api_v1_1_0_functions.has_admin_data_access(perm TEXT) returns boolean +as $has_admin_data_access$ +DECLARE + uuid_header text; + key_exists boolean; + has_permission boolean; +BEGIN + SELECT admin_api_v1_1_0_functions.get_api_key_uuid() INTO uuid_header; + + SELECT + CASE WHEN EXISTS ( + SELECT uuid + FROM public.support_administrative_key_uuids aku + WHERE aku.uuid = uuid_header) + THEN 1::BOOLEAN + ELSE 0::BOOLEAN + END + INTO key_exists; + + SELECT + CASE WHEN EXISTS ( + SELECT permissions + FROM public.support_administrative_key_uuids aku + WHERE aku.uuid = uuid_header + AND aku.permissions like '%' || perm || '%') + THEN 1::BOOLEAN + ELSE 0::BOOLEAN + END + INTO has_permission; + + -- This log event is an INSERT. When called from a VIEW (a SELECT-only context), + -- a call to log_admin_api_event() fails. So, we'll RAISE INFO right here, so we can + -- see the resultse of access checks in the log. We might later comment this out if + -- it becomes too noisy. + RAISE INFO 'ADMIN_API has_access_check % % %', uuid_header, key_exists, has_permission; + + RETURN key_exists AND has_permission; +END; +$has_admin_data_access$ LANGUAGE plpgsql; + +-- Takes an email address and, if that address is not in the access table, +-- inserts it. If the address already exists, the insert is skipped. +-- +-- ### Example from REST client +-- POST http://localhost:3000/rpc/add_tribal_access_email +-- authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} +-- content-profile: admin_api_v1_1_0 +-- content-type: application/json +-- Prefer: params=single-object +-- // Not actually a key UUID. +-- X-Api-User-Id: 18ef0e72-8976-11ee-ad35-3f80b454d3cc +-- { +-- "email": "darth.vader@deathstar.org" +-- } +create or replace function admin_api_v1_1_0.add_tribal_access_email(params JSON) +returns BOOLEAN +as $add_tribal_access_email$ +DECLARE + already_exists INTEGER; + read_tribal_id INTEGER; +BEGIN + -- If the API user has insert permissions, give it a go + IF admin_api_v1_1_0_functions.has_admin_data_access('CREATE') + THEN + -- Are they already in the table? + SELECT count(up.email) + FROM public.users_userpermission as up + WHERE LOWER(email) = LOWER(params->>'email') INTO already_exists; + + -- If they are, we're going to exit. + IF already_exists <> 0 + THEN + RETURN 0; + END IF; + + -- Grab the permission ID that we need for the insert below. + -- We want the 'read-tribal' permission, which has a human-readable + -- slug. But, we need it's ID, because that is the PK. + SELECT up.id INTO read_tribal_id + FROM public.users_permission AS up + WHERE up.slug = 'read-tribal'; + + IF already_exists = 0 + THEN + -- Can we make the 1 not magic... do a select into. + INSERT INTO public.users_userpermission + (email, permission_id, user_id) + VALUES (LOWER(params->>'email'), read_tribal_id, null); + + RAISE INFO 'ADMIN_API add_tribal_access_email OK %', LOWER(params->>'email'); + RETURN admin_api_v1_1_0_functions.log_admin_api_event('tribal-access-email-added', + json_build_object('email', LOWER(params->>'email'))); + END IF; + ELSE + RETURN 0; + END IF; +end; +$add_tribal_access_email$ LANGUAGE plpgsql; + +-- Adds many email addresses. Calls `add_tribal_access_email` for each address. +-- +-- ### Example from REST client +-- POST http://localhost:3000/rpc/add_tribal_access_emails +-- authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} +-- content-profile: admin_api_v1_1_0 +-- content-type: application/json +-- Prefer: params=single-object +-- // Not actually a key UUID. +-- X-Api-User-Id: 18ef0e72-8976-11ee-ad35-3f80b454d3cc +-- { +-- "emails": [ +-- "darth.vader@deathstar.org", +-- "bob.darth.vader@deathstar.org", +-- "darthy.vader@deathstar.org", +-- "bob@deathstar.org" +-- ] +-- } +CREATE OR REPLACE FUNCTION admin_api_v1_1_0.add_tribal_access_emails(params JSON) +returns BOOLEAN +as $add_tribal_access_emails$ +DECLARE + ele TEXT; + em record; +BEGIN + IF admin_api_v1_1_0_functions.has_admin_data_access('CREATE') + THEN + -- This is a FOR loop over a JSON array in plPgSQL + FOR em IN (SELECT json_array_elements_text((params->>'emails')::JSON) ele) + LOOP + -- PERFORM is how to execute code that does not return anything. + -- If a SELECT was used here, the SQL compiler would complain. + PERFORM admin_api_v1_1_0.add_tribal_access_email(json_build_object('email', em.ele)::JSON); + END LOOP; + RETURN 1; + END IF; + RETURN 0; +END; +$add_tribal_access_emails$ LANGUAGE plpgsql; + +-- Removes the email. Will remove multiple rows. That shouldn't happen, but still. +-- +-- ### Example from REST client +-- POST http://localhost:3000/rpc/remove_tribal_access_email +-- authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} +-- content-profile: admin_api_v1_1_0 +-- content-type: application/json +-- Prefer: params=single-object +-- // Not actually a key UUID. +-- X-Api-User-Id: 18ef0e72-8976-11ee-ad35-3f80b454d3cc +-- { +-- "email": "darth.vader@deathstar.org" +-- } +CREATE OR REPLACE FUNCTION admin_api_v1_1_0.remove_tribal_access_email(params JSON) +returns BOOLEAN +as $remove_tribal_access_email$ +DECLARE + affected_rows INTEGER; +BEGIN + + IF admin_api_v1_1_0_functions.has_admin_data_access('DELETE') + THEN + -- Delete rows where the email address matches + DELETE FROM public.users_userpermission as up + WHERE LOWER(up.email) = LOWER(params->>'email'); + -- This is the Postgres way to find out how many rows + -- were affected by a DELETE. + GET DIAGNOSTICS affected_rows = ROW_COUNT; + -- If that is greater than zero, we were successful. + IF affected_rows > 0 + THEN + RETURN admin_api_v1_1_0_functions.log_admin_api_event('tribal-access-email-removed', + json_build_object('email', LOWER(params->>'email'))); + ELSE + RETURN 0; + END IF; + ELSE + -- If we did not have permission, consider it a failure. + RETURN 0; + END IF; +end; +$remove_tribal_access_email$ LANGUAGE plpgsql; + +-- Removes many email addresses. Calls `remove_tribal_access_email` for each address. +-- +-- ### Example from REST client +-- POST http://localhost:3000/rpc/remove_tribal_access_emails +-- authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} +-- content-profile: admin_api_v1_1_0 +-- content-type: application/json +-- Prefer: params=single-object +-- // Not actually a key UUID. +-- X-Api-User-Id: 18ef0e72-8976-11ee-ad35-3f80b454d3cc +-- { +-- "emails": [ +-- "darth.vader@deathstar.org", +-- "bob.darth.vader@deathstar.org", +-- "darthy.vader@deathstar.org", +-- "bob@deathstar.org" +-- ] +-- } +CREATE OR REPLACE FUNCTION admin_api_v1_1_0.remove_tribal_access_emails(params JSON) +returns BOOLEAN +as $remove_tribal_access_emails$ +DECLARE + ele TEXT; + em record; +BEGIN + IF admin_api_v1_1_0_functions.has_admin_data_access('DELETE') + THEN + FOR em IN (SELECT json_array_elements_text((params->>'emails')::JSON) ele) + LOOP + PERFORM admin_api_v1_1_0.remove_tribal_access_email(json_build_object('email', em.ele)::JSON); + END LOOP; + RETURN 1; + END IF; + RETURN 0; +END; +$remove_tribal_access_emails$ LANGUAGE plpgsql; + + + + +--The function below add_tribal_api_key_access adds read access to a tribal API for a specified email. +--It checks if the API user has read permissions. +--Otherwise, it adds the email with 'read-tribal' permission, logs the event, and returns true. + +CREATE OR REPLACE FUNCTION admin_api_v1_1_0.add_tribal_api_key_access(params JSON) +RETURNS JSON +AS $add_tribal_api_key_access$ +DECLARE + user_exists BOOLEAN; +BEGIN + -- If the API user has create permissions, we can proceed + IF admin_api_v1_1_0_functions.has_admin_data_access('CREATE') THEN + -- Check if the user with the given email + SELECT EXISTS ( + SELECT 1 + FROM public.dissemination_TribalApiAccessKeyIds + WHERE LOWER(email) = LOWER(params->>'email') + ) + INTO user_exists; + + -- If the user already exists, it means they have access. + -- For purposes of this function, lets call that "succses", and return true. + IF user_exists THEN + RAISE INFO 'ADMIN_API add_tribal_api_key_access ALREADY_EXISTS %', LOWER(params->>'email'); + RETURN json_build_object( + 'result', 'success', + 'message', 'User with this key already exists')::JSON; + + END IF; + + -- If the user does not exist, add a new record + INSERT INTO public.dissemination_TribalApiAccessKeyIds (email, key_id, date_added) + VALUES (LOWER(params->>'email'), params->>'key_id', CURRENT_TIMESTAMP); + RAISE INFO 'ADMIN_API add_tribal_api_key_access ACCESS_GRANTED % %', LOWER(params->>'email'), params->>'key_id'; + RETURN json_build_object( + 'result', 'success', + 'message', 'User access granted')::JSON; + ELSE + -- If the user does not have CREATE permissions, then we should return a message to that effect. + -- It is a permissions error, but still, we need to know this failed. + RAISE INFO 'ADMIN_API add_tribal_api_key_access ADMIN_LACKS_CREATE'; + RETURN json_build_object( + 'result', 'failure', + 'message', 'Admin user lacks CREATE permissions')::JSON; + END IF; + + -- Return false by default. + RAISE INFO 'ADMIN_API add_tribal_api_key_access WAT %', LOWER(params->>'email'); + RETURN json_build_object( + 'result', 'failure', + 'message', 'Unknown error in access addition')::JSON; +END; +$add_tribal_api_key_access$ LANGUAGE plpgsql; + +-- The function below removes tribal API key access for a specified email. +-- It checks if the API user has read permissions. +-- If the email exists in the database with 'read-tribal' permission, it removes the entry, logs the removal event, and returns true. +-- If the email doesn't exist or the user lacks proper permissions, the function returns false. + +CREATE OR REPLACE FUNCTION admin_api_v1_1_0.remove_tribal_api_key_access(params JSON) +RETURNS JSON +AS $remove_tribal_api_key_access$ +DECLARE + user_exists BOOLEAN; +BEGIN + -- If the API user has read permissions, give it a go + IF admin_api_v1_1_0_functions.has_admin_data_access('DELETE') THEN + -- Check if the user with the given email exists + SELECT EXISTS ( + SELECT 1 + FROM public.dissemination_TribalApiAccessKeyIds + WHERE LOWER(email) = LOWER(params->>'email') + ) + INTO user_exists; + + -- If the user exists, remove the record + IF user_exists THEN + DELETE FROM public.dissemination_TribalApiAccessKeyIds + WHERE LOWER(email) = LOWER(params->>'email'); + RAISE INFO 'ADMIN_API remove_tribal_api_key_access ACCESS_REMOVED %', LOWER(params->>'email'); + RETURN json_build_object( + 'result', 'success', + 'message', 'Removed record')::JSON; + ELSE + RAISE INFO 'ADMIN_API remove_tribal_api_key_access DID_NOT_EXIST %', LOWER(params->>'email'); + RETURN json_build_object( + 'result', 'failure', + 'message', 'User did not exist in table')::JSON; + END IF; + ELSE + RAISE INFO 'ADMIN_API remove_tribal_api_key_access ADMIN_LACKS_DELETE'; + RETURN json_build_object( + 'result', 'failure', + 'message', 'Admin user lacks DELETE permissions')::JSON; -- Return false if the API user doesn't have read permissions + END IF; + RAISE INFO 'ADMIN_API add_tribal_api_key_access WAT %', LOWER(params->>'email'); + RETURN json_build_object( + 'result', 'failure', + 'message', 'Uknown error in access removal')::JSON; +END; +$remove_tribal_api_key_access$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION admin_api_v1_1_0.request_file_access( + report_id TEXT +) RETURNS JSON LANGUAGE plpgsql AS +$$ +DECLARE + v_uuid_header TEXT; + v_access_uuid VARCHAR(200); + v_key_exists BOOLEAN; + v_key_added_date DATE; +BEGIN + + SELECT admin_api_v1_1_0_functions.get_api_key_uuid() INTO v_uuid_header; + + -- Check if the provided API key exists in public.dissemination_TribalApiAccessKeyIds + SELECT + EXISTS( + SELECT 1 + FROM public.dissemination_TribalApiAccessKeyIds + WHERE key_id = v_uuid_header + ) INTO v_key_exists; + + + -- Get the added date of the key from public.dissemination_TribalApiAccessKeyIds + SELECT date_added + INTO v_key_added_date + FROM public.dissemination_TribalApiAccessKeyIds + WHERE key_id = v_uuid_header; + + + -- Check if the key is less than 6 months old + IF v_uuid_header IS NOT NULL AND v_key_exists AND v_key_added_date >= CURRENT_DATE - INTERVAL '6 months' THEN + -- Generate UUID (using PostgreSQL's gen_random_uuid function) + SELECT gen_random_uuid() INTO v_access_uuid; + + -- Inserting data into the one_time_access table + INSERT INTO public.dissemination_onetimeaccess (uuid, api_key_id, timestamp, report_id) + VALUES (v_access_uuid::UUID, v_uuid_header, CURRENT_TIMESTAMP, report_id); + + -- Return the UUID to the user + RETURN json_build_object('access_uuid', v_access_uuid); + ELSE + -- Return an error for unauthorized access + RETURN json_build_object('error', 'Unauthorized access or key older than 6 months')::JSON; + END IF; +END; +$$; + + + +commit; + +NOTIFY pgrst, 'reload schema'; + +begin; + + +--------------------------------------- +-- accesses +--------------------------------------- +-- public.audit_access definition + +-- Drop table + +-- DROP TABLE public.audit_access; + +CREATE OR REPLACE VIEW admin_api_v1_1_0.audit_access AS + SELECT + aa.role, + aa.fullname, + aa.email, + aa.sac_id, + aa.user_id + FROM + public.audit_access aa + WHERE + admin_api_v1_1_0_functions.has_admin_data_access('READ') + ORDER BY aa.id +; + +CREATE OR REPLACE VIEW admin_api_v1_1_0.singleauditchecklist AS + SELECT + sac.id, + sac.date_created, + sac.submission_status, + sac.data_source, + sac.transition_name, + sac.transition_date, + sac.report_id, + sac.audit_type, + sac.general_information, + sac.audit_information, + sac.federal_awards, + sac.corrective_action_plan, + sac.findings_text, + sac.findings_uniform_guidance, + sac.additional_ueis, + sac.additional_eins, + sac.secondary_auditors, + sac.notes_to_sefa, + sac.auditor_certification, + sac.auditee_certification, + sac.tribal_data_consent, + sac.cognizant_agency, + sac.oversight_agency, + sac.submitted_by_id + from + public.audit_singleauditchecklist sac + where + admin_api_v1_1_0_functions.has_admin_data_access('READ') + order by sac.id +; + +CREATE OR REPLACE VIEW admin_api_v1_1_0.tribal_access AS + SELECT + uup.email, + up.slug as permission + FROM + users_userpermission uup, + users_permission up + WHERE + (uup.permission_id = up.id) + AND (up.slug = 'read-tribal') + AND admin_api_v1_1_0_functions.has_admin_data_access('READ') + ORDER BY uup.id +; + +CREATE OR REPLACE VIEW admin_api_v1_1_0.admin_api_events AS + SELECT + ae.timestamp, + ae.api_key_uuid, + ae.event, + ae.event_data + FROM + public.support_adminapievent ae + WHERE + admin_api_v1_1_0_functions.has_admin_data_access('READ') + ORDER BY ae.id +; + + +commit; + +notify pgrst, + 'reload schema'; diff --git a/backend/dissemination/sql/api/admin_api_v1_1_0/teardown.sql b/backend/dissemination/sql/api/admin_api_v1_1_0/teardown.sql new file mode 100644 index 0000000000..e9bbae297e --- /dev/null +++ b/backend/dissemination/sql/api/admin_api_v1_1_0/teardown.sql @@ -0,0 +1,17 @@ + +begin; + +DROP SCHEMA IF EXISTS admin_api_v1_1_0 CASCADE; + +commit; + +notify pgrst, + 'reload schema'; +begin; + + drop view if exists admin_api_v1_1_0.audit_access; + +commit; + +notify pgrst, + 'reload schema'; From de6c7ea83653a0547055e817a678ba7a0146fb75 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Wed, 2 Oct 2024 07:49:11 -0400 Subject: [PATCH 11/89] Adding GH action, script --- .../fac-backup-and-sling-scheduled.yml | 36 +++++++ backend/tools/fac-backup-and-sling.sh | 101 ++++++++++++++++++ 2 files changed, 137 insertions(+) create mode 100644 .github/workflows/fac-backup-and-sling-scheduled.yml create mode 100755 backend/tools/fac-backup-and-sling.sh diff --git a/.github/workflows/fac-backup-and-sling-scheduled.yml b/.github/workflows/fac-backup-and-sling-scheduled.yml new file mode 100644 index 0000000000..cf4ca4aa78 --- /dev/null +++ b/.github/workflows/fac-backup-and-sling-scheduled.yml @@ -0,0 +1,36 @@ +--- +name: Backup the database with fac-backup-utility +### Common Commands: +# ./fac-backup-util.sh v0.1.5 scheduled_backup +# ./fac-backup-util.sh v0.1.5 daily_backup +on: + workflow_call: + inputs: + environment: + required: true + type: string + util_version: + description: Version for fac backup utility to use (ex. vX.Y.Z) + required: true + type: string + backup_operation: + description: Operation for fac-backup-utility + required: true + type: string +jobs: + fac-backup-scheduled: + name: Perform Database Backup + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + env: + space: ${{ inputs.environment }} + steps: + - name: Backup FAC Database, Sync Media, and sling API tables + uses: cloud-gov/cg-cli-tools@main + with: + cf_username: ${{ secrets.CF_USERNAME }} + cf_password: ${{ secrets.CF_PASSWORD }} + cf_org: gsa-tts-oros-fac + cf_space: ${{ env.space }} + command: cf run-task gsa-fac -k 7G -m 3G --name backup_util_scheduled --command "./tools/fac-backup-and-sling.sh ${{ inputs.util_version }} ${{ inputs.backup_operation }}" + diff --git a/backend/tools/fac-backup-and-sling.sh b/backend/tools/fac-backup-and-sling.sh new file mode 100755 index 0000000000..74df66c601 --- /dev/null +++ b/backend/tools/fac-backup-and-sling.sh @@ -0,0 +1,101 @@ +#!/bin/bash +set -e +source tools/util_startup.sh +source tools/setup_env.sh +setup_env + +version=$1 +run_option=$2 +s3_name="fac-private-s3" +backup_s3_name="backups" +db_name="fac-db" +backup_db_name="fac-snapshot-db" +initial_date=$(date +%Y%m%d%H%M) +scheduled_date=$(date +%m-%d-%H) +daily_date=$(date +%m-%d) + +mkdir tmp && cd tmp || return + +GetUtil() { + curl -x "$https_proxy" -L "https://github.com/GSA-TTS/fac-backup-utility/releases/download/$version/gov.gsa.fac.cgov-util-$version-linux-amd64.tar.gz" -O + tar -xvf "gov.gsa.fac.cgov-util-$version-linux-amd64.tar.gz" && rm "gov.gsa.fac.cgov-util-$version-linux-amd64.tar.gz" +} + +GetSling() { + curl -x "$https_proxy" -LO 'https://github.com/slingdata-io/sling-cli/releases/latest/download/sling_linux_amd64.tar.gz' + tar xf sling_linux_amd64.tar.gz + rm -f sling_linux_amd64.tar.gz + chmod +x sling +} + +RunSling() { + ./sling run -r dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml +} + +InstallAWS() { + ./gov.gsa.fac.cgov-util install_aws +} + +AWSS3Sync() { + ./gov.gsa.fac.cgov-util s3_sync --source_s3 s3://"$1"/ --dest_s3 s3://"$2"/ +} + +RDSToS3Dump() { + ./gov.gsa.fac.cgov-util db_to_s3 --db "$1" --s3path s3://"$2"/"$3"/ +} + +RDSToRDS() { + ./gov.gsa.fac.cgov-util db_to_db --src_db "$1" --dest_db "$2" --operation "$3" +} + +if [ "$run_option" == "initial_backup" ]; then + GetUtil + InstallAWS + gonogo "install_aws" + RDSToS3Dump "$db_name" "$backup_s3_name" "initial/$initial_date" + gonogo "db_to_s3" + RDSToRDS "$db_name" "$backup_db_name" "initial" + gonogo "db_to_db" + AWSS3Sync "$s3_name" "$backup_s3_name" + gonogo "s3_sync" + RunSling + gonogo "sling" +elif [ "$run_option" == "deploy_backup" ]; then + GetUtil + InstallAWS + gonogo "install_aws" + RDSToRDS "$db_name" "$backup_db_name" "backup" + gonogo "db_to_db" + AWSS3Sync "$s3_name" "$backup_s3_name" + gonogo "s3_sync" + RunSling + gonogo "sling" +elif [ "$run_option" == "scheduled_backup" ]; then + GetUtil + InstallAWS + gonogo "install_aws" + RDSToS3Dump "$db_name" "$backup_s3_name" "scheduled/$scheduled_date" + gonogo "db_to_s3" + AWSS3Sync "$s3_name" "$backup_s3_name" + gonogo "s3_sync" + RunSling + gonogo "sling" +elif [ "$run_option" == "daily_backup" ]; then + GetUtil + InstallAWS + gonogo "install_aws" + RDSToS3Dump "$db_name" "$backup_s3_name" "daily/$daily_date" + gonogo "db_to_s3" + AWSS3Sync "$s3_name" "$backup_s3_name" + gonogo "s3_sync" + RunSling + gonogo "sling" +elif [ "$run_option" == "media_sync" ]; then + GetUtil + InstallAWS + gonogo "install_aws" + AWSS3Sync "$s3_name" "$backup_s3_name" + gonogo "s3_sync" + RunSling + gonogo "sling" +fi From b30ba03066dda81a40de0a58e3f770f1aebd66d6 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Wed, 2 Oct 2024 08:35:04 -0400 Subject: [PATCH 12/89] Expand the admin API to increase visibility We should be able to inspect more tables, for debugging. --- .../sql/api/admin_api_v1_1_1/standup.sql | 115 +++++++++++------- 1 file changed, 68 insertions(+), 47 deletions(-) diff --git a/backend/dissemination/sql/api/admin_api_v1_1_1/standup.sql b/backend/dissemination/sql/api/admin_api_v1_1_1/standup.sql index aca364f30e..8f5f483bd1 100644 --- a/backend/dissemination/sql/api/admin_api_v1_1_1/standup.sql +++ b/backend/dissemination/sql/api/admin_api_v1_1_1/standup.sql @@ -470,53 +470,6 @@ begin; -- DROP TABLE public.audit_access; -CREATE OR REPLACE VIEW admin_api_v1_1_1.audit_access AS - SELECT - aa.role, - aa.fullname, - aa.email, - aa.sac_id, - aa.user_id - FROM - public.audit_access aa - WHERE - admin_api_v1_1_1_functions.has_admin_data_access('READ') - ORDER BY aa.id -; - -CREATE OR REPLACE VIEW admin_api_v1_1_1.singleauditchecklist AS - SELECT - sac.id, - sac.date_created, - sac.submission_status, - sac.data_source, - sac.transition_name, - sac.transition_date, - sac.report_id, - sac.audit_type, - sac.general_information, - sac.audit_information, - sac.federal_awards, - sac.corrective_action_plan, - sac.findings_text, - sac.findings_uniform_guidance, - sac.additional_ueis, - sac.additional_eins, - sac.secondary_auditors, - sac.notes_to_sefa, - sac.auditor_certification, - sac.auditee_certification, - sac.tribal_data_consent, - sac.cognizant_agency, - sac.oversight_agency, - sac.submitted_by_id - from - public.audit_singleauditchecklist sac - where - admin_api_v1_1_1_functions.has_admin_data_access('READ') - order by sac.id -; - CREATE OR REPLACE VIEW admin_api_v1_1_1.tribal_access AS SELECT uup.email, @@ -544,6 +497,74 @@ CREATE OR REPLACE VIEW admin_api_v1_1_1.admin_api_events AS ORDER BY ae.id ; +----------------- +-- Expose more of the internal tables for analysis/trouble-shooting. +----------------- +CREATE OR REPLACE VIEW admin_api_v1_1_1.audit_access AS + SELECT * FROM admin_api_v1_1_1.audit_access + WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') +; + +CREATE OR REPLACE VIEW admin_api_v1_1_1.audit_deletedaccess AS + SELECT * FROM admin_api_v1_1_1.audit_deletedaccess + WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') +; + +CREATE OR REPLACE VIEW admin_api_v1_1_1.audit_excelfile AS + SELECT * FROM admin_api_v1_1_1.audit_excelfile + WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') +; + +CREATE OR REPLACE VIEW admin_api_v1_1_1.singleauditchecklist AS + SELECT * + FROM public.audit_singleauditchecklist sac + WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') +; + +CREATE OR REPLACE VIEW admin_api_v1_1_1.audit_sacvalidationwaiver AS + SELECT * FROM admin_api_v1_1_1.audit_sacvalidationwaiver + WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') +; + +CREATE OR REPLACE VIEW admin_api_v1_1_1.audit_singleauditreportfile AS + SELECT * FROM admin_api_v1_1_1.audit_singleauditreportfile + WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') +; + +CREATE OR REPLACE VIEW admin_api_v1_1_1.audit_submissionevent AS + SELECT * FROM admin_api_v1_1_1.audit_submissionevent + WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') +; + +CREATE OR REPLACE VIEW admin_api_v1_1_1.auth_user AS + SELECT * FROM admin_api_v1_1_1.auth_user + WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') +; + +CREATE OR REPLACE VIEW admin_api_v1_1_1.django_migrations AS + SELECT * FROM admin_api_v1_1_1.django_migrations + WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') +; + +CREATE OR REPLACE VIEW admin_api_v1_1_1.support_adminapievent AS + SELECT * FROM admin_api_v1_1_1.support_adminapievent + WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') +; + +CREATE OR REPLACE VIEW admin_api_v1_1_1.support_cognizantassignment AS + SELECT * FROM admin_api_v1_1_1.support_cognizantassignment + WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') +; + +CREATE OR REPLACE VIEW admin_api_v1_1_1.support_cognizantbaseline AS + SELECT * FROM admin_api_v1_1_1.support_cognizantbaseline + WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') +; + +CREATE OR REPLACE VIEW admin_api_v1_1_1.users_userpermission AS + SELECT * FROM admin_api_v1_1_1.users_userpermission + WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') +; commit; From 1337097b428ecf4295b92a3c6a05f0f9704f49ca Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Thu, 3 Oct 2024 15:31:41 -0400 Subject: [PATCH 13/89] Full-up, with performance testing This brings up the full API, and is ready for performance testing in `preview`. --- .../sql/api/admin_api_v1_1_1/standup.sql | 24 +- .../sql/api/api_v1_0_3/teardown.sql | 2 - .../api/public_api_v2_0_0_alpha/public.rest | 20 + .../api/public_api_v2_0_0_alpha/standup.sql | 73 ++- .../api/public_api_v2_0_0_alpha/teardown.sql | 1 + .../sql/api/public_data_v1_0_0/standup.sql} | 0 .../sql/api/public_data_v1_0_0/teardown.sql | 30 ++ .../fac-snapshot-db/additional_eins.sql | 3 + .../fac-snapshot-db/additional_ueis.sql | 3 + .../sql/indexes/fac-snapshot-db/combined.sql | 3 + .../corrective_action_plans.sql | 7 +- .../fac-snapshot-db/federal_awards.sql | 6 + .../sql/indexes/fac-snapshot-db/findings.sql | 3 + .../indexes/fac-snapshot-db/findings_text.sql | 3 + .../sql/indexes/fac-snapshot-db/general.sql | 3 + .../indexes/fac-snapshot-db/notes_to_sefa.sql | 3 + .../indexes/fac-snapshot-db/passthrough.sql | 3 + .../fac-snapshot-db/secondary_auditors.sql | 5 +- .../public_data_v1_0_0.yaml | 132 ++--- .../public_metadata_v1_0_0.yaml | 73 +++ .../tribal_data_v1_0_0.yaml | 90 ++++ backend/docker-compose-db-only.yml | 71 +++ backend/run.sh | 11 +- backend/tools/api_teardown.sh | 50 +- .../load_public_dissem_data/data/config.json | 90 ---- .../data/public_data_v1_0_0.yaml | 499 ------------------ backend/tools/setup_local_env.sh | 34 +- backend/tools/sling_first_run.sh | 40 ++ backend/tools/sling_support_functions.sh | 15 + backend/tools/variables.sh | 2 + backend/util/__init__.py | 0 backend/util/api_perf_test/perf.py | 119 +++++ .../{tools => util}/collect_scan_metrics.py | 0 .../{tools => util}/fac-backup-and-sling.sh | 19 +- .../{tools => util}/generate_xlsx_files.py | 0 .../load_public_dissem_data/.gitignore | 0 .../load_public_dissem_data/Dockerfile | 1 + .../load_public_dissem_data/Makefile | 1 + .../load_public_dissem_data/README.md | 0 .../load_public_dissem_data/data/README.md | 0 .../load_public_dissem_data/data/config.json | 90 ++++ .../data/tribal_sling.yaml | 116 ++++ .../load_public_data_locally.sh | 1 + .../test_update_program_data.py | 0 .../{tools => util}/update_program_data.py | 0 45 files changed, 867 insertions(+), 779 deletions(-) rename backend/{tools/__init__.py => dissemination/sql/api/public_data_v1_0_0/standup.sql} (100%) create mode 100644 backend/dissemination/sql/api/public_data_v1_0_0/teardown.sql create mode 100644 backend/dissemination/sql/sling/public_data_v1_0_0/public_metadata_v1_0_0.yaml create mode 100644 backend/dissemination/sql/sling/public_data_v1_0_0/tribal_data_v1_0_0.yaml create mode 100644 backend/docker-compose-db-only.yml delete mode 100644 backend/tools/load_public_dissem_data/data/config.json delete mode 100644 backend/tools/load_public_dissem_data/data/public_data_v1_0_0.yaml create mode 100644 backend/tools/sling_first_run.sh create mode 100644 backend/tools/sling_support_functions.sh create mode 100644 backend/util/__init__.py create mode 100644 backend/util/api_perf_test/perf.py rename backend/{tools => util}/collect_scan_metrics.py (100%) rename backend/{tools => util}/fac-backup-and-sling.sh (88%) rename backend/{tools => util}/generate_xlsx_files.py (100%) rename backend/{tools => util}/load_public_dissem_data/.gitignore (100%) rename backend/{tools => util}/load_public_dissem_data/Dockerfile (97%) rename backend/{tools => util}/load_public_dissem_data/Makefile (70%) rename backend/{tools => util}/load_public_dissem_data/README.md (100%) rename backend/{tools => util}/load_public_dissem_data/data/README.md (100%) create mode 100644 backend/util/load_public_dissem_data/data/config.json create mode 100644 backend/util/load_public_dissem_data/data/tribal_sling.yaml rename backend/{tools => util}/load_public_dissem_data/load_public_data_locally.sh (98%) rename backend/{tools => util}/test_update_program_data.py (100%) rename backend/{tools => util}/update_program_data.py (100%) diff --git a/backend/dissemination/sql/api/admin_api_v1_1_1/standup.sql b/backend/dissemination/sql/api/admin_api_v1_1_1/standup.sql index 8f5f483bd1..c502d76c12 100644 --- a/backend/dissemination/sql/api/admin_api_v1_1_1/standup.sql +++ b/backend/dissemination/sql/api/admin_api_v1_1_1/standup.sql @@ -501,17 +501,17 @@ CREATE OR REPLACE VIEW admin_api_v1_1_1.admin_api_events AS -- Expose more of the internal tables for analysis/trouble-shooting. ----------------- CREATE OR REPLACE VIEW admin_api_v1_1_1.audit_access AS - SELECT * FROM admin_api_v1_1_1.audit_access + SELECT * FROM public.audit_access WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') ; CREATE OR REPLACE VIEW admin_api_v1_1_1.audit_deletedaccess AS - SELECT * FROM admin_api_v1_1_1.audit_deletedaccess + SELECT * FROM public.audit_deletedaccess WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') ; CREATE OR REPLACE VIEW admin_api_v1_1_1.audit_excelfile AS - SELECT * FROM admin_api_v1_1_1.audit_excelfile + SELECT * FROM public.audit_excelfile WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') ; @@ -522,47 +522,47 @@ CREATE OR REPLACE VIEW admin_api_v1_1_1.singleauditchecklist AS ; CREATE OR REPLACE VIEW admin_api_v1_1_1.audit_sacvalidationwaiver AS - SELECT * FROM admin_api_v1_1_1.audit_sacvalidationwaiver + SELECT * FROM public.audit_sacvalidationwaiver WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') ; CREATE OR REPLACE VIEW admin_api_v1_1_1.audit_singleauditreportfile AS - SELECT * FROM admin_api_v1_1_1.audit_singleauditreportfile + SELECT * FROM public.audit_singleauditreportfile WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') ; CREATE OR REPLACE VIEW admin_api_v1_1_1.audit_submissionevent AS - SELECT * FROM admin_api_v1_1_1.audit_submissionevent + SELECT * FROM public.audit_submissionevent WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') ; CREATE OR REPLACE VIEW admin_api_v1_1_1.auth_user AS - SELECT * FROM admin_api_v1_1_1.auth_user + SELECT * FROM public.auth_user WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') ; CREATE OR REPLACE VIEW admin_api_v1_1_1.django_migrations AS - SELECT * FROM admin_api_v1_1_1.django_migrations + SELECT * FROM public.django_migrations WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') ; CREATE OR REPLACE VIEW admin_api_v1_1_1.support_adminapievent AS - SELECT * FROM admin_api_v1_1_1.support_adminapievent + SELECT * FROM public.support_adminapievent WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') ; CREATE OR REPLACE VIEW admin_api_v1_1_1.support_cognizantassignment AS - SELECT * FROM admin_api_v1_1_1.support_cognizantassignment + SELECT * FROM public.support_cognizantassignment WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') ; CREATE OR REPLACE VIEW admin_api_v1_1_1.support_cognizantbaseline AS - SELECT * FROM admin_api_v1_1_1.support_cognizantbaseline + SELECT * FROM public.support_cognizantbaseline WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') ; CREATE OR REPLACE VIEW admin_api_v1_1_1.users_userpermission AS - SELECT * FROM admin_api_v1_1_1.users_userpermission + SELECT * FROM public.users_userpermission WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') ; diff --git a/backend/dissemination/sql/api/api_v1_0_3/teardown.sql b/backend/dissemination/sql/api/api_v1_0_3/teardown.sql index 7e1b118b45..358ecd172d 100644 --- a/backend/dissemination/sql/api/api_v1_0_3/teardown.sql +++ b/backend/dissemination/sql/api/api_v1_0_3/teardown.sql @@ -1,6 +1,4 @@ BEGIN; - -- This wipes out the schema and all attached objects, - -- including all of our views. DROP SCHEMA IF EXISTS api_v1_0_3 CASCADE; COMMIT; diff --git a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/public.rest b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/public.rest index e70df70e64..a673d2e087 100644 --- a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/public.rest +++ b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/public.rest @@ -19,9 +19,29 @@ x-api-user-id: {{xApiUserId}} accept-profile: public_api_v2_0_0_alpha Accept: application/vnd.pgrst.plan +### +GET {{scheme}}://{{apiUrl}}/federal_awards?batch_number=eq.200 +authorization: {{authorization}} +x-api-user-id: {{xApiUserId}} +accept-profile: public_api_v2_0_0_alpha +Accept: application/vnd.pgrst.plan + + +### +GET {{scheme}}://{{apiUrl}}/federal_awards?batch_number=eq.200&limit=1 +authorization: {{authorization}} +x-api-user-id: {{xApiUserId}} +accept-profile: public_api_v2_0_0_alpha + ### GET {{scheme}}://{{apiUrl}}/combined?report_id=eq.2021-12-CENSUS-0000250449 authorization: {{authorization}} x-api-user-id: {{xApiUserId}} accept-profile: public_api_v2_0_0_alpha Accept: application/vnd.pgrst.plan + +### +GET {{scheme}}://{{apiUrl}}/rpc/compute_batch?row_id=41000 +authorization: {{authorization}} +x-api-user-id: {{xApiUserId}} +accept-profile: public_api_v2_0_0_alpha diff --git a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/standup.sql b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/standup.sql index bbeed84011..37f9aa6c52 100644 --- a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/standup.sql +++ b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/standup.sql @@ -40,26 +40,37 @@ $$ BEGIN DROP SCHEMA IF EXISTS public_api_v2_0_0_alpha CASCADE; DROP SCHEMA IF EXISTS public_api_v2_0_0_alpha_functions CASCADE; - CREATE SCHEMA IF NOT EXISTS public_api_v2_0_0_alpha; CREATE SCHEMA IF NOT EXISTS public_api_v2_0_0_alpha_functions; + -- Functions are loaded before sling comes up. GRANT USAGE ON SCHEMA public_api_v2_0_0_alpha_functions TO api_fac_gov; + GRANT USAGE ON SCHEMA public_api_v2_0_0_alpha TO api_fac_gov; + GRANT USAGE ON SCHEMA public_data_v1_0_0 TO api_fac_gov; -- Grant access to tables and views ALTER DEFAULT PRIVILEGES IN SCHEMA public_api_v2_0_0_alpha GRANT SELECT - -- this includes views - ON tables - TO api_fac_gov; + ON tables + TO api_fac_gov; + + ALTER DEFAULT PRIVILEGES + IN SCHEMA public_data_v1_0_0 + GRANT SELECT + ON TABLES + TO api_fac_gov; -- Grant access to sequences, if we have them - GRANT USAGE ON SCHEMA public_api_v2_0_0_alpha to api_fac_gov; GRANT SELECT, USAGE ON ALL SEQUENCES IN SCHEMA public_api_v2_0_0_alpha TO api_fac_gov; + + GRANT SELECT, USAGE + ON ALL SEQUENCES + IN SCHEMA public_data_v1_0_0 + TO api_fac_gov; END $$ @@ -69,21 +80,35 @@ COMMIT; notify pgrst, 'reload schema'; ------------------------------------------------------ +--- -- FUNCTIONS ------------------------------------------------------ -CREATE OR REPLACE FUNCTION public_api_v2_0_0_alpha.rows_per_batch() +-- + +CREATE OR REPLACE FUNCTION public_api_v2_0_0_alpha_functions.rows_per_batch() RETURNS integer LANGUAGE sql IMMUTABLE PARALLEL SAFE AS 'SELECT 20000'; -create or replace function public_api_v2_0_0_alpha_functions.batches (diss_table text) +CREATE OR REPLACE FUNCTION public_api_v2_0_0_alpha_functions.batch (id bigint) + RETURNS bigint + AS $batch$ + DECLARE result bigint; + DECLARE RPB integer; + BEGIN + SELECT public_api_v2_0_0_alpha_functions.rows_per_batch() INTO RPB; + SELECT div(id, RPB) INTO result; + RETURN result; + END + $batch$ + LANGUAGE plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION public_api_v2_0_0_alpha_functions.batches (diss_table text) returns integer as $batches$ declare count integer; declare RPB integer; begin - select public_api_v2_0_0_alpha.rows_per_batch() into RPB; + select public_api_v2_0_0_alpha_functions.rows_per_batch() into RPB; case when diss_table = 'additional_eins' then select div(count(*), RPB) into count @@ -126,9 +151,29 @@ end $batches$ language plpgsql; -notify pgrst, 'reload schema'; +CREATE OR REPLACE FUNCTION public_api_v2_0_0_alpha.compute_batch(row_id bigint) + RETURNS BIGINT + AS $$ + DECLARE result bigint; + BEGIN + SELECT public_api_v2_0_0_alpha_functions.batch(row_id) INTO result; + return result; + END + $$ LANGUAGE plpgsql; + + +create or replace function public_api_v2_0_0_alpha.get_federal_award_batch (batch_no bigint) +RETURNS SETOF record +as $batches$ + SELECT * + FROM public_data_v1_0_0.federal_awards + WHERE batch_number = batch_no; +$batches$ +language sql; +NOTIFY pgrst, 'reload schema'; + BEGIN; CREATE VIEW public_api_v2_0_0_alpha.additional_eins AS @@ -214,6 +259,12 @@ CREATE VIEW public_api_v2_0_0_alpha.combined AS SELECT * FROM public_data_v1_0_0.combined comb ; +--------------------------------------- +-- metadata +--------------------------------------- +CREATE VIEW public_api_v2_0_0_alpha.metadata AS + SELECT * FROM public_data_v1_0_0.metadata + ; COMMIT; diff --git a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/teardown.sql b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/teardown.sql index e4570c81bf..33fe43ce3e 100644 --- a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/teardown.sql +++ b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/teardown.sql @@ -7,5 +7,6 @@ BEGIN; COMMIT; + notify pgrst, 'reload schema'; diff --git a/backend/tools/__init__.py b/backend/dissemination/sql/api/public_data_v1_0_0/standup.sql similarity index 100% rename from backend/tools/__init__.py rename to backend/dissemination/sql/api/public_data_v1_0_0/standup.sql diff --git a/backend/dissemination/sql/api/public_data_v1_0_0/teardown.sql b/backend/dissemination/sql/api/public_data_v1_0_0/teardown.sql new file mode 100644 index 0000000000..84b8ebefd7 --- /dev/null +++ b/backend/dissemination/sql/api/public_data_v1_0_0/teardown.sql @@ -0,0 +1,30 @@ +-- Drop it. We regenerate it +DROP SCHEMA IF EXISTS public_data_v1_0_0 CASCADE; + +-- We need these sequences for bringing the public tables back. +-- That is, sling needs them. So, lets build them right now. +CREATE SCHEMA IF NOT EXISTS public_data_v1_0_0; + +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_additional_eins; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_additional_ueis; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_combined; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_corrective_action_plans; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_federal_awards; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_findings_text; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_findings; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_general; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_notes_to_sefa; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_passthrough; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_secondary_auditors; + +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_additional_eins START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_additional_ueis START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_combined START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_corrective_action_plans START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_federal_awards START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_findings_text START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_findings START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_general START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_notes_to_sefa START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_passthrough START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_secondary_auditors START 1; diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/additional_eins.sql b/backend/dissemination/sql/indexes/fac-snapshot-db/additional_eins.sql index 09169f0c48..4530e7b00a 100644 --- a/backend/dissemination/sql/indexes/fac-snapshot-db/additional_eins.sql +++ b/backend/dissemination/sql/indexes/fac-snapshot-db/additional_eins.sql @@ -6,3 +6,6 @@ CREATE INDEX IF NOT EXISTS fac_snapshot_db_additional_eins_report_id_idx CREATE INDEX IF NOT EXISTS fac_snapshot_db_additional_eins_report_id_fad on public_data_v1_0_0.additional_eins (report_id, fac_accepted_date); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_batch_additional_eins + ON public_data_v1_0_0.additional_eins (batch_number); diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/additional_ueis.sql b/backend/dissemination/sql/indexes/fac-snapshot-db/additional_ueis.sql index 5835b7f049..7316d74a2e 100644 --- a/backend/dissemination/sql/indexes/fac-snapshot-db/additional_ueis.sql +++ b/backend/dissemination/sql/indexes/fac-snapshot-db/additional_ueis.sql @@ -6,3 +6,6 @@ CREATE INDEX IF NOT EXISTS fac_snapshot_db_additional_ueis_report_id_idx CREATE INDEX IF NOT EXISTS fac_snapshot_db_additional_ueis_report_id_fad on public_data_v1_0_0.additional_ueis (report_id, fac_accepted_date); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_batch_additional_ueis + ON public_data_v1_0_0.additional_ueis (batch_number); diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/combined.sql b/backend/dissemination/sql/indexes/fac-snapshot-db/combined.sql index 21b9668c8c..aa8771955c 100644 --- a/backend/dissemination/sql/indexes/fac-snapshot-db/combined.sql +++ b/backend/dissemination/sql/indexes/fac-snapshot-db/combined.sql @@ -73,3 +73,6 @@ CREATE INDEX IF NOT EXISTS fac_snapshot_db_combined_audit_year_idx CREATE INDEX IF NOT EXISTS fac_snapshot_db_combined_aln_idx on public_data_v1_0_0.combined (aln); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_batch_combined + ON public_data_v1_0_0.combined (batch_number); diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/corrective_action_plans.sql b/backend/dissemination/sql/indexes/fac-snapshot-db/corrective_action_plans.sql index 5dba259c88..65174c5e5d 100644 --- a/backend/dissemination/sql/indexes/fac-snapshot-db/corrective_action_plans.sql +++ b/backend/dissemination/sql/indexes/fac-snapshot-db/corrective_action_plans.sql @@ -4,8 +4,11 @@ CREATE INDEX IF NOT EXISTS fac_snapshot_db_corrective_action_plans_report_id_idx on public_data_v1_0_0.corrective_action_plans (report_id); -CREATE INDEX IF NOT EXISTS fac_snapshot_db_corrective_action_plans_report_id_fad +CREATE INDEX IF NOT EXISTS fac_snapshot_db_corrective_action_plans_report_id_fad_idx on public_data_v1_0_0.corrective_action_plans (report_id, fac_accepted_date); -CREATE INDEX IF NOT EXISTS fac_snapshot_db_corrective_action_plans_report_id_fad_frn +CREATE INDEX IF NOT EXISTS fac_snapshot_db_corrective_action_plans_report_id_fad_frn_idx on public_data_v1_0_0.corrective_action_plans (report_id, fac_accepted_date, finding_ref_number); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_batch_corrective_action_plans_idx + ON public_data_v1_0_0.corrective_action_plans (batch_number); diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/federal_awards.sql b/backend/dissemination/sql/indexes/fac-snapshot-db/federal_awards.sql index 5a6998f169..f59f983c50 100644 --- a/backend/dissemination/sql/indexes/fac-snapshot-db/federal_awards.sql +++ b/backend/dissemination/sql/indexes/fac-snapshot-db/federal_awards.sql @@ -1,6 +1,9 @@ --------------------------------------- -- INDEXES on federal_awards --------------------------------------- +CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_awards_id_idx + on public_data_v1_0_0.federal_awards (id); + CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_awards_report_id_idx on public_data_v1_0_0.federal_awards (report_id); @@ -27,3 +30,6 @@ CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_awards_rid_aln_fad CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_awards_aln_cnt ON public_data_v1_0_0.federal_awards (aln,findings_count); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_batch_fa + ON public_data_v1_0_0.federal_awards (batch_number); diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/findings.sql b/backend/dissemination/sql/indexes/fac-snapshot-db/findings.sql index ea83d06316..f01f7f2b56 100644 --- a/backend/dissemination/sql/indexes/fac-snapshot-db/findings.sql +++ b/backend/dissemination/sql/indexes/fac-snapshot-db/findings.sql @@ -13,4 +13,7 @@ CREATE INDEX IF NOT EXISTS fac_snapshot_db_findings_rid_aref CREATE INDEX IF NOT EXISTS fac_snapshot_db_findings_rid_aref_fad on public_data_v1_0_0.findings (report_id, award_reference, fac_accepted_date); +CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_batch_findings_idx + ON public_data_v1_0_0.findings (batch_number); + diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/findings_text.sql b/backend/dissemination/sql/indexes/fac-snapshot-db/findings_text.sql index 4efee2f99a..fc1cabde72 100644 --- a/backend/dissemination/sql/indexes/fac-snapshot-db/findings_text.sql +++ b/backend/dissemination/sql/indexes/fac-snapshot-db/findings_text.sql @@ -6,3 +6,6 @@ CREATE INDEX IF NOT EXISTS fac_snapshot_db_findings_text_report_id_idx CREATE INDEX IF NOT EXISTS fac_snapshot_db_findings_text_report_id_fad on public_data_v1_0_0.findings_text (report_id, fac_accepted_date); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_batch_findings_text_idx + ON public_data_v1_0_0.findings_text (batch_number); diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/general.sql b/backend/dissemination/sql/indexes/fac-snapshot-db/general.sql index f590653112..f4e819d109 100644 --- a/backend/dissemination/sql/indexes/fac-snapshot-db/general.sql +++ b/backend/dissemination/sql/indexes/fac-snapshot-db/general.sql @@ -51,3 +51,6 @@ CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditee_ein_idx CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_audit_year_idx on public_data_v1_0_0.general (audit_year); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_batch_general_idx + ON public_data_v1_0_0.general (batch_number); diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/notes_to_sefa.sql b/backend/dissemination/sql/indexes/fac-snapshot-db/notes_to_sefa.sql index 00acf4209b..74649b828c 100644 --- a/backend/dissemination/sql/indexes/fac-snapshot-db/notes_to_sefa.sql +++ b/backend/dissemination/sql/indexes/fac-snapshot-db/notes_to_sefa.sql @@ -6,3 +6,6 @@ CREATE INDEX IF NOT EXISTS fac_snapshot_db_notes_to_sefa_report_id_idx CREATE INDEX IF NOT EXISTS fac_snapshot_db_notes_to_sefa_report_id_fad on public_data_v1_0_0.notes_to_sefa (report_id, fac_accepted_date); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_batch_notes_to_sefa_idx + ON public_data_v1_0_0.notes_to_sefa (batch_number); diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/passthrough.sql b/backend/dissemination/sql/indexes/fac-snapshot-db/passthrough.sql index 673209b62a..7d156e2fe8 100644 --- a/backend/dissemination/sql/indexes/fac-snapshot-db/passthrough.sql +++ b/backend/dissemination/sql/indexes/fac-snapshot-db/passthrough.sql @@ -6,3 +6,6 @@ CREATE INDEX IF NOT EXISTS fac_snapshot_db_passthrough_report_id_idx CREATE INDEX IF NOT EXISTS fac_snapshot_db_passthrough_report_id_fad on public_data_v1_0_0.passthrough (report_id, fac_accepted_date); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_batch_passthrough_idx + ON public_data_v1_0_0.passthrough (batch_number); diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/secondary_auditors.sql b/backend/dissemination/sql/indexes/fac-snapshot-db/secondary_auditors.sql index 3680e04bc9..30684072ea 100644 --- a/backend/dissemination/sql/indexes/fac-snapshot-db/secondary_auditors.sql +++ b/backend/dissemination/sql/indexes/fac-snapshot-db/secondary_auditors.sql @@ -4,5 +4,8 @@ CREATE INDEX IF NOT EXISTS fac_snapshot_db_secondary_auditors_report_id_idx on public_data_v1_0_0.secondary_auditors (report_id); -CREATE INDEX IF NOT EXISTS fac_snapshot_db_secondary_auditors_report_id_fad +CREATE INDEX IF NOT EXISTS fac_snapshot_db_secondary_auditors_report_id_fad_idx on public_data_v1_0_0.secondary_auditors (report_id, fac_accepted_date); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_batch_secondary_auditors_idx + ON public_data_v1_0_0.secondary_auditors (batch_number); diff --git a/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml b/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml index 48f6fe2e25..58ef7e1bcd 100644 --- a/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml +++ b/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml @@ -16,13 +16,13 @@ streams: # The first table we create is the general table. All of the # general table is public data, so we copy it wholesale. public.dissemination_general: - mode: incremental - update_key: id + mode: full-refresh object: public_data_v1_0_0.general sql: | -- All of the general table is public data. SELECT - gen.id, + gen.id as id, + div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_general'), 20000) as batch_number, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -109,17 +109,16 @@ streams: gen.type_audit_code FROM public.dissemination_general gen - WHERE {incremental_where_cond} ORDER BY gen.id # All additional EINs are public. public.additional_eins: - mode: incremental - update_key: ein.id + mode: full-refresh object: public_data_v1_0_0.additional_eins sql: | SELECT - ein.id, + ein.id as id, + div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_additional_eins'), 20000) as batch_number, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -131,18 +130,16 @@ streams: public.dissemination_additionalein ein WHERE gen.report_id = ein.report_id - AND - {incremental_where_cond} ORDER BY ein.id # All of the additional UEI info is public info. public.dissemination_additional_ueis: - mode: incremental - update_key: uei.id + mode: full-refresh object: public_data_v1_0_0.additional_ueis sql: | SELECT - uei.id, + uei.id as id, + div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_additional_ueis'), 20000) as batch_number, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -154,18 +151,16 @@ streams: public.dissemination_additionaluei uei WHERE gen.report_id = uei.report_id - AND - {incremental_where_cond} ORDER BY uei.id # Corrective action plans are NOT always public. public.dissemination_corrective_action_plan: - mode: incremental - update_key: cap.id + mode: full-refresh object: public_data_v1_0_0.corrective_action_plans sql: | SELECT - cap.id, + cap.id as id, + div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_corrective_action_plans'), 20000) as batch_number, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -182,18 +177,16 @@ streams: AND -- Only include the public corrective action plans. gen.is_public = true - AND - {incremental_where_cond} ORDER BY cap.id # All Federal award data is public. public.dissemination_federalaward: - mode: incremental - update_key: award.id + mode: full-refresh object: public_data_v1_0_0.federal_awards sql: | SELECT - award.id, + award.id as id, + div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_federal_awards'), 20000) as batch_number, dg.report_id, dg.auditee_uei, dg.audit_year, @@ -224,18 +217,16 @@ streams: public.dissemination_federalaward award WHERE award.report_id = dg.report_id - AND - {incremental_where_cond} ORDER BY award.id # The findings table is public. public.dissemination_findings: - mode: incremental - update_key: finding.id + mode: full-refresh object: public_data_v1_0_0.findings sql: | SELECT - finding.id, + finding.id as id, + div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_findings'), 20000) as batch_number, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -257,18 +248,16 @@ streams: public.dissemination_finding finding WHERE finding.report_id = gen.report_id - AND - {incremental_where_cond} ORDER BY finding.id # Findings text is NOT always public. public.dissemination_findingstext: - mode: incremental - update_key: ft.id + mode: full-refresh object: public_data_v1_0_0.findings_text sql: | SELECT - ft.id, + ft.id as id, + div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_findings_text'), 20000) as batch_number, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -283,18 +272,16 @@ streams: ft.report_id = gen.report_id AND gen.is_public = true - AND - {incremental_where_cond} ORDER BY ft.id # The notes to SEFA are NOT all public. public.dissemination_notes: - mode: incremental - update_key: note.id + mode: full-refresh object: public_data_v1_0_0.notes_to_sefa sql: | SELECT - note.id, + note.id as id, + div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_notes_to_sefa'), 20000) as batch_number, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -314,18 +301,16 @@ streams: AND -- Some notes are not public. gen.is_public = true - AND - {incremental_where_cond} ORDER BY note.id # All passthrough information is public. public.dissemination_passthrough: - mode: incremental - update_key: pass.id + mode: full-refresh object: public_data_v1_0_0.passthrough sql: | SELECT - pass.id, + pass.id as id, + div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_passthrough'), 20000) as batch_number, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -339,18 +324,16 @@ streams: public.dissemination_passthrough pass WHERE gen.report_id = pass.report_id - AND - {incremental_where_cond} ORDER BY pass.id # All secondary auditor info is public. public.secondary_auditors: - mode: incremental - update_key: sa.id + mode: full-refresh object: public_data_v1_0_0.secondary_auditors sql: | SELECT - sa.id, + sa.id as id, + div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_secondary_auditors'), 20000) as batch_number, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -371,8 +354,6 @@ streams: public.dissemination_secondaryauditor sa WHERE sa.report_id = gen.report_id - AND - {incremental_where_cond} ORDER BY sa.id public.combined: @@ -380,14 +361,15 @@ streams: object: public_data_v1_0_0.combined sql: | SELECT - nextval('public_data_v1_0_0.seq_combined') AS id, dg.report_id, + div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_combined'), 20000) as batch_number, dfa.award_reference, df.reference_number, concat(dfa.federal_agency_prefix,'.',dfa.federal_award_extension) as aln, -- -- general -- + dg.id as general_row_id, dg.auditee_uei, dg.audit_year, dg.agencies_with_prior_findings, @@ -452,6 +434,7 @@ streams: -- -- federal_award -- + dfa.id as federal_award_row_id, dfa.additional_award_identification, dfa.amount_expended, dfa.audit_report_type, @@ -473,6 +456,7 @@ streams: -- -- finding -- + df.id as finding_row_id, df.is_material_weakness, df.is_modified_opinion, df.is_other_findings, @@ -485,6 +469,7 @@ streams: -- -- passthrough -- + dp.id as passthrough_row_id, dp.passthrough_id, dp.passthrough_name FROM @@ -497,50 +482,3 @@ streams: LEFT JOIN public.dissemination_passthrough dp ON dfa.report_id = dp.report_id AND dfa.award_reference = dp.award_reference - - # This looks at the tables after they have been updated, and - # generates metadata that agencies use to determine if they - # have completely pulled all of the data in their systems. - public_data_v1_0_0.metadata: - mode: full-refresh - object: public_data_v1_0_0.metadata_v2_0_0_a - sql: | - SELECT 'additional_eins' - AS table, COUNT(*), public_api_v2_0_0_alpha_functions.batches('additional_eins') - FROM public_data_v1_0_0.additional_eins - UNION - SELECT 'additional_ueis' - AS table, COUNT(*), public_api_v2_0_0_alpha_functions.batches('additional_ueis') - FROM public_data_v1_0_0.additional_ueis - UNION - SELECT 'combined' - AS table, COUNT(*), public_api_v2_0_0_alpha_functions.batches('combined') - FROM public_data_v1_0_0.combined - UNION - SELECT 'federal_awards' - AS table, COUNT(*), public_api_v2_0_0_alpha_functions.batches('federal_awards') - FROM public_data_v1_0_0.federal_awards - UNION - SELECT 'findings_text' - AS table, COUNT(*), public_api_v2_0_0_alpha_functions.batches('findings_text') - FROM public_data_v1_0_0.findings_text - UNION - SELECT 'findings' - AS table, COUNT(*), public_api_v2_0_0_alpha_functions.batches('findings') - FROM public_data_v1_0_0.findings - UNION - SELECT 'general' - AS table, COUNT(*), public_api_v2_0_0_alpha_functions.batches('general') - FROM public_data_v1_0_0.general - UNION - SELECT 'notes_to_sefa' - AS table, COUNT(*), public_api_v2_0_0_alpha_functions.batches('notes_to_sefa') - FROM public_data_v1_0_0.notes_to_sefa - UNION - SELECT 'passthrough' - AS table, COUNT(*), public_api_v2_0_0_alpha_functions.batches('passthrough') - FROM public_data_v1_0_0.passthrough - UNION - SELECT 'secondary_auditors' - AS table, COUNT(*), public_api_v2_0_0_alpha_functions.batches('secondary_auditors') - FROM public_data_v1_0_0.secondary_auditors diff --git a/backend/dissemination/sql/sling/public_data_v1_0_0/public_metadata_v1_0_0.yaml b/backend/dissemination/sql/sling/public_data_v1_0_0/public_metadata_v1_0_0.yaml new file mode 100644 index 0000000000..34afe850d1 --- /dev/null +++ b/backend/dissemination/sql/sling/public_data_v1_0_0/public_metadata_v1_0_0.yaml @@ -0,0 +1,73 @@ +# This uses an env var called PG to +# set the connection string. +source: FAC_SNAPSHOT_URI +target: FAC_SNAPSHOT_URI +env: + SLING_ALLOW_EMPTY: true + +# Handy +# SELECT column_name +# FROM information_schema.columns +# WHERE table_schema = 'public' +# AND table_name = 'dissemination_general' +# AND column_name NOT IN ('id'); + +streams: + # This looks at the tables after they have been updated, and + # generates metadata that agencies use to determine if they + # have completely pulled all of the data in their systems. + public_data_v1_0_0.metadata: + mode: full-refresh + object: public_data_v1_0_0.metadata + sql: | + SELECT 'additional_eins' + AS table, COUNT(*) --, public_api_v2_0_0_alpha_functions.batches('additional_eins') + FROM public_data_v1_0_0.additional_eins + UNION + SELECT 'additional_ueis' + AS table, COUNT(*) --, public_api_v2_0_0_alpha_functions.batches('additional_ueis') + FROM public_data_v1_0_0.additional_ueis + UNION + SELECT 'combined' + AS table, COUNT(*) --, public_api_v2_0_0_alpha_functions.batches('combined') + FROM public_data_v1_0_0.combined + UNION + SELECT 'federal_awards' + AS table, COUNT(*) --, public_api_v2_0_0_alpha_functions.batches('federal_awards') + FROM public_data_v1_0_0.federal_awards + UNION + SELECT 'findings_text' + AS table, COUNT(*) --, public_api_v2_0_0_alpha_functions.batches('findings_text') + FROM public_data_v1_0_0.findings_text + UNION + SELECT 'findings' + AS table, COUNT(*) --, public_api_v2_0_0_alpha_functions.batches('findings') + FROM public_data_v1_0_0.findings + UNION + SELECT 'general' + AS table, COUNT(*) --, public_api_v2_0_0_alpha_functions.batches('general') + FROM public_data_v1_0_0.general + UNION + SELECT 'notes_to_sefa' + AS table, COUNT(*) --, public_api_v2_0_0_alpha_functions.batches('notes_to_sefa') + FROM public_data_v1_0_0.notes_to_sefa + UNION + SELECT 'passthrough' + AS table, COUNT(*) --, public_api_v2_0_0_alpha_functions.batches('passthrough') + FROM public_data_v1_0_0.passthrough + UNION + SELECT 'secondary_auditors' + AS table, COUNT(*) --, public_api_v2_0_0_alpha_functions.batches('secondary_auditors') + FROM public_data_v1_0_0.secondary_auditors + UNION + SELECT 'tribal_corrective_action_plans' + AS table, COUNT(*) --, public_api_v2_0_0_alpha_functions.batches('tribal_corrective_action_plans') + FROM public_data_v1_0_0.tribal_corrective_action_plans + UNION + SELECT 'tribal_findings_text' + AS table, COUNT(*) --, public_api_v2_0_0_alpha_functions.batches('tribal_findings_text') + FROM public_data_v1_0_0.tribal_findings_text + UNION + SELECT 'tribal_notes_to_sefa' + AS table, COUNT(*) --, public_api_v2_0_0_alpha_functions.batches('tribal_notes_to_sefa') + FROM public_data_v1_0_0.tribal_notes_to_sefa diff --git a/backend/dissemination/sql/sling/public_data_v1_0_0/tribal_data_v1_0_0.yaml b/backend/dissemination/sql/sling/public_data_v1_0_0/tribal_data_v1_0_0.yaml new file mode 100644 index 0000000000..071294fa23 --- /dev/null +++ b/backend/dissemination/sql/sling/public_data_v1_0_0/tribal_data_v1_0_0.yaml @@ -0,0 +1,90 @@ +# This uses an env var called PG to +# set the connection string. +source: FAC_SNAPSHOT_URI +target: FAC_SNAPSHOT_URI +env: + SLING_ALLOW_EMPTY: true + +# Handy +# SELECT column_name +# FROM information_schema.columns +# WHERE table_schema = 'public' +# AND table_name = 'dissemination_general' +# AND column_name NOT IN ('id'); + +streams: + # Corrective action plans are NOT always public. + public.dissemination_corrective_action_plan: + mode: full-refresh + object: public_data_v1_0_0.tribal_corrective_action_plans + sql: | + SELECT + cap.id as id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + cap.contains_chart_or_table, + cap.finding_ref_number, + cap.planned_action + FROM + public_data_v1_0_0.general gen, + public.dissemination_captext cap + WHERE + cap.report_id = gen.report_id + AND + -- Only include the public corrective action plans. + gen.is_public = false + ORDER BY cap.id + + # Findings text is NOT always public. + public.dissemination_findingstext: + mode: full-refresh + object: public_data_v1_0_0.tribal_findings_text + sql: | + SELECT + ft.id as id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + ft.finding_ref_number, + ft.contains_chart_or_table, + ft.finding_text + FROM + public_data_v1_0_0.general gen, + public.dissemination_findingtext ft + WHERE + ft.report_id = gen.report_id + AND + gen.is_public = false + ORDER BY ft.id + + # The notes to SEFA are NOT all public. + public.dissemination_notes: + mode: full-refresh + object: public_data_v1_0_0.tribal_notes_to_sefa + sql: | + SELECT + note.id as id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + note.accounting_policies, + note.contains_chart_or_table, + note.content, + note.is_minimis_rate_used, + note.note_title as title, + note.rate_explained + FROM + public_data_v1_0_0.general gen, + public.dissemination_note note + WHERE + note.report_id = gen.report_id + AND + -- Some notes are not public. + gen.is_public = false + ORDER BY note.id diff --git a/backend/docker-compose-db-only.yml b/backend/docker-compose-db-only.yml new file mode 100644 index 0000000000..a88741898c --- /dev/null +++ b/backend/docker-compose-db-only.yml @@ -0,0 +1,71 @@ +services: + #--------------------------------------------- + # Postgres DB + #--------------------------------------------- + db: + image: "postgres:15" + environment: + POSTGRES_HOST_AUTH_METHOD: trust + volumes: + - postgres-data:/var/lib/postgresql/data/ + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -d postgres -U postgres"] + interval: 10s + timeout: 5s + retries: 10 + db2: + image: "postgres:15" + environment: + POSTGRES_HOST_AUTH_METHOD: "trust" + volumes: + - postgres-data2:/var/lib/postgresql/data/ + ports: + - "5431:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -d postgres -U postgres"] + interval: 10s + timeout: 5s + retries: 10 + + #--------------------------------------------- + # Minio (S3 clone) + #--------------------------------------------- + minio: + container_name: "minio" + image: minio/minio + command: server /tmp/minio --console-address ":9002" + ports: + - "9001:9000" + - "9002:9002" + volumes: + - "minio-vol:/tmp/minio" + + #--------------------------------------------- + # PostgREST API provider + #--------------------------------------------- + api: + image: ghcr.io/gsa-tts/fac/postgrest:latest + ports: + - "3000:3000" + expose: + - "3000" + environment: + # This now requires us to `sling` data to db2 for local API testing. + PGRST_DB_URI: postgres://postgres@db2:5432/postgres + # PGRST2_DB_URI: postgres://postgres@db:5431/postgres + PGRST_OPENAPI_SERVER_PROXY_URI: http://127.0.0.1:3000 + PGRST_DB_ANON_ROLE: anon + # See https://postgrest.org/en/stable/references/api/schemas.html#multiple-schemas for multiple schemas + PGRST_DB_SCHEMAS: "api_v1_0_3,api_v1_1_0,admin_api_v1_1_0,public_api_v2_0_0_alpha" + PGRST_JWT_SECRET: ${PGRST_JWT_SECRET:-32_chars_fallback_secret_testing} # Fallback value for testing environments + # Enable this to inspect the DB plans for queries via EXPLAIN + PGRST_DB_PLAN_ENABLED: 1 + depends_on: + db2: + condition: service_healthy +volumes: + postgres-data: + postgres-data2: + minio-vol: diff --git a/backend/run.sh b/backend/run.sh index 0024e08a82..ce257309a8 100755 --- a/backend/run.sh +++ b/backend/run.sh @@ -10,9 +10,11 @@ source tools/util_startup.sh # for local envs (LOCAL or TESTING) and cloud.gov source tools/setup_env.sh source tools/api_teardown.sh -source tools/build_indexes.sh source tools/migrate_app_tables.sh +source tools/sling_support_functions.sh +source tools/sling_first_run.sh source tools/api_standup.sh +source tools/build_indexes.sh source tools/seed_cog_baseline.sh ##### @@ -32,6 +34,13 @@ gonogo "api_teardown" # migrate_app_tables gonogo "migrate_app_tables" +##### +# PREP API TABLES +# This runs sling and preps tables in the snapshot DB. +# Only runs if the tables are not present (e.g. first deploy) +sling_first_run +gonogo "sling_first_run" + ##### # API STANDUP # Standup the API, which may depend on migration changes diff --git a/backend/tools/api_teardown.sh b/backend/tools/api_teardown.sh index ded6ebbcbb..3fc3addfb4 100644 --- a/backend/tools/api_teardown.sh +++ b/backend/tools/api_teardown.sh @@ -1,37 +1,5 @@ source tools/util_startup.sh -function teardown_public_tables () { - run_sql $FAC_DB_URI 'dissemination/sql' 'sling' 'public_data_v1_0_0' 'teardown.sql' - return 0 -} - -function standup_public_table_placeholders () { - # run_sql has its own gonogo built in. - # $SLING_EXE run -r dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml - run_sql $FAC_DB_URI 'dissemination/sql' 'sling' 'public_data_v1_0_0' 'standup.sql' - return 0 -} - -function run_sql_for_public_apis () { - local location="$1" - local sql_file="$2" - local base_path='dissemination/sql' - local location='api' - - for api_version in "${public_api_versions[@]}" - do - if [ -f ${base_path}/${location}/${api_version}/${sql_file} ]; then - # $PSQL_EXE $FAC_DB_URI < ${base_path}/${location}/${api_version}/${sql_file} - run_sql $FAC_DB_URI $base_path $location $api_version $sql_file - else - echo "API FILE NOT FOUND/SKIPPED ${location}/${api_version}/${sql_file}" - fi - done - -} - - - function api_teardown { startup_log "API_TEARDOWN" "BEGIN" local base_path='dissemination/sql' @@ -43,16 +11,16 @@ function api_teardown { # just the permissions at this point. That is, we're tearing down # the API portion. The data will remain in the DB. - #run_sql_for_public_apis 'api' 'teardown.sql' - for api_version in "${public_api_versions[@]}" + for index in "${!api_versions[@]}" do - if [ -f ${base_path}/${location}/${api_version}/${sql_file} ]; then - # $PSQL_EXE $FAC_DB_URI < ${base_path}/${location}/${api_version}/${sql_file} - run_sql $FAC_DB_URI $base_path $location $api_version $sql_file - gonogo "$api_version teardown.sql" - else - echo "API FILE NOT FOUND/SKIPPED ${location}/${api_version}/${sql_file}" - fi + local api_version="${api_versions[index]}" + local required_table="${api_required_tables[index]}" + if [ -f ${base_path}/${location}/${api_version}/${sql_file} ]; then + run_sql $FAC_SNAPSHOT_URI $base_path $location $api_version $sql_file + gonogo "$api_version teardown.sql" + else + echo "API FILE NOT FOUND/SKIPPED ${location}/${api_version}/${sql_file}" + fi done startup_log "API_TEARDOWN" "END" diff --git a/backend/tools/load_public_dissem_data/data/config.json b/backend/tools/load_public_dissem_data/data/config.json deleted file mode 100644 index 4551b86b62..0000000000 --- a/backend/tools/load_public_dissem_data/data/config.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "user-provided": [ - { - "label": "mc", - "name": "backups", - "tags": [], - "instance_guid": "UUIDALPHA1", - "instance_name": "backups", - "binding_guid": "UUIDALPHA2", - "binding_name": null, - "credentials": { - "access_key_id": "longtest", - "secret_access_key": "longtest", - "bucket": "backups", - "endpoint": "http://127.0.0.1:9001", - "admin_username": "minioadmin", - "admin_password": "minioadmin" - } - }, - { - "label": "mc", - "name": "fac-private-s3", - "tags": [], - "instance_guid": "UUIDALPHA1", - "instance_name": "fac-private-s3", - "binding_guid": "UUIDALPHA2", - "binding_name": null, - "credentials": { - "access_key_id": "longtest", - "secret_access_key": "longtest", - "bucket": "fac-private-s3", - "endpoint": "http://127.0.0.1:9001", - "admin_username": "minioadmin", - "admin_password": "minioadmin" - } - } - ], - "aws-rds": [ - { - "label": "fac-db", - "provider": null, - "plan": null, - "name": "fac-db", - "tags": [ - "database", - "docker" - ], - "instance_guid": "UUIDINDIA1", - "instance_name": "db", - "binding_guid": "UUIDINDIA2", - "binding_name": null, - "credentials": { - "db_name": "postgres", - "host": "db", - "name": "postgres", - "password": "", - "port": "5432", - "uri": "postgres://postgres@db:5432/postgres?sslmode=disable", - "username": "postgres" - }, - "syslog_drain_url": null, - "volume_mounts": [] - }, - { - "label": "fac-snapshot-db", - "provider": null, - "plan": null, - "name": "fac-snapshot-db", - "tags": [ - "database", - "docker" - ], - "instance_guid": "UUIDJULIET1", - "instance_name": "db", - "binding_guid": "UUIDJULIET2", - "binding_name": null, - "credentials": { - "db_name": "postgres", - "host": "db2", - "name": "postgres", - "password": "", - "port": "5432", - "uri": "postgres://postgres@db2:5432/postgres?sslmode=disable", - "username": "postgres" - }, - "syslog_drain_url": null, - "volume_mounts": [] - } - ] -} diff --git a/backend/tools/load_public_dissem_data/data/public_data_v1_0_0.yaml b/backend/tools/load_public_dissem_data/data/public_data_v1_0_0.yaml deleted file mode 100644 index 093de28a6b..0000000000 --- a/backend/tools/load_public_dissem_data/data/public_data_v1_0_0.yaml +++ /dev/null @@ -1,499 +0,0 @@ -# This uses an env var called PG to -# set the connection string. -source: FAC_SNAPSHOT_URI -target: FAC_SNAPSHOT_URI -env: - SLING_ALLOW_EMPTY: true - -# Handy -# SELECT column_name -# FROM information_schema.columns -# WHERE table_schema = 'public' -# AND table_name = 'dissemination_general' -# AND column_name NOT IN ('id'); - -streams: - # The first table we create is the general table. All of the - # general table is public data, so we copy it wholesale. - public.dissemination_general: - mode: incremental - update_key: id - object: public_data_v1_0_0.general - sql: | - -- All of the general table is public data. - SELECT - gen.id, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.agencies_with_prior_findings, - gen.audit_period_covered, - gen.audit_type, - gen.auditee_address_line_1, - gen.auditee_certified_date, - gen.auditee_certify_name, - gen.auditee_certify_title, - gen.auditee_city, - gen.auditee_contact_name, - gen.auditee_contact_title, - gen.auditee_ein, - gen.auditee_email, - gen.auditee_name, - gen.auditee_phone, - gen.auditee_state, - gen.auditee_zip, - gen.auditor_address_line_1, - gen.auditor_certified_date, - gen.auditor_certify_name, - gen.auditor_certify_title, - gen.auditor_city, - gen.auditor_contact_name, - gen.auditor_contact_title, - gen.auditor_country, - gen.auditor_ein, - gen.auditor_email, - gen.auditor_firm_name, - gen.auditor_foreign_address, - gen.auditor_phone, - gen.auditor_state, - gen.auditor_zip, - gen.cognizant_agency, - gen.data_source, - gen.date_created, - gen.dollar_threshold, - gen.entity_type, - gen.fac_accepted_date, - gen.fy_end_date, - gen.fy_start_date, - gen.gaap_results, - gen.is_additional_ueis, - gen.is_aicpa_audit_guide_included, - gen.is_going_concern_included, - gen.is_internal_control_deficiency_disclosed, - gen.is_internal_control_material_weakness_disclosed, - gen.is_low_risk_auditee, - gen.is_material_noncompliance_disclosed, - CASE EXISTS - ( - SELECT - ein.report_id - FROM - dissemination_additionalein ein - WHERE - ein.report_id = gen.report_id - ) - WHEN FALSE THEN 'No' - ELSE 'Yes' - END AS is_multiple_eins, - gen.is_public, - CASE EXISTS - ( - SELECT - aud.report_id - FROM - dissemination_secondaryauditor aud - WHERE - aud.report_id = gen.report_id - ) - WHEN FALSE THEN 'No' - ELSE 'Yes' - END AS is_secondary_auditors, - gen.is_sp_framework_required, - gen.number_months, - gen.oversight_agency, - gen.ready_for_certification_date, - gen.sp_framework_basis, - gen.sp_framework_opinions, - gen.submitted_date, - gen.total_amount_expended, - gen.type_audit_code - FROM - public.dissemination_general gen - WHERE {incremental_where_cond} - ORDER BY gen.id - - # All additional EINs are public. - public.additional_eins: - mode: incremental - update_key: ein.id - object: public_data_v1_0_0.additional_eins - sql: | - SELECT - ein.id, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - ein.additional_ein - FROM - public_data_v1_0_0.general gen, - public.dissemination_additionalein ein - WHERE - gen.report_id = ein.report_id - AND - {incremental_where_cond} - ORDER BY ein.id - - # All of the additional UEI info is public info. - public.dissemination_additional_ueis: - mode: incremental - update_key: uei.id - object: public_data_v1_0_0.additional_ueis - sql: | - SELECT - uei.id, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - uei.additional_uei - FROM - public_data_v1_0_0.general gen, - public.dissemination_additionaluei uei - WHERE - gen.report_id = uei.report_id - AND - {incremental_where_cond} - ORDER BY uei.id - - # Corrective action plans are NOT always public. - public.dissemination_corrective_action_plan: - mode: incremental - update_key: cap.id - object: public_data_v1_0_0.corrective_action_plans - sql: | - SELECT - cap.id, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - cap.contains_chart_or_table, - cap.finding_ref_number, - cap.planned_action - FROM - public_data_v1_0_0.general gen, - public.dissemination_captext cap - WHERE - cap.report_id = gen.report_id - AND - -- Only include the public corrective action plans. - gen.is_public = true - AND - {incremental_where_cond} - ORDER BY cap.id - - # All Federal award data is public. - public.dissemination_federalaward: - mode: incremental - update_key: award.id - object: public_data_v1_0_0.federal_awards - sql: | - SELECT - award.id, - dg.report_id, - dg.auditee_uei, - dg.audit_year, - dg.fac_accepted_date, - concat(award.federal_agency_prefix,'.',award.federal_award_extension) as aln, - --- - award.additional_award_identification, - award.amount_expended, - award.audit_report_type, - award.award_reference, - award.cluster_name, - award.cluster_total, - award.federal_agency_prefix, - award.federal_award_extension, - award.federal_program_name, - award.federal_program_total, - award.findings_count, - award.is_direct, - award.is_loan, - award.is_major, - award.is_passthrough_award, - award.loan_balance, - award.other_cluster_name, - award.passthrough_amount, - award.state_cluster_name - FROM - public_data_v1_0_0.general dg, - public.dissemination_federalaward award - WHERE - award.report_id = dg.report_id - AND - {incremental_where_cond} - ORDER BY award.id - - # The findings table is public. - public.dissemination_findings: - mode: incremental - update_key: finding.id - object: public_data_v1_0_0.findings - sql: | - SELECT - finding.id, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - finding.award_reference, - finding.is_material_weakness, - finding.is_modified_opinion, - finding.is_other_findings, - finding.is_other_matters, - finding.is_questioned_costs, - finding.is_repeat_finding, - finding.is_significant_deficiency, - finding.prior_finding_ref_numbers, - finding.reference_number, - finding.type_requirement - FROM - public_data_v1_0_0.general gen, - public.dissemination_finding finding - WHERE - finding.report_id = gen.report_id - AND - {incremental_where_cond} - ORDER BY finding.id - - # Findings text is NOT always public. - public.dissemination_findingstext: - mode: incremental - update_key: ft.id - object: public_data_v1_0_0.findings_text - sql: | - SELECT - ft.id, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - ft.finding_ref_number, - ft.contains_chart_or_table, - ft.finding_text - FROM - public_data_v1_0_0.general gen, - public.dissemination_findingtext ft - WHERE - ft.report_id = gen.report_id - AND - gen.is_public = true - AND - {incremental_where_cond} - ORDER BY ft.id - - # The notes to SEFA are NOT all public. - public.dissemination_notes: - mode: incremental - update_key: note.id - object: public_data_v1_0_0.notes_to_sefa - sql: | - SELECT - note.id, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - note.accounting_policies, - note.contains_chart_or_table, - note.content, - note.is_minimis_rate_used, - note.note_title as title, - note.rate_explained - FROM - public_data_v1_0_0.general gen, - public.dissemination_note note - WHERE - note.report_id = gen.report_id - AND - -- Some notes are not public. - gen.is_public = true - AND - {incremental_where_cond} - ORDER BY note.id - - # All passthrough information is public. - public.dissemination_passthrough: - mode: incremental - update_key: pass.id - object: public_data_v1_0_0.passthrough - sql: | - SELECT - pass.id, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - pass.award_reference, - pass.passthrough_id, - pass.passthrough_name - FROM - public_data_v1_0_0.general gen, - public.dissemination_passthrough pass - WHERE - gen.report_id = pass.report_id - AND - {incremental_where_cond} - ORDER BY pass.id - - # All secondary auditor info is public. - public.secondary_auditors: - mode: incremental - update_key: sa.id - object: public_data_v1_0_0.secondary_auditors - sql: | - SELECT - sa.id, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - sa.address_city, - sa.address_state, - sa.address_street, - sa.address_zipcode, - sa.auditor_ein, - sa.auditor_name, - sa.contact_email, - sa.contact_name, - sa.contact_phone, - sa.contact_title - FROM - public_data_v1_0_0.general gen, - public.dissemination_secondaryauditor sa - WHERE - sa.report_id = gen.report_id - AND - {incremental_where_cond} - ORDER BY sa.id - - public.combined: - mode: full-refresh - object: public_data_v1_0_0.combined - sql: | - SELECT - nextval('public_data_v1_0_0.seq_combined') AS id, - dg.report_id, - dfa.award_reference, - df.reference_number, - concat(dfa.federal_agency_prefix,'.',dfa.federal_award_extension) as aln, - -- - -- general - -- - dg.auditee_uei, - dg.audit_year, - dg.agencies_with_prior_findings, - dg.audit_period_covered, - dg.audit_type, - dg.auditee_address_line_1, - dg.auditee_certified_date, - dg.auditee_certify_name, - dg.auditee_certify_title, - dg.auditee_city, - dg.auditee_contact_name, - dg.auditee_contact_title, - dg.auditee_ein, - dg.auditee_email, - dg.auditee_name, - dg.auditee_phone, - dg.auditee_state, - dg.auditee_zip, - dg.auditor_address_line_1, - dg.auditor_certified_date, - dg.auditor_certify_name, - dg.auditor_certify_title, - dg.auditor_city, - dg.auditor_contact_name, - dg.auditor_contact_title, - dg.auditor_country, - dg.auditor_ein, - dg.auditor_email, - dg.auditor_firm_name, - dg.auditor_foreign_address, - dg.auditor_phone, - dg.auditor_state, - dg.auditor_zip, - dg.cognizant_agency, - dg.data_source, - dg.date_created, - dg.dollar_threshold, - dg.entity_type, - dg.fac_accepted_date, - dg.fy_end_date, - dg.fy_start_date, - dg.gaap_results, - dg.is_additional_ueis, - dg.is_aicpa_audit_guide_included, - dg.is_going_concern_included, - dg.is_internal_control_deficiency_disclosed, - dg.is_internal_control_material_weakness_disclosed, - dg.is_low_risk_auditee, - dg.is_material_noncompliance_disclosed, - dg.is_multiple_eins, - dg.is_public, - dg.is_secondary_auditors, - dg.is_sp_framework_required, - dg.number_months, - dg.oversight_agency, - dg.ready_for_certification_date, - dg.sp_framework_basis, - dg.sp_framework_opinions, - dg.submitted_date, - dg.total_amount_expended, - dg.type_audit_code, - -- - -- federal_award - -- - dfa.additional_award_identification, - dfa.amount_expended, - dfa.audit_report_type, - dfa.cluster_name, - dfa.cluster_total, - dfa.federal_agency_prefix, - dfa.federal_award_extension, - dfa.federal_program_name, - dfa.federal_program_total, - dfa.findings_count, - dfa.is_direct, - dfa.is_loan, - dfa.is_major, - dfa.is_passthrough_award, - dfa.loan_balance, - dfa.other_cluster_name, - dfa.passthrough_amount, - dfa.state_cluster_name, - -- - -- finding - -- - df.is_material_weakness, - df.is_modified_opinion, - df.is_other_findings, - df.is_other_matters, - df.is_questioned_costs, - df.is_repeat_finding, - df.is_significant_deficiency, - df.prior_finding_ref_numbers, - df.type_requirement, - -- - -- passthrough - -- - dp.passthrough_id, - dp.passthrough_name - FROM - public.dissemination_federalaward dfa - LEFT JOIN public_data_v1_0_0.general dg - ON dfa.report_id = dg.report_id - LEFT JOIN public.dissemination_finding df - ON dfa.report_id = df.report_id - AND dfa.award_reference = df.award_reference - LEFT JOIN public.dissemination_passthrough dp - ON dfa.report_id = dp.report_id - AND dfa.award_reference = dp.award_reference diff --git a/backend/tools/setup_local_env.sh b/backend/tools/setup_local_env.sh index ed2f76d437..68f2e705a6 100644 --- a/backend/tools/setup_local_env.sh +++ b/backend/tools/setup_local_env.sh @@ -1,20 +1,5 @@ source tools/util_startup.sh -function install_local_sling { - curl -LO 'https://github.com/slingdata-io/sling-cli/releases/latest/download/sling_linux_amd64.tar.gz' - gonogo "curl sling" - tar xf sling_linux_amd64.tar.gz - gonogo "tar xf sling" - rm -f sling_linux_amd64.tar.gz - gonogo "rm sling.tar" - chmod +x sling - gonogo "chmod sling" - mv sling /usr/local/bin/sling - gonogo "mv sling" - export SLING_EXE='/usr/local/bin/sling' - return 0 -} - function setup_local_env { if [[ "${ENV}" == "LOCAL" || "${ENV}" == "TESTING" ]]; then @@ -32,8 +17,25 @@ function setup_local_env { export FAC_DB_URI=${DATABASE_URL}?sslmode=disable export FAC_SNAPSHOT_URI=${SNAPSHOT_URL}?sslmode=disable export PSQL_EXE='psql -v ON_ERROR_STOP=on' - # install_local_sling + export SLING_EXE='/bin/sling' + export CGOV_UTIL_EXE='/bin/cgov-util' + # Locally, we need to pull in sling. + # In production, it gets pulled in via the build/deploy process. + curl -LO 'https://github.com/slingdata-io/sling-cli/releases/latest/download/sling_linux_amd64.tar.gz' + tar xf sling_linux_amd64.tar.gz + rm -f sling_linux_amd64.tar.gz + chmod +x sling + mv sling /bin/sling + # And we need cgov-util + curl -L -O https://github.com/GSA-TTS/fac-backup-utility/releases/download/v0.1.8/gov.gsa.fac.cgov-util-v0.1.8-linux-amd64.tar.gz + tar xvzf gov.gsa.fac.cgov-util-v0.1.8-linux-amd64.tar.gz gov.gsa.fac.cgov-util + chmod 755 gov.gsa.fac.cgov-util + mv gov.gsa.fac.cgov-util /bin/cgov-util + # We need a config.json in the directory we are running + # things from (or PWD). + cp util/load_public_dissem_data/data/config.json . + gonogo "local copy of config for cgov-util" return 0 fi; } diff --git a/backend/tools/sling_first_run.sh b/backend/tools/sling_first_run.sh new file mode 100644 index 0000000000..0049391ed2 --- /dev/null +++ b/backend/tools/sling_first_run.sh @@ -0,0 +1,40 @@ +source tools/util_startup.sh + +function sling_first_run() { + startup_log "SLING_FIRST_RUN" "Slinging data to fac-snapshot if needed" + + check_table_exists $FAC_SNAPSHOT_URI 'public.dissemination_general' + local is_general_table=$? + if [ $is_general_table -ne 0 ]; then + # This is the first run. + startup_log "SLING_FIRST_RUN" "Running cgov-util INITIAL." + $CGOV_UTIL_EXE db_to_db \ + --src_db fac-db \ + --dest_db fac-snapshot-db \ + --operation initial + fi + + # If the metadata table exists, it means sling has run to completion. + check_table_exists $FAC_SNAPSHOT_URI 'public_data_v1_0_0.metadata' + local is_metadata_table=$? + # We need to load some functions for sling to complete, because + # we use those functions as part of the metadata table generation. + local base_path='dissemination/sql' + local location='sling' + + # Only run sling if the tables in the secondary DB do not exist. + if [ $is_metadata_table -ne 0 ]; then + startup_log "SLING_FIRST_RUN" "API tables don't exist; running sling." + $SLING_EXE run -r dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml + gonogo "sling public data for API tables" + $SLING_EXE run -r dissemination/sql/sling/public_data_v1_0_0/tribal_data_v1_0_0.yaml + gonogo "sling tribal data for API tables" + $SLING_EXE run -r dissemination/sql/sling/public_data_v1_0_0/public_metadata_v1_0_0.yaml + gonogo "sling tribal data for API tables" + startup_log "SLING_FIRST_RUN" "Successfully ran sling to generate tables." + else + startup_log "SLING_FIRST_RUN" "API tables exist; skipping sling." + fi + startup_log "SLING_FIRST_RUN" "Done" + return 0 +} diff --git a/backend/tools/sling_support_functions.sh b/backend/tools/sling_support_functions.sh new file mode 100644 index 0000000000..644a5f3434 --- /dev/null +++ b/backend/tools/sling_support_functions.sh @@ -0,0 +1,15 @@ +source tools/util_startup.sh + +function sling_support_functions() { + startup_log "SLING_SUPPORT_FUNCTIONS" "Loading functions needed by sling" + + check_table_exists $FAC_SNAPSHOT_URI 'public_data_v1_0_0.general' + gonogo "sling_support_functions general exists" + + # We need to load some functions for sling to complete, because + # we use those functions as part of the metadata table generation. + local base_path='dissemination/sql' + local location='sling' + run_sql $FAC_SNAPSHOT_URI $base_path $location 'public_data_v1_0_0' 'sling_functions.sql' + return $? +} diff --git a/backend/tools/variables.sh b/backend/tools/variables.sh index e1aaf992e2..43d7d0fa74 100644 --- a/backend/tools/variables.sh +++ b/backend/tools/variables.sh @@ -8,6 +8,7 @@ declare -a api_versions=( "admin_api_v1_1_0" "admin_api_v1_1_1" "public_api_v2_0_0_alpha" + "public_data_v1_0_0" ) # These are the tables that must be present # in order to stand up that API. @@ -18,6 +19,7 @@ declare -a api_required_tables=( "public.support_administrative_key_uuids" "public.support_administrative_key_uuids" "public_data_v1_0_0.general" + "public.dissemination_general" ) declare -a db2_indexes=( diff --git a/backend/util/__init__.py b/backend/util/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/util/api_perf_test/perf.py b/backend/util/api_perf_test/perf.py new file mode 100644 index 0000000000..a349101706 --- /dev/null +++ b/backend/util/api_perf_test/perf.py @@ -0,0 +1,119 @@ +import requests +import os +import time +from pprint import pprint +import math + +URI = "http://localhost:3000" + +# GET {{scheme}}://{{apiUrl}}/general?report_id=eq.2021-12-CENSUS-0000250449 +# authorization: {{authorization}} +# x-api-user-id: {{xApiUserId}} +# accept-profile: public_api_v2_0_0_alpha +# Accept: application/vnd.pgrst.plan + + +def fetch_fa_exp(api_version): + total_cost = 0 + for offset in range(0, 4000000, 20000): + query = f"{URI}/federal_awards?limit=20000&offset={offset}" + headers = { + "accept-profile": api_version, + "accept": "application/vnd.pgrst.plan+json", + "x-api-user-id": os.getenv("API_KEY_ID"), + "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", + } + + resp = requests.get(query, headers=headers) + # We get back a list of one plan, and we want the total cost. + total_cost += resp.json()[0]["Plan"]["Total Cost"] + return math.floor(total_cost) + + +def fetch_fa_time(api_version): + total_cost = 0 + for offset in range(0, 4000000, 20000): + print(f"api {api_version} offset {offset}") + query = f"{URI}/federal_awards?limit=20000&offset={offset}" + headers = { + "accept-profile": api_version, + "x-api-user-id": os.getenv("API_KEY_ID"), + "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", + } + t0 = time.time() + resp = requests.get(query, headers=headers) + t1 = time.time() + # We get back a list of one plan, and we want the total cost. + total_cost += t1 - t0 + return math.floor(total_cost) + + +def fetch_fa_batches_exp(): + total_cost = 0 + for batch_no in range(0, 235): + query = f"{URI}/federal_awards?batch_number=eq.{batch_no}" + headers = { + "accept-profile": "public_api_v2_0_0_alpha", + "accept": "application/vnd.pgrst.plan+json", + "x-api-user-id": os.getenv("API_KEY_ID"), + "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", + } + + resp = requests.get(query, headers=headers) + # We get back a list of one plan, and we want the total cost. + total_cost += resp.json()[0]["Plan"]["Total Cost"] + return math.floor(total_cost) + + +def fetch_fa_batches_time(): + total_cost = 0 + for batch_no in range(0, 235): + print(f"batch number: {batch_no}") + query = f"{URI}/federal_awards?batch_number=eq.{batch_no}" + headers = { + "accept-profile": "public_api_v2_0_0_alpha", + "x-api-user-id": os.getenv("API_KEY_ID"), + "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", + } + t0 = time.time() + resp = requests.get(query, headers=headers) + t1 = time.time() + # We get back a list of one plan, and we want the total cost. + total_cost += t1 - t0 + return math.floor(total_cost) + + +if __name__ == "__main__": + results1 = {} + results2 = {} + results3 = {} + results4 = {} + + results1["ap110"] = fetch_fa_exp("api_v1_1_0") + results1["public200"] = fetch_fa_exp("public_api_v2_0_0_alpha") + results1["public200_batches"] = fetch_fa_batches_exp() + + min = math.inf + for k, v in results1.items(): + if v < min: + min = v + for k, v in results1.items(): + results2[k] = math.floor(v / min) + + print("Running timing tests... ~5m") + + results3["ap110"] = fetch_fa_time("api_v1_1_0") + results3["public200"] = fetch_fa_time("public_api_v2_0_0_alpha") + results3["public200_batches"] = fetch_fa_batches_time() + + min = math.inf + for k, v in results3.items(): + if v < min: + min = v + for k, v in results3.items(): + results4[k] = math.floor(v / min) + + pprint(results1) + pprint(results2) + pprint(results3) + pprint(results4) diff --git a/backend/tools/collect_scan_metrics.py b/backend/util/collect_scan_metrics.py similarity index 100% rename from backend/tools/collect_scan_metrics.py rename to backend/util/collect_scan_metrics.py diff --git a/backend/tools/fac-backup-and-sling.sh b/backend/util/fac-backup-and-sling.sh similarity index 88% rename from backend/tools/fac-backup-and-sling.sh rename to backend/util/fac-backup-and-sling.sh index 74df66c601..f4d54550f3 100755 --- a/backend/tools/fac-backup-and-sling.sh +++ b/backend/util/fac-backup-and-sling.sh @@ -30,6 +30,7 @@ GetSling() { RunSling() { ./sling run -r dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml + ./sling run -r dissemination/sql/sling/public_data_v1_0_0/tribal_data_v1_0_0.yaml } InstallAWS() { @@ -52,6 +53,8 @@ if [ "$run_option" == "initial_backup" ]; then GetUtil InstallAWS gonogo "install_aws" + GetSling + gonogo "insall_sling" RDSToS3Dump "$db_name" "$backup_s3_name" "initial/$initial_date" gonogo "db_to_s3" RDSToRDS "$db_name" "$backup_db_name" "initial" @@ -59,17 +62,19 @@ if [ "$run_option" == "initial_backup" ]; then AWSS3Sync "$s3_name" "$backup_s3_name" gonogo "s3_sync" RunSling - gonogo "sling" + gonogo "run_sling" elif [ "$run_option" == "deploy_backup" ]; then GetUtil InstallAWS gonogo "install_aws" + GetSling + gonogo "insall_sling" RDSToRDS "$db_name" "$backup_db_name" "backup" gonogo "db_to_db" AWSS3Sync "$s3_name" "$backup_s3_name" gonogo "s3_sync" RunSling - gonogo "sling" + gonogo "run_sling" elif [ "$run_option" == "scheduled_backup" ]; then GetUtil InstallAWS @@ -78,24 +83,24 @@ elif [ "$run_option" == "scheduled_backup" ]; then gonogo "db_to_s3" AWSS3Sync "$s3_name" "$backup_s3_name" gonogo "s3_sync" - RunSling - gonogo "sling" elif [ "$run_option" == "daily_backup" ]; then GetUtil InstallAWS gonogo "install_aws" + GetSling + gonogo "insall_sling" + RDSToRDS "$db_name" "$backup_db_name" "backup" + gonogo "db_to_db" RDSToS3Dump "$db_name" "$backup_s3_name" "daily/$daily_date" gonogo "db_to_s3" AWSS3Sync "$s3_name" "$backup_s3_name" gonogo "s3_sync" RunSling - gonogo "sling" + gonogo "run_sling" elif [ "$run_option" == "media_sync" ]; then GetUtil InstallAWS gonogo "install_aws" AWSS3Sync "$s3_name" "$backup_s3_name" gonogo "s3_sync" - RunSling - gonogo "sling" fi diff --git a/backend/tools/generate_xlsx_files.py b/backend/util/generate_xlsx_files.py similarity index 100% rename from backend/tools/generate_xlsx_files.py rename to backend/util/generate_xlsx_files.py diff --git a/backend/tools/load_public_dissem_data/.gitignore b/backend/util/load_public_dissem_data/.gitignore similarity index 100% rename from backend/tools/load_public_dissem_data/.gitignore rename to backend/util/load_public_dissem_data/.gitignore diff --git a/backend/tools/load_public_dissem_data/Dockerfile b/backend/util/load_public_dissem_data/Dockerfile similarity index 97% rename from backend/tools/load_public_dissem_data/Dockerfile rename to backend/util/load_public_dissem_data/Dockerfile index 473c5cf2a1..3992bac453 100644 --- a/backend/tools/load_public_dissem_data/Dockerfile +++ b/backend/util/load_public_dissem_data/Dockerfile @@ -9,6 +9,7 @@ RUN apt-get install -y \ WORKDIR /layered COPY data/sling.yaml . +COPY data/tribal_sling.yaml . COPY data/config.json . # Curl in `sling` diff --git a/backend/tools/load_public_dissem_data/Makefile b/backend/util/load_public_dissem_data/Makefile similarity index 70% rename from backend/tools/load_public_dissem_data/Makefile rename to backend/util/load_public_dissem_data/Makefile index 0d4bdba335..87570ff825 100644 --- a/backend/tools/load_public_dissem_data/Makefile +++ b/backend/util/load_public_dissem_data/Makefile @@ -2,6 +2,7 @@ NETWORK?=backend-web-1 build: cp ../../dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml data/sling.yaml + cp ../../dissemination/sql/sling/public_data_v1_0_0/tribal_data_v1_0_0.yaml data/tribal_sling.yaml docker build -t facloaddata . run: diff --git a/backend/tools/load_public_dissem_data/README.md b/backend/util/load_public_dissem_data/README.md similarity index 100% rename from backend/tools/load_public_dissem_data/README.md rename to backend/util/load_public_dissem_data/README.md diff --git a/backend/tools/load_public_dissem_data/data/README.md b/backend/util/load_public_dissem_data/data/README.md similarity index 100% rename from backend/tools/load_public_dissem_data/data/README.md rename to backend/util/load_public_dissem_data/data/README.md diff --git a/backend/util/load_public_dissem_data/data/config.json b/backend/util/load_public_dissem_data/data/config.json new file mode 100644 index 0000000000..2b1fc22521 --- /dev/null +++ b/backend/util/load_public_dissem_data/data/config.json @@ -0,0 +1,90 @@ +{ + "user-provided": [ + { + "label": "mc", + "name": "backups", + "tags": [], + "instance_guid": "UUIDALPHA1", + "instance_name": "backups", + "binding_guid": "UUIDALPHA2", + "binding_name": null, + "credentials": { + "access_key_id": "longtest", + "secret_access_key": "longtest", + "bucket": "backups", + "endpoint": "http://127.0.0.1:9001", + "admin_username": "minioadmin", + "admin_password": "minioadmin" + } + }, + { + "label": "mc", + "name": "fac-private-s3", + "tags": [], + "instance_guid": "UUIDALPHA1", + "instance_name": "fac-private-s3", + "binding_guid": "UUIDALPHA2", + "binding_name": null, + "credentials": { + "access_key_id": "longtest", + "secret_access_key": "longtest", + "bucket": "fac-private-s3", + "endpoint": "http://127.0.0.1:9001", + "admin_username": "minioadmin", + "admin_password": "minioadmin" + } + } + ], + "aws-rds": [ + { + "label": "fac-db", + "provider": null, + "plan": null, + "name": "fac-db", + "tags": [ + "database", + "docker" + ], + "instance_guid": "UUIDINDIA1", + "instance_name": "db", + "binding_guid": "UUIDINDIA2", + "binding_name": null, + "credentials": { + "db_name": "postgres", + "host": "db", + "name": "postgres", + "password": "", + "port": "5432", + "uri": "postgres://postgres@db:5432/postgres?sslmode=disable", + "username": "postgres" + }, + "syslog_drain_url": null, + "volume_mounts": [] + }, + { + "label": "fac-snapshot-db", + "provider": null, + "plan": null, + "name": "fac-snapshot-db", + "tags": [ + "database", + "docker" + ], + "instance_guid": "UUIDJULIET1", + "instance_name": "db", + "binding_guid": "UUIDJULIET2", + "binding_name": null, + "credentials": { + "db_name": "postgres", + "host": "db2", + "name": "postgres", + "password": "", + "port": "5432", + "uri": "postgres://postgres@db2:5432/postgres?sslmode=disable", + "username": "postgres" + }, + "syslog_drain_url": null, + "volume_mounts": [] + } + ] +} diff --git a/backend/util/load_public_dissem_data/data/tribal_sling.yaml b/backend/util/load_public_dissem_data/data/tribal_sling.yaml new file mode 100644 index 0000000000..943d2b7cb0 --- /dev/null +++ b/backend/util/load_public_dissem_data/data/tribal_sling.yaml @@ -0,0 +1,116 @@ +# This uses an env var called PG to +# set the connection string. +source: FAC_SNAPSHOT_URI +target: FAC_SNAPSHOT_URI +env: + SLING_ALLOW_EMPTY: true + +# Handy +# SELECT column_name +# FROM information_schema.columns +# WHERE table_schema = 'public' +# AND table_name = 'dissemination_general' +# AND column_name NOT IN ('id'); + +streams: + # Corrective action plans are NOT always public. + public.dissemination_corrective_action_plan: + mode: incremental + update_key: cap.id + object: public_data_v1_0_0.tribal_corrective_action_plans + sql: | + SELECT + cap.id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + cap.contains_chart_or_table, + cap.finding_ref_number, + cap.planned_action + FROM + public_data_v1_0_0.general gen, + public.dissemination_captext cap + WHERE + cap.report_id = gen.report_id + AND + -- Only include the public corrective action plans. + gen.is_public = false + AND + {incremental_where_cond} + ORDER BY cap.id + + # Findings text is NOT always public. + public.dissemination_findingstext: + mode: incremental + update_key: ft.id + object: public_data_v1_0_0.tribal_findings_text + sql: | + SELECT + ft.id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + ft.finding_ref_number, + ft.contains_chart_or_table, + ft.finding_text + FROM + public_data_v1_0_0.general gen, + public.dissemination_findingtext ft + WHERE + ft.report_id = gen.report_id + AND + gen.is_public = false + AND + {incremental_where_cond} + ORDER BY ft.id + + # The notes to SEFA are NOT all public. + public.dissemination_notes: + mode: incremental + update_key: note.id + object: public_data_v1_0_0.tribal_notes_to_sefa + sql: | + SELECT + note.id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + note.accounting_policies, + note.contains_chart_or_table, + note.content, + note.is_minimis_rate_used, + note.note_title as title, + note.rate_explained + FROM + public_data_v1_0_0.general gen, + public.dissemination_note note + WHERE + note.report_id = gen.report_id + AND + -- Some notes are not public. + gen.is_public = false + AND + {incremental_where_cond} + ORDER BY note.id + + # Should only update the tables above... + public_data_v1_0_0.metadata: + mode: incremental + object: public_data_v1_0_0.metadata_v2_0_0_a + sql: | + SELECT 'tribal_corrective_action_plans' + AS table, COUNT(*), public_api_v2_0_0_alpha_functions.batches('tribal_corrective_action_plans') + FROM public_data_v1_0_0.tribal_corrective_action_plans + UNION + SELECT 'tribal_findings_text' + AS table, COUNT(*), public_api_v2_0_0_alpha_functions.batches('tribal_findings_text') + FROM public_data_v1_0_0.tribal_findings_text + UNION + SELECT 'tribal_notes_to_sefa' + AS table, COUNT(*), public_api_v2_0_0_alpha_functions.batches('tribal_notes_to_sefa') + FROM public_data_v1_0_0.tribal_notes_to_sefa diff --git a/backend/tools/load_public_dissem_data/load_public_data_locally.sh b/backend/util/load_public_dissem_data/load_public_data_locally.sh similarity index 98% rename from backend/tools/load_public_dissem_data/load_public_data_locally.sh rename to backend/util/load_public_dissem_data/load_public_data_locally.sh index 881988a720..e4b6860d65 100755 --- a/backend/tools/load_public_dissem_data/load_public_data_locally.sh +++ b/backend/util/load_public_dissem_data/load_public_data_locally.sh @@ -88,3 +88,4 @@ fi # data from fac-snapshot-db (in the dissem_* tables) to a set of # tables that the API will point at. sling run -r /layered/sling.yaml +sling run -r /layered/tribal_sling.yaml diff --git a/backend/tools/test_update_program_data.py b/backend/util/test_update_program_data.py similarity index 100% rename from backend/tools/test_update_program_data.py rename to backend/util/test_update_program_data.py diff --git a/backend/tools/update_program_data.py b/backend/util/update_program_data.py similarity index 100% rename from backend/tools/update_program_data.py rename to backend/util/update_program_data.py From 65f73659fa5fa4ca3d265e91be191cae376ba460 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Fri, 4 Oct 2024 14:19:57 -0400 Subject: [PATCH 14/89] Updates for live test. --- backend/tools/api_standup.sh | 2 +- backend/tools/api_teardown.sh | 3 +- backend/tools/sling_first_run.sh | 2 + backend/tools/variables.sh | 2 +- backend/util/api_perf_test/perf.py | 4 + backend/util/load_public_dissem_data/Makefile | 2 - .../data/tribal_sling.yaml | 116 ------------------ .../load_public_data_locally.sh | 62 +++++----- 8 files changed, 42 insertions(+), 151 deletions(-) delete mode 100644 backend/util/load_public_dissem_data/data/tribal_sling.yaml diff --git a/backend/tools/api_standup.sh b/backend/tools/api_standup.sh index f8fbfa669a..502edc110c 100644 --- a/backend/tools/api_standup.sh +++ b/backend/tools/api_standup.sh @@ -19,7 +19,7 @@ function api_standup { echo "check_table_exists $required_table $result" if [ $result -eq 0 ]; then run_sql $FAC_SNAPSHOT_URI $base_path $location $api_version 'standup.sql' - gonogo "$api_version teardown.sql" + gonogo "$api_version standup.sql" else echo "API TABLE NOT FOUND/SKIPPED $required_table not found for $api_version" fi diff --git a/backend/tools/api_teardown.sh b/backend/tools/api_teardown.sh index 3fc3addfb4..086140a1e5 100644 --- a/backend/tools/api_teardown.sh +++ b/backend/tools/api_teardown.sh @@ -13,8 +13,7 @@ function api_teardown { for index in "${!api_versions[@]}" do - local api_version="${api_versions[index]}" - local required_table="${api_required_tables[index]}" + local api_version="${api_versions[index]}" if [ -f ${base_path}/${location}/${api_version}/${sql_file} ]; then run_sql $FAC_SNAPSHOT_URI $base_path $location $api_version $sql_file gonogo "$api_version teardown.sql" diff --git a/backend/tools/sling_first_run.sh b/backend/tools/sling_first_run.sh index 0049391ed2..cd5bdede72 100644 --- a/backend/tools/sling_first_run.sh +++ b/backend/tools/sling_first_run.sh @@ -17,6 +17,8 @@ function sling_first_run() { # If the metadata table exists, it means sling has run to completion. check_table_exists $FAC_SNAPSHOT_URI 'public_data_v1_0_0.metadata' local is_metadata_table=$? + echo "check_table_exists public_data_v1_0_0.metadata $is_metadata_table" + # We need to load some functions for sling to complete, because # we use those functions as part of the metadata table generation. local base_path='dissemination/sql' diff --git a/backend/tools/variables.sh b/backend/tools/variables.sh index 43d7d0fa74..ae36f60476 100644 --- a/backend/tools/variables.sh +++ b/backend/tools/variables.sh @@ -9,7 +9,7 @@ declare -a api_versions=( "admin_api_v1_1_1" "public_api_v2_0_0_alpha" "public_data_v1_0_0" - ) + ) # These are the tables that must be present # in order to stand up that API. declare -a api_required_tables=( diff --git a/backend/util/api_perf_test/perf.py b/backend/util/api_perf_test/perf.py index a349101706..76815c5992 100644 --- a/backend/util/api_perf_test/perf.py +++ b/backend/util/api_perf_test/perf.py @@ -113,7 +113,11 @@ def fetch_fa_batches_time(): for k, v in results3.items(): results4[k] = math.floor(v / min) + # results1 is the raw EXPLAIN cost of downloading all of federal_awards pprint(results1) + # results2 is the ratio pprint(results2) + # results3 is the raw timing pprint(results3) + # results4 is the ratio for timings pprint(results4) diff --git a/backend/util/load_public_dissem_data/Makefile b/backend/util/load_public_dissem_data/Makefile index 87570ff825..079d45b022 100644 --- a/backend/util/load_public_dissem_data/Makefile +++ b/backend/util/load_public_dissem_data/Makefile @@ -1,8 +1,6 @@ NETWORK?=backend-web-1 build: - cp ../../dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml data/sling.yaml - cp ../../dissemination/sql/sling/public_data_v1_0_0/tribal_data_v1_0_0.yaml data/tribal_sling.yaml docker build -t facloaddata . run: diff --git a/backend/util/load_public_dissem_data/data/tribal_sling.yaml b/backend/util/load_public_dissem_data/data/tribal_sling.yaml deleted file mode 100644 index 943d2b7cb0..0000000000 --- a/backend/util/load_public_dissem_data/data/tribal_sling.yaml +++ /dev/null @@ -1,116 +0,0 @@ -# This uses an env var called PG to -# set the connection string. -source: FAC_SNAPSHOT_URI -target: FAC_SNAPSHOT_URI -env: - SLING_ALLOW_EMPTY: true - -# Handy -# SELECT column_name -# FROM information_schema.columns -# WHERE table_schema = 'public' -# AND table_name = 'dissemination_general' -# AND column_name NOT IN ('id'); - -streams: - # Corrective action plans are NOT always public. - public.dissemination_corrective_action_plan: - mode: incremental - update_key: cap.id - object: public_data_v1_0_0.tribal_corrective_action_plans - sql: | - SELECT - cap.id, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - cap.contains_chart_or_table, - cap.finding_ref_number, - cap.planned_action - FROM - public_data_v1_0_0.general gen, - public.dissemination_captext cap - WHERE - cap.report_id = gen.report_id - AND - -- Only include the public corrective action plans. - gen.is_public = false - AND - {incremental_where_cond} - ORDER BY cap.id - - # Findings text is NOT always public. - public.dissemination_findingstext: - mode: incremental - update_key: ft.id - object: public_data_v1_0_0.tribal_findings_text - sql: | - SELECT - ft.id, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - ft.finding_ref_number, - ft.contains_chart_or_table, - ft.finding_text - FROM - public_data_v1_0_0.general gen, - public.dissemination_findingtext ft - WHERE - ft.report_id = gen.report_id - AND - gen.is_public = false - AND - {incremental_where_cond} - ORDER BY ft.id - - # The notes to SEFA are NOT all public. - public.dissemination_notes: - mode: incremental - update_key: note.id - object: public_data_v1_0_0.tribal_notes_to_sefa - sql: | - SELECT - note.id, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - note.accounting_policies, - note.contains_chart_or_table, - note.content, - note.is_minimis_rate_used, - note.note_title as title, - note.rate_explained - FROM - public_data_v1_0_0.general gen, - public.dissemination_note note - WHERE - note.report_id = gen.report_id - AND - -- Some notes are not public. - gen.is_public = false - AND - {incremental_where_cond} - ORDER BY note.id - - # Should only update the tables above... - public_data_v1_0_0.metadata: - mode: incremental - object: public_data_v1_0_0.metadata_v2_0_0_a - sql: | - SELECT 'tribal_corrective_action_plans' - AS table, COUNT(*), public_api_v2_0_0_alpha_functions.batches('tribal_corrective_action_plans') - FROM public_data_v1_0_0.tribal_corrective_action_plans - UNION - SELECT 'tribal_findings_text' - AS table, COUNT(*), public_api_v2_0_0_alpha_functions.batches('tribal_findings_text') - FROM public_data_v1_0_0.tribal_findings_text - UNION - SELECT 'tribal_notes_to_sefa' - AS table, COUNT(*), public_api_v2_0_0_alpha_functions.batches('tribal_notes_to_sefa') - FROM public_data_v1_0_0.tribal_notes_to_sefa diff --git a/backend/util/load_public_dissem_data/load_public_data_locally.sh b/backend/util/load_public_dissem_data/load_public_data_locally.sh index e4b6860d65..6c6bcfafb7 100755 --- a/backend/util/load_public_dissem_data/load_public_data_locally.sh +++ b/backend/util/load_public_dissem_data/load_public_data_locally.sh @@ -58,34 +58,38 @@ else echo "Loaded lots of data without error, apparently." fi -# Download cgov-util. -# This will let us do a backup from the main database (fac-db) -# to the secondary database (fac-snapshot-db). This mirrors what -# happens in production. -pushd /app/data +# We'll stop here. Now, on first boot, the FAC should see if things are +# in place in DB2. If not, the app startup process will take +# care of creating new tables, etc. -# Run the backup of the dissemination_ tables from -# fac-db to fac-snapshot-db. -check_table_exists $FAC_SNAPSHOT_URI "public.dissemination_general" -result=$? -# cgov-util wants to know the environment it is in. -export ENV="LOCAL" -if [ $result -ne 0 ]; then - # First run if it does not exist. - /layered/cgov-util db_to_db \ - --src_db fac-db \ - --dest_db fac-snapshot-db \ - --operation initial -else - /layered/cgov-util db_to_db \ - --src_db fac-db \ - --dest_db fac-snapshot-db \ - --operation backup -fi +# # Download cgov-util. +# # This will let us do a backup from the main database (fac-db) +# # to the secondary database (fac-snapshot-db). This mirrors what +# # happens in production. +# pushd /app/data + +# # Run the backup of the dissemination_ tables from +# # fac-db to fac-snapshot-db. +# check_table_exists $FAC_SNAPSHOT_URI "public.dissemination_general" +# result=$? +# # cgov-util wants to know the environment it is in. +# export ENV="LOCAL" +# if [ $result -ne 0 ]; then +# # First run if it does not exist. +# /layered/cgov-util db_to_db \ +# --src_db fac-db \ +# --dest_db fac-snapshot-db \ +# --operation initial +# else +# /layered/cgov-util db_to_db \ +# --src_db fac-db \ +# --dest_db fac-snapshot-db \ +# --operation backup +# fi -# Now, we're going to run sling. -# This will create the API tables. It essentially does a copy of -# data from fac-snapshot-db (in the dissem_* tables) to a set of -# tables that the API will point at. -sling run -r /layered/sling.yaml -sling run -r /layered/tribal_sling.yaml +# # Now, we're going to run sling. +# # This will create the API tables. It essentially does a copy of +# # data from fac-snapshot-db (in the dissem_* tables) to a set of +# # tables that the API will point at. +# sling run -r /layered/sling.yaml +# sling run -r /layered/tribal_sling.yaml From 1be0b83ad2a3abf15a2237c5a97585c0f489ad87 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Sat, 5 Oct 2024 12:26:20 -0400 Subject: [PATCH 15/89] Improving/simplifying It is cleaner than it was, and may make it easier to debug the `preview` deploy. --- .../commands/generate_dump_files.py | 112 +++++++ .../sql/api/api_v1_0_3/standup.sql | 78 ++--- .../sql/api/api_v1_1_0/standup.sql | 101 ++---- .../sql/api/finalize/standup.sql | 3 + .../sql/api/finalize/teardown.sql | 3 + .../sql/api/permissions/standup.sql | 5 + .../sql/api/permissions/teardown.sql | 15 + .../sql/api/prep_snapshot_db/standup.sql | 6 + .../sql/api/prep_snapshot_db/teardown.sql | 0 .../public.rest | 18 +- .../sql/api/public_api_v1_0_0/standup.sql | 295 ++++++++++++++++++ .../sql/api/public_api_v1_0_0/teardown.sql | 12 + .../api/public_api_v2_0_0_alpha/standup.sql | 273 ---------------- .../api/public_api_v2_0_0_alpha/teardown.sql | 12 - .../sql/api/public_data_v1_0_0/standup.sql | 6 + .../sql/api/public_data_v1_0_0/teardown.sql | 30 -- .../dissemination/sql/api/roles/standup.sql | 7 + .../dissemination/sql/api/roles/teardown.sql | 36 +++ .../dissemination/sql/api/schemas/standup.sql | 0 .../sql/api/schemas/teardown.sql | 3 + .../sql/api/sequences/standup.sql | 1 + .../sql/api/sequences/teardown.sql | 25 ++ .../public_metadata_v1_0_0.yaml | 26 +- backend/docker-compose-db-only.yml | 2 +- backend/docker-compose.yml | 2 +- backend/tools/api_standup.sh | 30 +- backend/tools/api_teardown.sh | 25 +- backend/tools/setup_cgov_env.sh | 2 +- backend/tools/setup_local_env.sh | 2 +- backend/tools/sling_first_run.sh | 4 + backend/tools/util_startup.sh | 12 +- backend/tools/variables.sh | 73 +++-- backend/util/api_perf_test/perf.py | 10 +- .../util/load_public_dissem_data/Dockerfile | 8 +- .../load_public_data_locally.sh | 36 +-- terraform/shared/modules/env/postgrest.tf | 4 +- 36 files changed, 709 insertions(+), 568 deletions(-) create mode 100644 backend/dissemination/management/commands/generate_dump_files.py create mode 100644 backend/dissemination/sql/api/finalize/standup.sql create mode 100644 backend/dissemination/sql/api/finalize/teardown.sql create mode 100644 backend/dissemination/sql/api/permissions/standup.sql create mode 100644 backend/dissemination/sql/api/permissions/teardown.sql create mode 100644 backend/dissemination/sql/api/prep_snapshot_db/standup.sql create mode 100644 backend/dissemination/sql/api/prep_snapshot_db/teardown.sql rename backend/dissemination/sql/api/{public_api_v2_0_0_alpha => public_api_v1_0_0}/public.rest (74%) create mode 100644 backend/dissemination/sql/api/public_api_v1_0_0/standup.sql create mode 100644 backend/dissemination/sql/api/public_api_v1_0_0/teardown.sql delete mode 100644 backend/dissemination/sql/api/public_api_v2_0_0_alpha/standup.sql delete mode 100644 backend/dissemination/sql/api/public_api_v2_0_0_alpha/teardown.sql create mode 100644 backend/dissemination/sql/api/roles/standup.sql create mode 100644 backend/dissemination/sql/api/roles/teardown.sql create mode 100644 backend/dissemination/sql/api/schemas/standup.sql create mode 100644 backend/dissemination/sql/api/schemas/teardown.sql create mode 100644 backend/dissemination/sql/api/sequences/standup.sql create mode 100644 backend/dissemination/sql/api/sequences/teardown.sql diff --git a/backend/dissemination/management/commands/generate_dump_files.py b/backend/dissemination/management/commands/generate_dump_files.py new file mode 100644 index 0000000000..78a413ed81 --- /dev/null +++ b/backend/dissemination/management/commands/generate_dump_files.py @@ -0,0 +1,112 @@ +import json +from collections import namedtuple as NT + +from django.core.management.base import BaseCommand +from dissemination.models import ( + General, + FederalAward, + Finding, + FindingText, + AdditionalUei, + AdditionalEin, + CapText, + Note, + Passthrough, + SecondaryAuditor, +) + +from django.forms import model_to_dict +from django.core.serializers.json import DjangoJSONEncoder +from django.db.models import Model + +Table = NT("Table", "model,name,is_public_table") +# Exclude General here. +TABLES_TO_DUMP = [ + # Public tables + Table(FederalAward, "federal_awards", True), + Table(Finding, "findings", True), + Table(Passthrough, "passthroughs", True), + Table(AdditionalUei, "additional_ueis", True), + Table(AdditionalEin, "additional_eins", True), + Table(SecondaryAuditor, "secondary_auditors", True), + # Suppressed tables + Table(FindingText, "findings_text", False), + Table(CapText, "corrective_action_plan_text", False), + Table(Note, "notes_to_sefa", False), +] + + +# https://stackoverflow.com/questions/757022/how-do-you-serialize-a-model-instance-in-django +class ExtendedEncoder(DjangoJSONEncoder): + def default(self, o): + if isinstance(o, Model): + d = model_to_dict(o) + if "id" in d: + del d["id"] + return d + return super().default(o) + + +def dump_general(audit_year): + public_report_ids = [] + private_report_ids = [] + objs = General.objects.filter(audit_year=audit_year) + with open(f"{audit_year}-general.json", "w") as fp: + fp.write("[") + first = True + for o in objs: + if o.is_public: + public_report_ids.append(o.report_id) + else: + private_report_ids.append(o.report_id) + if first: + fp.write("\n") + first = False + else: + fp.write(",\n") + fp.write("\t") + fp.write(json.dumps(o, cls=ExtendedEncoder)) + fp.write("\n]\n") + fp.close() + return (public_report_ids, private_report_ids) + + +def dump_table(table, audit_year, report_ids): + with open(f"{audit_year}-{table.name}.json", "w") as fp: + fp.write("[") + first = True + for rid in report_ids: + objs = table.model.objects.filter(report_id=rid) + for o in objs: + if first: + fp.write("\n") + first = False + else: + fp.write(",\n") + fp.write("\t") + fp.write(json.dumps(o, cls=ExtendedEncoder)) + fp.write("\n]\n") + fp.close() + + +class Command(BaseCommand): + help = """ + Runs sql scripts to recreate access tables for the postgrest API. + """ + + def add_arguments(self, parser): + parser.add_argument( + "-y", "--year", choices=[f"20{x}" for x in range(16, 24)], default=False + ) + + def handle(self, *args, **options): + audit_year = options["year"] + (public_report_ids, private_report_ids) = dump_general(audit_year) + for table in TABLES_TO_DUMP: + # If it is a public table, dump everything. + # If it is not a public table, then we only dump + # the report IDs that were marked is_public=True + if table.is_public_table: + dump_table(table, audit_year, public_report_ids + private_report_ids) + else: + dump_table(table, audit_year, public_report_ids) diff --git a/backend/dissemination/sql/api/api_v1_0_3/standup.sql b/backend/dissemination/sql/api/api_v1_0_3/standup.sql index aa91f37887..4a7f33c993 100644 --- a/backend/dissemination/sql/api/api_v1_0_3/standup.sql +++ b/backend/dissemination/sql/api/api_v1_0_3/standup.sql @@ -1,41 +1,33 @@ ------------------------------------------------------------------ --- authenticator role +-- GATE ------------------------------------------------------------------ -DO -$do$ -BEGIN - IF EXISTS ( - SELECT FROM pg_catalog.pg_roles - WHERE rolname = 'authenticator') THEN - RAISE NOTICE 'Role "authenticator" already exists. Skipping.'; - ELSE - CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; - END IF; -END -$do$; - ------------------------------------------------------------------- --- api_fac_gov role ------------------------------------------------------------------- -DO -$do$ -BEGIN - IF EXISTS ( - SELECT FROM pg_catalog.pg_roles - WHERE rolname = 'api_fac_gov') THEN - RAISE NOTICE 'Role "api_fac_gov" already exists. Skipping.'; - ELSE - CREATE ROLE api_fac_gov NOLOGIN; - END IF; -END -$do$; - -GRANT api_fac_gov TO authenticator; - -NOTIFY pgrst, 'reload schema'; - +-- We only want the API to run if certain conditions are met. +-- We could try and encode that in the `bash` portion of the code. +-- Or, we could just gate things at the top of our SQL. +-- If the conditions are not met, we should exit noisily. +-- A cast to regclass will fail with an exception if the table +-- does not exist. +DO LANGUAGE plpgsql +$GATE$ + DECLARE + the_schema varchar := 'public'; + the_table varchar := 'dissemination_general'; + api_ver varchar := 'API_v1_0_3'; + BEGIN + IF EXISTS ( + SELECT FROM pg_tables + WHERE schemaname = the_schema + AND tablename = the_table + ) + THEN + RAISE info '% Gate condition met. Continuing.', api_ver; + ELSE + RAISE exception '% %.% not found.', api_ver, the_schema, the_table; + END IF; + END +$GATE$; -BEGIN; +SELECT 'public.dissemination_general'::regclass; DO $APIV103$ @@ -43,7 +35,9 @@ $APIV103$ DROP SCHEMA IF EXISTS api_v1_0_3 CASCADE; DROP SCHEMA IF EXISTS api_v1_0_3_functions CASCADE; - IF NOT EXISTS (select schema_name from information_schema.schemata where schema_name = 'api_v1_0_3') then + IF NOT EXISTS (select schema_name + from information_schema.schemata + where schema_name = 'api_v1_0_3') then CREATE SCHEMA api_v1_0_3; CREATE SCHEMA api_v1_0_3_functions; @@ -70,9 +64,6 @@ $APIV103$ $APIV103$ ; -COMMIT; -NOTIFY pgrst, 'reload schema'; - ------------------------------------------------------------------ -- functions ------------------------------------------------------------------ @@ -83,12 +74,9 @@ BEGIN END; $has_tribal_data_access$ LANGUAGE plpgsql; -NOTIFY pgrst, 'reload schema'; - ------------------------------------------------------------------ -- VIEWs ------------------------------------------------------------------ -BEGIN; --------------------------------------- -- finding_text --------------------------------------- @@ -416,9 +404,3 @@ create view api_v1_0_3.additional_eins as (gen.is_public = false and api_v1_0_3_functions.has_tribal_data_access())) order by ein.id ; - -commit; - -notify pgrst, - 'reload schema'; - diff --git a/backend/dissemination/sql/api/api_v1_1_0/standup.sql b/backend/dissemination/sql/api/api_v1_1_0/standup.sql index 053d9288f1..2df37faf7e 100644 --- a/backend/dissemination/sql/api/api_v1_1_0/standup.sql +++ b/backend/dissemination/sql/api/api_v1_1_0/standup.sql @@ -1,33 +1,31 @@ -DO -$do$ -BEGIN - IF EXISTS ( - SELECT FROM pg_catalog.pg_roles - WHERE rolname = 'authenticator') THEN - RAISE NOTICE 'Role "authenticator" already exists. Skipping.'; - ELSE - CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; - END IF; -END -$do$; - -DO -$do$ -BEGIN - IF EXISTS ( - SELECT FROM pg_catalog.pg_roles - WHERE rolname = 'api_fac_gov') THEN - RAISE NOTICE 'Role "api_fac_gov" already exists. Skipping.'; - ELSE - CREATE ROLE api_fac_gov NOLOGIN; - END IF; -END -$do$; - -GRANT api_fac_gov TO authenticator; - -NOTIFY pgrst, 'reload schema'; -begin; +------------------------------------------------------------------ +-- GATE +------------------------------------------------------------------ +-- We only want the API to run if certain conditions are met. +-- We could try and encode that in the `bash` portion of the code. +-- Or, we could just gate things at the top of our SQL. +-- If the conditions are not met, we should exit noisily. +-- A cast to regclass will fail with an exception if the table +-- does not exist. +DO LANGUAGE plpgsql +$GATE$ + DECLARE + the_schema varchar := 'public'; + the_table varchar := 'dissemination_general'; + api_ver varchar := 'API_v1_1_0'; + BEGIN + IF EXISTS ( + SELECT FROM pg_tables + WHERE schemaname = the_schema + AND tablename = the_table + ) + THEN + RAISE info '% Gate condition met. Continuing.', api_ver; + ELSE + RAISE exception '% %.% not found.', api_ver, the_schema, the_table; + END IF; + END +$GATE$; do $$ @@ -74,41 +72,10 @@ end $$ ; --- https://postgrest.org/en/stable/references/api/openapi.html --- This is the title (version number) and description (text). -COMMENT ON SCHEMA api_v1_1_0 IS -$$v1.1.0 - -A RESTful API that serves data from the SF-SAC.$$; - - -commit; - -notify pgrst, - 'reload schema'; - --- WARNING --- Under PostgreSQL 12, the functions below work. --- Under PostgreSQL 14, these will break. --- --- Note the differences: --- --- raise info 'Works under PostgreSQL 12'; --- raise info 'request.header.x-magic %', (SELECT current_setting('request.header.x-magic', true)); --- raise info 'request.jwt.claim.expires %', (SELECT current_setting('request.jwt.claim.expires', true)); --- raise info 'Works under PostgreSQL 14'; --- raise info 'request.headers::json->>x-magic %', (SELECT current_setting('request.headers', true)::json->>'x-magic'); --- raise info 'request.jwt.claims::json->expires %', (SELECT current_setting('request.jwt.claims', true)::json->>'expires'); --- --- To quote the work of Dav Pilkey, "remember this now." - ----------------------------------------------------- -- FUNCTIONS ----------------------------------------------------- - -notify pgrst, 'reload schema'; - CREATE OR REPLACE FUNCTION api_v1_1_0_functions.get_header(item text) RETURNS text AS $get_header$ declare res text; @@ -217,10 +184,6 @@ create index IF NOT EXISTS batch_by_id_dfa on public.dissemination_federalaward using btree(api_v1_1_0_functions.batch(public.dissemination_federalaward.id)); - -NOTIFY pgrst, 'reload schema'; -begin; - --------------------------------------- -- finding_text --------------------------------------- @@ -519,11 +482,3 @@ create view api_v1_1_0.additional_eins as gen.report_id = ein.report_id order by ein.id ; - -commit; - - - -notify pgrst, - 'reload schema'; - diff --git a/backend/dissemination/sql/api/finalize/standup.sql b/backend/dissemination/sql/api/finalize/standup.sql new file mode 100644 index 0000000000..d1168e7a39 --- /dev/null +++ b/backend/dissemination/sql/api/finalize/standup.sql @@ -0,0 +1,3 @@ +-- PostgREST likes to know when the schemas and things +-- attached to them change. +NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/api/finalize/teardown.sql b/backend/dissemination/sql/api/finalize/teardown.sql new file mode 100644 index 0000000000..d1168e7a39 --- /dev/null +++ b/backend/dissemination/sql/api/finalize/teardown.sql @@ -0,0 +1,3 @@ +-- PostgREST likes to know when the schemas and things +-- attached to them change. +NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/api/permissions/standup.sql b/backend/dissemination/sql/api/permissions/standup.sql new file mode 100644 index 0000000000..4b2efa9e56 --- /dev/null +++ b/backend/dissemination/sql/api/permissions/standup.sql @@ -0,0 +1,5 @@ +-- This grants access to the tables and views that were created +-- to the API server. +GRANT SELECT ON ALL TABLES IN SCHEMA api_v1_0_3 TO api_fac_gov; +GRANT SELECT ON ALL TABLES IN SCHEMA api_v1_1_0 TO api_fac_gov; +GRANT SELECT ON ALL TABLES IN SCHEMA public_api_v1_0_0 TO api_fac_gov; diff --git a/backend/dissemination/sql/api/permissions/teardown.sql b/backend/dissemination/sql/api/permissions/teardown.sql new file mode 100644 index 0000000000..c3bd1249a8 --- /dev/null +++ b/backend/dissemination/sql/api/permissions/teardown.sql @@ -0,0 +1,15 @@ + + +-- TABLES: SELECT | INSERT | UPDATE | DELETE | TRUNCATE | REFERENCES | TRIGGER | MAINTAIN +ALTER DEFAULT PRIVILEGES + IN SCHEMA public_data_v1_0_0 + GRANT SELECT + ON TABLES + TO api_fac_gov; + +-- SEQ: USAGE | SELECT | UPDATE +ALTER DEFAULT PRIVILEGES + IN SCHEMA public_data_v1_0_0 + GRANT SELECT, USAGE + ON SEQUENCES + TO api_fac_gov; diff --git a/backend/dissemination/sql/api/prep_snapshot_db/standup.sql b/backend/dissemination/sql/api/prep_snapshot_db/standup.sql new file mode 100644 index 0000000000..26f6fb448e --- /dev/null +++ b/backend/dissemination/sql/api/prep_snapshot_db/standup.sql @@ -0,0 +1,6 @@ +DO LANGUAGE plpgsql +$NOTE$ + BEGIN + RAISE info 'PREP_SNAPSHOT_DB standup no-op.'; + END; +$NOTE$; diff --git a/backend/dissemination/sql/api/prep_snapshot_db/teardown.sql b/backend/dissemination/sql/api/prep_snapshot_db/teardown.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/public.rest b/backend/dissemination/sql/api/public_api_v1_0_0/public.rest similarity index 74% rename from backend/dissemination/sql/api/public_api_v2_0_0_alpha/public.rest rename to backend/dissemination/sql/api/public_api_v1_0_0/public.rest index a673d2e087..adf0246f88 100644 --- a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/public.rest +++ b/backend/dissemination/sql/api/public_api_v1_0_0/public.rest @@ -16,14 +16,14 @@ Accept: application/vnd.pgrst.plan GET {{scheme}}://{{apiUrl}}/general?report_id=eq.2021-12-CENSUS-0000250449 authorization: {{authorization}} x-api-user-id: {{xApiUserId}} -accept-profile: public_api_v2_0_0_alpha +accept-profile: public_api_v1_0_0 Accept: application/vnd.pgrst.plan ### GET {{scheme}}://{{apiUrl}}/federal_awards?batch_number=eq.200 authorization: {{authorization}} x-api-user-id: {{xApiUserId}} -accept-profile: public_api_v2_0_0_alpha +accept-profile: public_api_v1_0_0 Accept: application/vnd.pgrst.plan @@ -31,17 +31,23 @@ Accept: application/vnd.pgrst.plan GET {{scheme}}://{{apiUrl}}/federal_awards?batch_number=eq.200&limit=1 authorization: {{authorization}} x-api-user-id: {{xApiUserId}} -accept-profile: public_api_v2_0_0_alpha +accept-profile: public_api_v1_0_0 ### GET {{scheme}}://{{apiUrl}}/combined?report_id=eq.2021-12-CENSUS-0000250449 authorization: {{authorization}} x-api-user-id: {{xApiUserId}} -accept-profile: public_api_v2_0_0_alpha +accept-profile: public_api_v1_0_0 Accept: application/vnd.pgrst.plan ### -GET {{scheme}}://{{apiUrl}}/rpc/compute_batch?row_id=41000 +GET {{scheme}}://{{apiUrl}}/rpc/get_batch_federal_awards?_batch=3&limit=10 authorization: {{authorization}} x-api-user-id: {{xApiUserId}} -accept-profile: public_api_v2_0_0_alpha +accept-profile: public_api_v1_0_0 + +### +GET {{scheme}}://{{apiUrl}}/metadata +authorization: {{authorization}} +x-api-user-id: {{xApiUserId}} +accept-profile: public_api_v1_0_0 diff --git a/backend/dissemination/sql/api/public_api_v1_0_0/standup.sql b/backend/dissemination/sql/api/public_api_v1_0_0/standup.sql new file mode 100644 index 0000000000..454e1c0bf8 --- /dev/null +++ b/backend/dissemination/sql/api/public_api_v1_0_0/standup.sql @@ -0,0 +1,295 @@ +------------------------------------------------------------------ +-- GATE +------------------------------------------------------------------ +-- We only want the API to run if certain conditions are met. +-- We could try and encode that in the `bash` portion of the code. +-- Or, we could just gate things at the top of our SQL. +-- If the conditions are not met, we should exit noisily. +-- A cast to regclass will fail with an exception if the table +-- does not exist. +DO LANGUAGE plpgsql +$GATE$ + DECLARE + the_schema varchar := 'public_data_v1_0_0'; + the_table varchar := 'metadata'; + api_ver varchar := 'PUBLIC_V1_0_0'; + BEGIN + IF EXISTS ( + SELECT FROM pg_tables + WHERE schemaname = the_schema + AND tablename = the_table + ) + THEN + RAISE info '% Gate condition met. Continuing.', api_ver; + ELSE + RAISE exception '% %.% not found.', api_ver, the_schema, the_table; + END IF; + END +$GATE$; +----------------------------------------------------- +-- PERMISSIONS +----------------------------------------------------- +do +$$ +BEGIN + DROP SCHEMA IF EXISTS public_api_v1_0_0 CASCADE; + DROP SCHEMA IF EXISTS public_api_v1_0_0_functions CASCADE; + CREATE SCHEMA IF NOT EXISTS public_api_v1_0_0; + CREATE SCHEMA IF NOT EXISTS public_api_v1_0_0_functions; + -- Functions are loaded before sling comes up. + + GRANT USAGE ON SCHEMA public_api_v1_0_0_functions TO api_fac_gov; + GRANT USAGE ON SCHEMA public_api_v1_0_0 TO api_fac_gov; + GRANT USAGE ON SCHEMA public_data_v1_0_0 TO api_fac_gov; + +END +$$ +; + +--- +-- FUNCTIONS +-- + +CREATE OR REPLACE FUNCTION public_api_v1_0_0_functions.rows_per_batch() + RETURNS integer + LANGUAGE sql IMMUTABLE PARALLEL SAFE AS + 'SELECT 20000'; + +CREATE OR REPLACE FUNCTION public_api_v1_0_0_functions.batch (id bigint) + RETURNS bigint + AS $batch$ + DECLARE result bigint; + DECLARE RPB integer; + BEGIN + SELECT public_api_v1_0_0_functions.rows_per_batch() INTO RPB; + SELECT div(id, RPB) INTO result; + RETURN result; + END + $batch$ + LANGUAGE plpgsql IMMUTABLE; + +CREATE OR REPLACE FUNCTION public_api_v1_0_0_functions.batches (_table text) +returns integer +as $batches$ +declare + count integer; + RPB integer; +begin + select public_api_v1_0_0_functions.rows_per_batch() into RPB; + case + when _table = 'additional_eins' then + select div(count(*), RPB) into count + from public_data_v1_0_0.additional_eins; + when _table = 'additional_ueis' then + select div(count(*), RPB) into count + from public_data_v1_0_0.additional_ueis; + when _table = 'combined' then + select div(count(*), RPB) into count + from public_data_v1_0_0.combined; + when _table = 'corrective_action_plans' then + select div(count(*), RPB) into count + from public_data_v1_0_0.corrective_action_plans; + when _table = 'federal_awards' then + select div(count(*), RPB) into count + from public_data_v1_0_0.federal_awards; + when _table = 'findings_text' then + select div(count(*), RPB) into count + from public_data_v1_0_0.findings_text; + when _table = 'findings' then + select div(count(*), RPB) into count + from public_data_v1_0_0.findings; + when _table = 'general' then + select div(count(*), RPB) into count + from public_data_v1_0_0.general; + when _table = 'notes_to_sefa' then + select div(count(*), RPB) into count + from public_data_v1_0_0.notes_to_sefa; + when _table = 'passthrough' then + select div(count(*), RPB) into count + from public_data_v1_0_0.passthrough; + when _table = 'secondary_auditors' then + select div(count(*), RPB) into count + from public_data_v1_0_0.secondary_auditors; + else + count := 0; + end case; + RETURN count; +end +$batches$ +language plpgsql; + +CREATE OR REPLACE FUNCTION public_api_v1_0_0.compute_batch(row_id bigint) + RETURNS BIGINT + AS $$ + DECLARE result bigint; + BEGIN + SELECT public_api_v1_0_0_functions.batch(row_id) INTO result; + return result; + END + $$ LANGUAGE plpgsql; + + +CREATE OR REPLACE FUNCTION public_api_v1_0_0.get_batch_additional_eins (_batch bigint) +RETURNS SETOF record +AS $get_batch$ +BEGIN + RETURN QUERY SELECT * + FROM public_data_v1_0_0.additional_eins + WHERE batch_number = _batch; +END; +$get_batch$ +LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION public_api_v1_0_0.get_batch_federal_awards (_batch bigint) +RETURNS SETOF public_data_v1_0_0.federal_awards +AS $get_batch$ +BEGIN + RETURN QUERY SELECT * + FROM public_data_v1_0_0.federal_awards + WHERE batch_number = _batch; +END; +$get_batch$ +LANGUAGE plpgsql; + +NOTIFY pgrst, 'reload schema'; + + +-- when _table = 'additional_ueis' then +-- RETURN QUERY SELECT * +-- FROM public_data_v1_0_0.additional_ueis +-- WHERE batch_number = _batch; +-- when _table = 'combined' then +-- RETURN QUERY SELECT * +-- FROM public_data_v1_0_0.combined +-- WHERE batch_number = _batch; +-- when _table = 'corrective_action_plans' then +-- RETURN QUERY SELECT * +-- FROM public_data_v1_0_0.corrective_action_plans +-- WHERE batch_number = _batch; +-- when _table = 'federal_awards' then +-- RETURN QUERY SELECT * +-- FROM public_data_v1_0_0.federal_awards +-- WHERE batch_number = _batch; +-- when _table = 'findings_text' then +-- RETURN QUERY SELECT * +-- FROM public_data_v1_0_0.findings_text +-- WHERE batch_number = _batch; +-- when _table = 'findings' then +-- RETURN QUERY SELECT * +-- FROM public_data_v1_0_0.findings +-- WHERE batch_number = _batch; +-- when _table = 'general' then +-- RETURN QUERY SELECT * +-- FROM public_data_v1_0_0.general +-- WHERE batch_number = _batch; +-- when _table = 'notes_to_sefa' then +-- RETURN QUERY SELECT * +-- FROM public_data_v1_0_0.notes_to_sefa +-- WHERE batch_number = _batch; +-- when _table = 'passthrough' then +-- RETURN QUERY SELECT * +-- FROM public_data_v1_0_0.passthrough +-- WHERE batch_number = _batch; +-- when _table = 'secondary_auditors' then +-- RETURN QUERY SELECT * +-- FROM public_data_v1_0_0.secondary_auditors +-- WHERE batch_number = _batch; +-- end case; +-- END; +-- $get_batch$ +-- language plpgsql; + + +NOTIFY pgrst, 'reload schema'; + +BEGIN; + +CREATE VIEW public_api_v1_0_0.additional_eins AS + SELECT * FROM public_data_v1_0_0.additional_eins ae + ORDER BY ae.id +; + +--------------------------------------- +-- additional_ueis +--------------------------------------- +create view public_api_v1_0_0.additional_ueis AS + SELECT * FROM public_data_v1_0_0.additional_ueis au + ORDER BY au.id +; + +--------------------------------------- +-- corrective_action_plan +--------------------------------------- +CREATE VIEW public_api_v1_0_0.corrective_action_plans AS + SELECT * FROM public_data_v1_0_0.corrective_action_plans cap + ORDER BY cap.id +; + +--------------------------------------- +-- finding +--------------------------------------- +CREATE VIEW public_api_v1_0_0.findings as + SELECT * FROM public_data_v1_0_0.findings f + ORDER BY f.id +; + +--------------------------------------- +-- finding_text +--------------------------------------- +CREATE VIEW public_api_v1_0_0.findings_text AS + SELECT * FROM public_data_v1_0_0.findings_text ft + ORDER BY ft.id +; + +--------------------------------------- +-- federal award +--------------------------------------- +CREATE VIEW public_api_v1_0_0.federal_awards AS + SELECT * FROM public_data_v1_0_0.federal_awards fa + ORDER BY fa.id +; + +--------------------------------------- +-- general +--------------------------------------- +CREATE VIEW public_api_v1_0_0.general AS + SELECT * FROM public_data_v1_0_0.general +; + +--------------------------------------- +-- notes_to_sefa +--------------------------------------- +create view public_api_v1_0_0.notes_to_sefa AS + SELECT * FROM public_data_v1_0_0.notes_to_sefa nts + ORDER BY nts.id +; + +--------------------------------------- +-- passthrough +--------------------------------------- +CREATE VIEW public_api_v1_0_0.passthrough AS + SELECT * FROM public_data_v1_0_0.passthrough p + ORDER BY p.id +; + +--------------------------------------- +-- auditor (secondary auditor) +--------------------------------------- +CREATE VIEW public_api_v1_0_0.secondary_auditors AS + SELECT * FROM public_data_v1_0_0.secondary_auditors sa + ORDER BY sa.id + ; + +--------------------------------------- +-- combined +--------------------------------------- +CREATE VIEW public_api_v1_0_0.combined AS + SELECT * FROM public_data_v1_0_0.combined comb + ; + +--------------------------------------- +-- metadata +--------------------------------------- +CREATE VIEW public_api_v1_0_0.metadata AS + SELECT * FROM public_data_v1_0_0.metadata + ; diff --git a/backend/dissemination/sql/api/public_api_v1_0_0/teardown.sql b/backend/dissemination/sql/api/public_api_v1_0_0/teardown.sql new file mode 100644 index 0000000000..e5feac30d5 --- /dev/null +++ b/backend/dissemination/sql/api/public_api_v1_0_0/teardown.sql @@ -0,0 +1,12 @@ +BEGIN; + -- + -- This drops all the views, too. + --- + DROP SCHEMA IF EXISTS public_api_v1_0_0 CASCADE; + DROP SCHEMA IF EXISTS public_api_v1_0_0_functions CASCADE; + +COMMIT; + + +notify pgrst, + 'reload schema'; diff --git a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/standup.sql b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/standup.sql deleted file mode 100644 index 37f9aa6c52..0000000000 --- a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/standup.sql +++ /dev/null @@ -1,273 +0,0 @@ ------------------------------------------------------ --- ROLES ------------------------------------------------------ -DO -$do$ -BEGIN - IF EXISTS ( - SELECT FROM pg_catalog.pg_roles - WHERE rolname = 'authenticator') THEN - RAISE NOTICE 'Role "authenticator" already exists. Skipping.'; - ELSE - CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; - END IF; -END -$do$; - -DO -$do$ -BEGIN - IF EXISTS ( - SELECT FROM pg_catalog.pg_roles - WHERE rolname = 'api_fac_gov') THEN - RAISE NOTICE 'Role "api_fac_gov" already exists. Skipping.'; - ELSE - CREATE ROLE api_fac_gov NOLOGIN; - END IF; -END -$do$; - -GRANT api_fac_gov TO authenticator; - -NOTIFY pgrst, 'reload schema'; -begin; - ------------------------------------------------------ --- PERMISSIONS ------------------------------------------------------ -do -$$ -BEGIN - DROP SCHEMA IF EXISTS public_api_v2_0_0_alpha CASCADE; - DROP SCHEMA IF EXISTS public_api_v2_0_0_alpha_functions CASCADE; - CREATE SCHEMA IF NOT EXISTS public_api_v2_0_0_alpha; - CREATE SCHEMA IF NOT EXISTS public_api_v2_0_0_alpha_functions; - -- Functions are loaded before sling comes up. - - GRANT USAGE ON SCHEMA public_api_v2_0_0_alpha_functions TO api_fac_gov; - GRANT USAGE ON SCHEMA public_api_v2_0_0_alpha TO api_fac_gov; - GRANT USAGE ON SCHEMA public_data_v1_0_0 TO api_fac_gov; - - -- Grant access to tables and views - ALTER DEFAULT PRIVILEGES - IN SCHEMA public_api_v2_0_0_alpha - GRANT SELECT - ON tables - TO api_fac_gov; - - ALTER DEFAULT PRIVILEGES - IN SCHEMA public_data_v1_0_0 - GRANT SELECT - ON TABLES - TO api_fac_gov; - - -- Grant access to sequences, if we have them - GRANT SELECT, USAGE - ON ALL SEQUENCES - IN SCHEMA public_api_v2_0_0_alpha - TO api_fac_gov; - - GRANT SELECT, USAGE - ON ALL SEQUENCES - IN SCHEMA public_data_v1_0_0 - TO api_fac_gov; - -END -$$ -; - -COMMIT; - -notify pgrst, 'reload schema'; - ---- --- FUNCTIONS --- - -CREATE OR REPLACE FUNCTION public_api_v2_0_0_alpha_functions.rows_per_batch() - RETURNS integer - LANGUAGE sql IMMUTABLE PARALLEL SAFE AS - 'SELECT 20000'; - -CREATE OR REPLACE FUNCTION public_api_v2_0_0_alpha_functions.batch (id bigint) - RETURNS bigint - AS $batch$ - DECLARE result bigint; - DECLARE RPB integer; - BEGIN - SELECT public_api_v2_0_0_alpha_functions.rows_per_batch() INTO RPB; - SELECT div(id, RPB) INTO result; - RETURN result; - END - $batch$ - LANGUAGE plpgsql IMMUTABLE; - -CREATE OR REPLACE FUNCTION public_api_v2_0_0_alpha_functions.batches (diss_table text) -returns integer -as $batches$ -declare count integer; -declare RPB integer; -begin - select public_api_v2_0_0_alpha_functions.rows_per_batch() into RPB; - case - when diss_table = 'additional_eins' then - select div(count(*), RPB) into count - from public_data_v1_0_0.additional_eins; - when diss_table = 'additional_ueis' then - select div(count(*), RPB) into count - from public_data_v1_0_0.additional_ueis; - when diss_table = 'combined' then - select div(count(*), RPB) into count - from public_data_v1_0_0.combined; - when diss_table = 'corrective_action_plans' then - select div(count(*), RPB) into count - from public_data_v1_0_0.corrective_action_plans; - when diss_table = 'federal_awards' then - select div(count(*), RPB) into count - from public_data_v1_0_0.federal_awards; - when diss_table = 'findings_text' then - select div(count(*), RPB) into count - from public_data_v1_0_0.findings_text; - when diss_table = 'findings' then - select div(count(*), RPB) into count - from public_data_v1_0_0.findings; - when diss_table = 'general' then - select div(count(*), RPB) into count - from public_data_v1_0_0.general; - when diss_table = 'notes_to_sefa' then - select div(count(*), RPB) into count - from public_data_v1_0_0.notes_to_sefa; - when diss_table = 'passthrough' then - select div(count(*), RPB) into count - from public_data_v1_0_0.passthrough; - when diss_table = 'secondary_auditors' then - select div(count(*), RPB) into count - from public_data_v1_0_0.secondary_auditors; - else - count := 0; - end case; - RETURN count; -end -$batches$ -language plpgsql; - -CREATE OR REPLACE FUNCTION public_api_v2_0_0_alpha.compute_batch(row_id bigint) - RETURNS BIGINT - AS $$ - DECLARE result bigint; - BEGIN - SELECT public_api_v2_0_0_alpha_functions.batch(row_id) INTO result; - return result; - END - $$ LANGUAGE plpgsql; - - -create or replace function public_api_v2_0_0_alpha.get_federal_award_batch (batch_no bigint) -RETURNS SETOF record -as $batches$ - SELECT * - FROM public_data_v1_0_0.federal_awards - WHERE batch_number = batch_no; -$batches$ -language sql; - - -NOTIFY pgrst, 'reload schema'; - -BEGIN; - -CREATE VIEW public_api_v2_0_0_alpha.additional_eins AS - SELECT * FROM public_data_v1_0_0.additional_eins ae - ORDER BY ae.id -; - ---------------------------------------- --- additional_ueis ---------------------------------------- -create view public_api_v2_0_0_alpha.additional_ueis AS - SELECT * FROM public_data_v1_0_0.additional_ueis au - ORDER BY au.id -; - ---------------------------------------- --- corrective_action_plan ---------------------------------------- -CREATE VIEW public_api_v2_0_0_alpha.corrective_action_plans AS - SELECT * FROM public_data_v1_0_0.corrective_action_plans cap - ORDER BY cap.id -; - ---------------------------------------- --- finding ---------------------------------------- -CREATE VIEW public_api_v2_0_0_alpha.findings as - SELECT * FROM public_data_v1_0_0.findings f - ORDER BY f.id -; - ---------------------------------------- --- finding_text ---------------------------------------- -CREATE VIEW public_api_v2_0_0_alpha.findings_text AS - SELECT * FROM public_data_v1_0_0.findings_text ft - ORDER BY ft.id -; - ---------------------------------------- --- federal award ---------------------------------------- -CREATE VIEW public_api_v2_0_0_alpha.federal_awards AS - SELECT * FROM public_data_v1_0_0.federal_awards fa - ORDER BY fa.id -; - ---------------------------------------- --- general ---------------------------------------- -CREATE VIEW public_api_v2_0_0_alpha.general AS - SELECT * FROM public_data_v1_0_0.general -; - ---------------------------------------- --- notes_to_sefa ---------------------------------------- -create view public_api_v2_0_0_alpha.notes_to_sefa AS - SELECT * FROM public_data_v1_0_0.notes_to_sefa nts - ORDER BY nts.id -; - ---------------------------------------- --- passthrough ---------------------------------------- -CREATE VIEW public_api_v2_0_0_alpha.passthrough AS - SELECT * FROM public_data_v1_0_0.passthrough p - ORDER BY p.id -; - ---------------------------------------- --- auditor (secondary auditor) ---------------------------------------- -CREATE VIEW public_api_v2_0_0_alpha.secondary_auditors AS - SELECT * FROM public_data_v1_0_0.secondary_auditors sa - ORDER BY sa.id - ; - ---------------------------------------- --- combined ---------------------------------------- -CREATE VIEW public_api_v2_0_0_alpha.combined AS - SELECT * FROM public_data_v1_0_0.combined comb - ; - ---------------------------------------- --- metadata ---------------------------------------- -CREATE VIEW public_api_v2_0_0_alpha.metadata AS - SELECT * FROM public_data_v1_0_0.metadata - ; - - -COMMIT; - -notify pgrst, - 'reload schema'; diff --git a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/teardown.sql b/backend/dissemination/sql/api/public_api_v2_0_0_alpha/teardown.sql deleted file mode 100644 index 33fe43ce3e..0000000000 --- a/backend/dissemination/sql/api/public_api_v2_0_0_alpha/teardown.sql +++ /dev/null @@ -1,12 +0,0 @@ -BEGIN; - -- - -- This drops all the views, too. - --- - DROP SCHEMA IF EXISTS public_api_v2_0_0_alpha CASCADE; - DROP SCHEMA IF EXISTS public_api_v2_0_0_alpha_functions CASCADE; - -COMMIT; - - -notify pgrst, - 'reload schema'; diff --git a/backend/dissemination/sql/api/public_data_v1_0_0/standup.sql b/backend/dissemination/sql/api/public_data_v1_0_0/standup.sql index e69de29bb2..e02d3d1929 100644 --- a/backend/dissemination/sql/api/public_data_v1_0_0/standup.sql +++ b/backend/dissemination/sql/api/public_data_v1_0_0/standup.sql @@ -0,0 +1,6 @@ +DO LANGUAGE plpgsql +$NOTE$ + BEGIN + RAISE info 'PUBLIC_DATA_V1_0_0 standup no-op.'; + END; +$NOTE$; diff --git a/backend/dissemination/sql/api/public_data_v1_0_0/teardown.sql b/backend/dissemination/sql/api/public_data_v1_0_0/teardown.sql index 84b8ebefd7..e69de29bb2 100644 --- a/backend/dissemination/sql/api/public_data_v1_0_0/teardown.sql +++ b/backend/dissemination/sql/api/public_data_v1_0_0/teardown.sql @@ -1,30 +0,0 @@ --- Drop it. We regenerate it -DROP SCHEMA IF EXISTS public_data_v1_0_0 CASCADE; - --- We need these sequences for bringing the public tables back. --- That is, sling needs them. So, lets build them right now. -CREATE SCHEMA IF NOT EXISTS public_data_v1_0_0; - -DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_additional_eins; -DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_additional_ueis; -DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_combined; -DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_corrective_action_plans; -DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_federal_awards; -DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_findings_text; -DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_findings; -DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_general; -DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_notes_to_sefa; -DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_passthrough; -DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_secondary_auditors; - -CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_additional_eins START 1; -CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_additional_ueis START 1; -CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_combined START 1; -CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_corrective_action_plans START 1; -CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_federal_awards START 1; -CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_findings_text START 1; -CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_findings START 1; -CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_general START 1; -CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_notes_to_sefa START 1; -CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_passthrough START 1; -CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_secondary_auditors START 1; diff --git a/backend/dissemination/sql/api/roles/standup.sql b/backend/dissemination/sql/api/roles/standup.sql new file mode 100644 index 0000000000..3ab5e55e39 --- /dev/null +++ b/backend/dissemination/sql/api/roles/standup.sql @@ -0,0 +1,7 @@ +DO LANGUAGE plpgsql +$NOTE$ + BEGIN + RAISE info 'ROLES teardown no-op.'; + END; +$NOTE$; + diff --git a/backend/dissemination/sql/api/roles/teardown.sql b/backend/dissemination/sql/api/roles/teardown.sql new file mode 100644 index 0000000000..44eda52bae --- /dev/null +++ b/backend/dissemination/sql/api/roles/teardown.sql @@ -0,0 +1,36 @@ +------------------------------------------------------------------ +-- authenticator role +------------------------------------------------------------------ +DO +$do$ +BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = 'authenticator') THEN + RAISE NOTICE 'ROLES Role "authenticator" already exists. Skipping.'; + ELSE + CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; + END IF; +END +$do$; + +------------------------------------------------------------------ +-- api_fac_gov role +------------------------------------------------------------------ +DO +$do$ +BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = 'api_fac_gov') THEN + RAISE NOTICE 'ROLES Role "api_fac_gov" already exists. Skipping.'; + ELSE + CREATE ROLE api_fac_gov NOLOGIN; + END IF; +END +$do$; + +GRANT api_fac_gov TO authenticator; + +NOTIFY pgrst, 'reload schema'; + diff --git a/backend/dissemination/sql/api/schemas/standup.sql b/backend/dissemination/sql/api/schemas/standup.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/dissemination/sql/api/schemas/teardown.sql b/backend/dissemination/sql/api/schemas/teardown.sql new file mode 100644 index 0000000000..dce01c65ef --- /dev/null +++ b/backend/dissemination/sql/api/schemas/teardown.sql @@ -0,0 +1,3 @@ +-- This has to exist so that `sling` can do its thing. +-- So we create it here, as a late step in the teardown. +CREATE SCHEMA IF NOT EXISTS public_data_v1_0_0; diff --git a/backend/dissemination/sql/api/sequences/standup.sql b/backend/dissemination/sql/api/sequences/standup.sql new file mode 100644 index 0000000000..224514cb5b --- /dev/null +++ b/backend/dissemination/sql/api/sequences/standup.sql @@ -0,0 +1 @@ +RAISE NOTICE 'SEQUENCES standup no-op.'; diff --git a/backend/dissemination/sql/api/sequences/teardown.sql b/backend/dissemination/sql/api/sequences/teardown.sql new file mode 100644 index 0000000000..c2dbbd95e5 --- /dev/null +++ b/backend/dissemination/sql/api/sequences/teardown.sql @@ -0,0 +1,25 @@ +-- We need these sequences for bringing the public tables back. +-- That is, sling needs them. So, lets build them right now. +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_additional_eins; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_additional_ueis; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_combined; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_corrective_action_plans; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_federal_awards; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_findings_text; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_findings; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_general; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_notes_to_sefa; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_passthrough; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_secondary_auditors; + +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_additional_eins START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_additional_ueis START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_combined START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_corrective_action_plans START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_federal_awards START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_findings_text START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_findings START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_general START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_notes_to_sefa START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_passthrough START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_secondary_auditors START 1; diff --git a/backend/dissemination/sql/sling/public_data_v1_0_0/public_metadata_v1_0_0.yaml b/backend/dissemination/sql/sling/public_data_v1_0_0/public_metadata_v1_0_0.yaml index 34afe850d1..1d22a3c7aa 100644 --- a/backend/dissemination/sql/sling/public_data_v1_0_0/public_metadata_v1_0_0.yaml +++ b/backend/dissemination/sql/sling/public_data_v1_0_0/public_metadata_v1_0_0.yaml @@ -21,53 +21,53 @@ streams: object: public_data_v1_0_0.metadata sql: | SELECT 'additional_eins' - AS table, COUNT(*) --, public_api_v2_0_0_alpha_functions.batches('additional_eins') + AS table, COUNT(*) --, public_api_v1_0_0_functions.batches('additional_eins') FROM public_data_v1_0_0.additional_eins UNION SELECT 'additional_ueis' - AS table, COUNT(*) --, public_api_v2_0_0_alpha_functions.batches('additional_ueis') + AS table, COUNT(*) --, public_api_v1_0_0_functions.batches('additional_ueis') FROM public_data_v1_0_0.additional_ueis UNION SELECT 'combined' - AS table, COUNT(*) --, public_api_v2_0_0_alpha_functions.batches('combined') + AS table, COUNT(*) --, public_api_v1_0_0_functions.batches('combined') FROM public_data_v1_0_0.combined UNION SELECT 'federal_awards' - AS table, COUNT(*) --, public_api_v2_0_0_alpha_functions.batches('federal_awards') + AS table, COUNT(*) --, public_api_v1_0_0_functions.batches('federal_awards') FROM public_data_v1_0_0.federal_awards UNION SELECT 'findings_text' - AS table, COUNT(*) --, public_api_v2_0_0_alpha_functions.batches('findings_text') + AS table, COUNT(*) --, public_api_v1_0_0_functions.batches('findings_text') FROM public_data_v1_0_0.findings_text UNION SELECT 'findings' - AS table, COUNT(*) --, public_api_v2_0_0_alpha_functions.batches('findings') + AS table, COUNT(*) --, public_api_v1_0_0_functions.batches('findings') FROM public_data_v1_0_0.findings UNION SELECT 'general' - AS table, COUNT(*) --, public_api_v2_0_0_alpha_functions.batches('general') + AS table, COUNT(*) --, public_api_v1_0_0_functions.batches('general') FROM public_data_v1_0_0.general UNION SELECT 'notes_to_sefa' - AS table, COUNT(*) --, public_api_v2_0_0_alpha_functions.batches('notes_to_sefa') + AS table, COUNT(*) --, public_api_v1_0_0_functions.batches('notes_to_sefa') FROM public_data_v1_0_0.notes_to_sefa UNION SELECT 'passthrough' - AS table, COUNT(*) --, public_api_v2_0_0_alpha_functions.batches('passthrough') + AS table, COUNT(*) --, public_api_v1_0_0_functions.batches('passthrough') FROM public_data_v1_0_0.passthrough UNION SELECT 'secondary_auditors' - AS table, COUNT(*) --, public_api_v2_0_0_alpha_functions.batches('secondary_auditors') + AS table, COUNT(*) --, public_api_v1_0_0_functions.batches('secondary_auditors') FROM public_data_v1_0_0.secondary_auditors UNION SELECT 'tribal_corrective_action_plans' - AS table, COUNT(*) --, public_api_v2_0_0_alpha_functions.batches('tribal_corrective_action_plans') + AS table, COUNT(*) --, public_api_v1_0_0_functions.batches('tribal_corrective_action_plans') FROM public_data_v1_0_0.tribal_corrective_action_plans UNION SELECT 'tribal_findings_text' - AS table, COUNT(*) --, public_api_v2_0_0_alpha_functions.batches('tribal_findings_text') + AS table, COUNT(*) --, public_api_v1_0_0_functions.batches('tribal_findings_text') FROM public_data_v1_0_0.tribal_findings_text UNION SELECT 'tribal_notes_to_sefa' - AS table, COUNT(*) --, public_api_v2_0_0_alpha_functions.batches('tribal_notes_to_sefa') + AS table, COUNT(*) --, public_api_v1_0_0_functions.batches('tribal_notes_to_sefa') FROM public_data_v1_0_0.tribal_notes_to_sefa diff --git a/backend/docker-compose-db-only.yml b/backend/docker-compose-db-only.yml index a88741898c..c4ac82d8a9 100644 --- a/backend/docker-compose-db-only.yml +++ b/backend/docker-compose-db-only.yml @@ -58,7 +58,7 @@ services: PGRST_OPENAPI_SERVER_PROXY_URI: http://127.0.0.1:3000 PGRST_DB_ANON_ROLE: anon # See https://postgrest.org/en/stable/references/api/schemas.html#multiple-schemas for multiple schemas - PGRST_DB_SCHEMAS: "api_v1_0_3,api_v1_1_0,admin_api_v1_1_0,public_api_v2_0_0_alpha" + PGRST_DB_SCHEMAS: "api_v1_0_3,api_v1_1_0,admin_api_v1_1_0,public_api_v1_0_0" PGRST_JWT_SECRET: ${PGRST_JWT_SECRET:-32_chars_fallback_secret_testing} # Fallback value for testing environments # Enable this to inspect the DB plans for queries via EXPLAIN PGRST_DB_PLAN_ENABLED: 1 diff --git a/backend/docker-compose.yml b/backend/docker-compose.yml index 46c4fa0ee2..fe3b29d28f 100644 --- a/backend/docker-compose.yml +++ b/backend/docker-compose.yml @@ -126,7 +126,7 @@ services: PGRST_OPENAPI_SERVER_PROXY_URI: http://127.0.0.1:3000 PGRST_DB_ANON_ROLE: anon # See https://postgrest.org/en/stable/references/api/schemas.html#multiple-schemas for multiple schemas - PGRST_DB_SCHEMAS: "api_v1_0_3,api_v1_1_0,admin_api_v1_1_0,public_api_v2_0_0_alpha" + PGRST_DB_SCHEMAS: "api_v1_0_3,api_v1_1_0,admin_api_v1_1_0,public_api_v1_0_0" PGRST_JWT_SECRET: ${PGRST_JWT_SECRET:-32_chars_fallback_secret_testing} # Fallback value for testing environments # Enable this to inspect the DB plans for queries via EXPLAIN PGRST_DB_PLAN_ENABLED: 1 diff --git a/backend/tools/api_standup.sh b/backend/tools/api_standup.sh index 502edc110c..0bc36ade83 100644 --- a/backend/tools/api_standup.sh +++ b/backend/tools/api_standup.sh @@ -1,35 +1,15 @@ source tools/util_startup.sh - function api_standup { startup_log "API_STANDUP" "BEGIN" - local base_path='dissemination/sql' - local location='api' - - # This loops by index, so we can use two arrays. - # public_api_versions and public_api_required_tables - for index in "${!api_versions[@]}" + # Loop by index. + for index in "${!standup_scripts[@]}" do - local api_version="${api_versions[index]}" - local required_table="${api_required_tables[index]}" - # If the file to stand up the API exists... - if [ -f ${base_path}/${location}/${api_version}/standup.sql ]; then - check_table_exists $FAC_SNAPSHOT_URI $required_table - local result=$? - echo "check_table_exists $required_table $result" - if [ $result -eq 0 ]; then - run_sql $FAC_SNAPSHOT_URI $base_path $location $api_version 'standup.sql' - gonogo "$api_version standup.sql" - else - echo "API TABLE NOT FOUND/SKIPPED $required_table not found for $api_version" - fi - else - echo "API FILE NOT FOUND/SKIPPED dissemination/sql/api/${api_version}/standup.sql" - fi + local subdir="${standup_scripts[index]}" + local path="dissemination/sql/api/${subdir}/standup.sql" + run_sql $FAC_SNAPSHOT_URI $path done - startup_log "API_STANDUP" "END" - return 0 } diff --git a/backend/tools/api_teardown.sh b/backend/tools/api_teardown.sh index 086140a1e5..0ee3046054 100644 --- a/backend/tools/api_teardown.sh +++ b/backend/tools/api_teardown.sh @@ -2,28 +2,13 @@ source tools/util_startup.sh function api_teardown { startup_log "API_TEARDOWN" "BEGIN" - local base_path='dissemination/sql' - local location='api' - local sql_file='teardown.sql' - - # The API now relies on actual tables. - # We can tear down the API safely, because we're tearing down - # just the permissions at this point. That is, we're tearing down - # the API portion. The data will remain in the DB. - - for index in "${!api_versions[@]}" + # Loop by index. + for index in "${!teardown_scripts[@]}" do - local api_version="${api_versions[index]}" - if [ -f ${base_path}/${location}/${api_version}/${sql_file} ]; then - run_sql $FAC_SNAPSHOT_URI $base_path $location $api_version $sql_file - gonogo "$api_version teardown.sql" - else - echo "API FILE NOT FOUND/SKIPPED ${location}/${api_version}/${sql_file}" - fi + local subdir="${teardown_scripts[index]}" + local path="dissemination/sql/api/${subdir}/teardown.sql" + run_sql $FAC_SNAPSHOT_URI $path done - startup_log "API_TEARDOWN" "END" - - # If these are both zero, we're all good. return 0 } diff --git a/backend/tools/setup_cgov_env.sh b/backend/tools/setup_cgov_env.sh index 007e4aa0aa..70bce2796b 100644 --- a/backend/tools/setup_cgov_env.sh +++ b/backend/tools/setup_cgov_env.sh @@ -45,7 +45,7 @@ function setup_cgov_env { export FAC_DB_URI="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "fac-db" ".[][] | select(.name == \$service_name) | .credentials.uri")" export FAC_SNAPSHOT_URI="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "fac-snapshot-db" ".[][] | select(.name == \$service_name) | .credentials.uri")" # https://stackoverflow.com/questions/37072245/check-return-status-of-psql-command-in-unix-shell-scripting - export PSQL_EXE='/home/vcap/deps/0/apt/usr/lib/postgresql/*/bin/psql -v ON_ERROR_STOP=on' + export PSQL_EXE='/home/vcap/deps/0/apt/usr/lib/postgresql/*/bin/psql --single-transaction -v ON_ERROR_STOP=on' return 0 } diff --git a/backend/tools/setup_local_env.sh b/backend/tools/setup_local_env.sh index 68f2e705a6..672eaa8108 100644 --- a/backend/tools/setup_local_env.sh +++ b/backend/tools/setup_local_env.sh @@ -16,7 +16,7 @@ function setup_local_env { # For database work export FAC_DB_URI=${DATABASE_URL}?sslmode=disable export FAC_SNAPSHOT_URI=${SNAPSHOT_URL}?sslmode=disable - export PSQL_EXE='psql -v ON_ERROR_STOP=on' + export PSQL_EXE='psql --single-transaction -v ON_ERROR_STOP=on' export SLING_EXE='/bin/sling' export CGOV_UTIL_EXE='/bin/cgov-util' diff --git a/backend/tools/sling_first_run.sh b/backend/tools/sling_first_run.sh index cd5bdede72..98d9aa7411 100644 --- a/backend/tools/sling_first_run.sh +++ b/backend/tools/sling_first_run.sh @@ -3,6 +3,9 @@ source tools/util_startup.sh function sling_first_run() { startup_log "SLING_FIRST_RUN" "Slinging data to fac-snapshot if needed" + # FIXME. We could do a clean bring-up, and have no data. + # This would end up creating tables, and subsequent runs pass. + # So. check_table_exists $FAC_SNAPSHOT_URI 'public.dissemination_general' local is_general_table=$? if [ $is_general_table -ne 0 ]; then @@ -26,6 +29,7 @@ function sling_first_run() { # Only run sling if the tables in the secondary DB do not exist. if [ $is_metadata_table -ne 0 ]; then + SLING_ALLOW_EMPTY=1 startup_log "SLING_FIRST_RUN" "API tables don't exist; running sling." $SLING_EXE run -r dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml gonogo "sling public data for API tables" diff --git a/backend/tools/util_startup.sh b/backend/tools/util_startup.sh index 357cb3df7f..335737fa86 100644 --- a/backend/tools/util_startup.sh +++ b/backend/tools/util_startup.sh @@ -11,6 +11,7 @@ function startup_log { function gonogo { if [ $? -eq 0 ]; then startup_log "STARTUP_CHECK" "$1 PASS" + return 0 else startup_log "STARTUP_CHECK" "$1 FAIL" exit -1 @@ -38,11 +39,8 @@ function check_schema_exists () { function run_sql () { local db_uri="$1" - local base_path="$2" - local location="$3" - local api_version="$4" - local sql_file="$5" - - $PSQL_EXE $db_uri < ${base_path}/${location}/${api_version}/${sql_file} - gonogo "run_sql < ${base_path}/${location}/${api_version}/${sql_file}" + local path="$2" + echo "BEGIN run_sql < $path" + $PSQL_EXE $db_uri < $path + gonogo "GONOGO run_sql < $path" } diff --git a/backend/tools/variables.sh b/backend/tools/variables.sh index ae36f60476..0eae7301d7 100644 --- a/backend/tools/variables.sh +++ b/backend/tools/variables.sh @@ -1,28 +1,49 @@ # These are our active APIs # admin_api_access_tables must run before # the admin API can be stood up. -declare -a api_versions=( +declare -a standup_scripts=( "api_v1_0_3" "api_v1_1_0" "admin_api_access_tables" "admin_api_v1_1_0" "admin_api_v1_1_1" - "public_api_v2_0_0_alpha" + "public_api_v1_0_0" "public_data_v1_0_0" + "permissions" + "finalize" ) -# These are the tables that must be present -# in order to stand up that API. -declare -a api_required_tables=( - "public.dissemination_general" - "public.dissemination_general" - "public.support_adminapievent" - "public.support_administrative_key_uuids" - "public.support_administrative_key_uuids" - "public_data_v1_0_0.general" - "public.dissemination_general" + +declare -a teardown_scripts=( + "api_v1_0_3" + "api_v1_1_0" + "admin_api_access_tables" + "admin_api_v1_1_0" + "admin_api_v1_1_1" + "public_api_v1_0_0" + "public_data_v1_0_0" + # We need some of this for the + # copying of data from DB1 to DB2. + # Specifically, schemas, roles, permissions, and sequences. + "schemas" + "roles" + "permissions" + "sequences" + "finalize" ) -declare -a db2_indexes=( +# # These are the tables that must be present +# # in order to stand up that API. +# declare -a api_required_tables=( +# "public.dissemination_general" +# "public.dissemination_general" +# "public.support_adminapievent" +# "public.support_administrative_key_uuids" +# "public.support_administrative_key_uuids" +# "public_data_v1_0_0.general" +# "public.dissemination_general" +# ) + +declare -a indexes=( "additional_eins" "additional_ueis" "combined" @@ -36,16 +57,16 @@ declare -a db2_indexes=( "secondary_auditors" ) -declare -a db2_indexes_required_tables=( - "public_data_v1_0_0.additional_eins" - "public_data_v1_0_0.additional_ueis" - "public_data_v1_0_0.combined" - "public_data_v1_0_0.corrective_action_plans" - "public_data_v1_0_0.federal_awards" - "public_data_v1_0_0.findings" - "public_data_v1_0_0.findings_text" - "public_data_v1_0_0.general" - "public_data_v1_0_0.notes_to_sefa" - "public_data_v1_0_0.passthrough" - "public_data_v1_0_0.secondary_auditors" -) +# declare -a db2_indexes_required_tables=( +# "public_data_v1_0_0.additional_eins" +# "public_data_v1_0_0.additional_ueis" +# "public_data_v1_0_0.combined" +# "public_data_v1_0_0.corrective_action_plans" +# "public_data_v1_0_0.federal_awards" +# "public_data_v1_0_0.findings" +# "public_data_v1_0_0.findings_text" +# "public_data_v1_0_0.general" +# "public_data_v1_0_0.notes_to_sefa" +# "public_data_v1_0_0.passthrough" +# "public_data_v1_0_0.secondary_auditors" +# ) diff --git a/backend/util/api_perf_test/perf.py b/backend/util/api_perf_test/perf.py index 76815c5992..75cd3f10a8 100644 --- a/backend/util/api_perf_test/perf.py +++ b/backend/util/api_perf_test/perf.py @@ -9,7 +9,7 @@ # GET {{scheme}}://{{apiUrl}}/general?report_id=eq.2021-12-CENSUS-0000250449 # authorization: {{authorization}} # x-api-user-id: {{xApiUserId}} -# accept-profile: public_api_v2_0_0_alpha +# accept-profile: public_api_v1_0_0 # Accept: application/vnd.pgrst.plan @@ -53,7 +53,7 @@ def fetch_fa_batches_exp(): for batch_no in range(0, 235): query = f"{URI}/federal_awards?batch_number=eq.{batch_no}" headers = { - "accept-profile": "public_api_v2_0_0_alpha", + "accept-profile": "public_api_v1_0_0", "accept": "application/vnd.pgrst.plan+json", "x-api-user-id": os.getenv("API_KEY_ID"), "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", @@ -71,7 +71,7 @@ def fetch_fa_batches_time(): print(f"batch number: {batch_no}") query = f"{URI}/federal_awards?batch_number=eq.{batch_no}" headers = { - "accept-profile": "public_api_v2_0_0_alpha", + "accept-profile": "public_api_v1_0_0", "x-api-user-id": os.getenv("API_KEY_ID"), "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", } @@ -90,7 +90,7 @@ def fetch_fa_batches_time(): results4 = {} results1["ap110"] = fetch_fa_exp("api_v1_1_0") - results1["public200"] = fetch_fa_exp("public_api_v2_0_0_alpha") + results1["public200"] = fetch_fa_exp("public_api_v1_0_0") results1["public200_batches"] = fetch_fa_batches_exp() min = math.inf @@ -103,7 +103,7 @@ def fetch_fa_batches_time(): print("Running timing tests... ~5m") results3["ap110"] = fetch_fa_time("api_v1_1_0") - results3["public200"] = fetch_fa_time("public_api_v2_0_0_alpha") + results3["public200"] = fetch_fa_time("public_api_v1_0_0") results3["public200_batches"] = fetch_fa_batches_time() min = math.inf diff --git a/backend/util/load_public_dissem_data/Dockerfile b/backend/util/load_public_dissem_data/Dockerfile index 3992bac453..686278867d 100644 --- a/backend/util/load_public_dissem_data/Dockerfile +++ b/backend/util/load_public_dissem_data/Dockerfile @@ -8,8 +8,6 @@ RUN apt-get install -y \ unzip WORKDIR /layered -COPY data/sling.yaml . -COPY data/tribal_sling.yaml . COPY data/config.json . # Curl in `sling` @@ -24,11 +22,9 @@ RUN curl -L -O https://github.com/GSA-TTS/fac-backup-utility/releases/download/v && chmod 755 gov.gsa.fac.cgov-util \ && mv gov.gsa.fac.cgov-util /layered/cgov-util -# This grabs a config file from the repo -# that is intended for local devleopment. Therefore, it has all the right values -# for talking to our stack. +# Where do we need the config? Try home? WORKDIR /root/.fac -RUN curl -L -O https://raw.githubusercontent.com/GSA-TTS/fac-backup-utility/refs/heads/main/config.json +COPY data/config.json . WORKDIR /app ENTRYPOINT [ "./load_public_data_locally.sh" ] diff --git a/backend/util/load_public_dissem_data/load_public_data_locally.sh b/backend/util/load_public_dissem_data/load_public_data_locally.sh index 6c6bcfafb7..96be4bf546 100755 --- a/backend/util/load_public_dissem_data/load_public_data_locally.sh +++ b/backend/util/load_public_dissem_data/load_public_data_locally.sh @@ -68,24 +68,24 @@ fi # # happens in production. # pushd /app/data -# # Run the backup of the dissemination_ tables from -# # fac-db to fac-snapshot-db. -# check_table_exists $FAC_SNAPSHOT_URI "public.dissemination_general" -# result=$? -# # cgov-util wants to know the environment it is in. -# export ENV="LOCAL" -# if [ $result -ne 0 ]; then -# # First run if it does not exist. -# /layered/cgov-util db_to_db \ -# --src_db fac-db \ -# --dest_db fac-snapshot-db \ -# --operation initial -# else -# /layered/cgov-util db_to_db \ -# --src_db fac-db \ -# --dest_db fac-snapshot-db \ -# --operation backup -# fi +# Run the backup of the dissemination_ tables from +# fac-db to fac-snapshot-db. +check_table_exists $FAC_SNAPSHOT_URI "public.dissemination_general" +result=$? +# cgov-util wants to know the environment it is in. +export ENV="LOCAL" +if [ $result -ne 0 ]; then + # First run if it does not exist. + /layered/cgov-util db_to_db \ + --src_db fac-db \ + --dest_db fac-snapshot-db \ + --operation initial +else + /layered/cgov-util db_to_db \ + --src_db fac-db \ + --dest_db fac-snapshot-db \ + --operation backup +fi # # Now, we're going to run sling. # # This will create the API tables. It essentially does a copy of diff --git a/terraform/shared/modules/env/postgrest.tf b/terraform/shared/modules/env/postgrest.tf index 0d0c4d7b69..772612dac0 100644 --- a/terraform/shared/modules/env/postgrest.tf +++ b/terraform/shared/modules/env/postgrest.tf @@ -10,7 +10,7 @@ resource "cloudfoundry_route" "postgrest" { resource "cloudfoundry_service_key" "postgrest" { name = "postgrest" - service_instance = module.database.instance_id + service_instance = module.snapshot-database.instance_id } data "docker_registry_image" "postgrest" { @@ -32,7 +32,7 @@ resource "cloudfoundry_app" "postgrest" { environment = { PGRST_DB_URI : cloudfoundry_service_key.postgrest.credentials.uri - PGRST_DB_SCHEMAS : "api_v1_0_3,api_v1_1_0,admin_api_v1_1_0" + PGRST_DB_SCHEMAS : "api_v1_0_3,api_v1_1_0,admin_api_v1_1_0,public_api_v1_0_0" PGRST_DB_ANON_ROLE : "anon" PGRST_JWT_SECRET : var.pgrst_jwt_secret PGRST_DB_MAX_ROWS : 20000 From a809beabd02d2f727fef21a90c4972d0c586b53b Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Wed, 9 Oct 2024 11:16:49 -0400 Subject: [PATCH 16/89] Ready for testing in preview Adds grabbing sling into the mix --- .github/workflows/deploy-application.yml | 263 +++++++++--------- .../fac-backup-and-sling-scheduled.yml | 2 +- backend/.profile | 7 + backend/run.sh | 1 - backend/tools/remove_api_from_fac_db.sh | 3 + backend/tools/sling_first_run.sh | 2 +- backend/tools/sling_support_functions.sh | 15 - 7 files changed, 147 insertions(+), 146 deletions(-) create mode 100644 backend/tools/remove_api_from_fac_db.sh delete mode 100644 backend/tools/sling_support_functions.sh diff --git a/.github/workflows/deploy-application.yml b/.github/workflows/deploy-application.yml index 64cd7019f7..f15029ceee 100644 --- a/.github/workflows/deploy-application.yml +++ b/.github/workflows/deploy-application.yml @@ -1,128 +1,135 @@ ---- -name: Deploy application to cloud.gov -on: - workflow_call: - inputs: - environment: - required: true - type: string - -jobs: - push-with-creds: - name: Deploy to cloud.gov with updated credentials - runs-on: ubuntu-latest - environment: ${{ inputs.environment }} - env: - space: ${{ inputs.environment }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Restore npm cache - uses: actions/cache@v4 - id: cache-npm - with: - path: ~/.npm - key: fac-build-npm-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - fac-build-npm- - fac-build- - - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Restore pip cache - uses: actions/cache@v4 - id: cache-pip - with: - path: | - ~/.cache/pip - /opt/hostedtoolcache/Python/ - key: fac-build-pip-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/dev-requirements.txt') }} - restore-keys: | - fac-build-pip-${{ hashFiles('**/requirements.txt') }}- - fac-build-pip- - fac-build- - - - name: Install npm dependencies - working-directory: ./backend - run: npm ci --production - - - name: Compile JS/CSS assets - working-directory: ./backend - run: npm run build - - - name: Update service keys - uses: cloud-gov/cg-cli-tools@main - env: - SAM_API_KEY: ${{ secrets.SAM_API_KEY }} - DJANGO_SECRET_LOGIN_KEY: $${{ secrets.DJANGO_SECRET_LOGIN_KEY }} - LOGIN_CLIENT_ID: $${{ secrets.LOGIN_CLIENT_ID }} - SECRET_KEY: ${{ secrets.SECRET_KEY }} - with: - cf_username: ${{ secrets.CF_USERNAME }} - cf_password: ${{ secrets.CF_PASSWORD }} - cf_org: gsa-tts-oros-fac - cf_space: ${{ env.space }} - cf_command: update-user-provided-service fac-key-service -p '"{\"SAM_API_KEY\":\"${{ secrets.SAM_API_KEY }}\", \"DJANGO_SECRET_LOGIN_KEY\":\"${{ secrets.DJANGO_SECRET_LOGIN_KEY }}\", \"LOGIN_CLIENT_ID\":\"${{ secrets.LOGIN_CLIENT_ID }}\", \"SECRET_KEY\":\"${{ secrets.SECRET_KEY}}\"}"' - - - name: Backup the database - # if: startsWith(github.ref, 'refs/tags/v1.') - uses: cloud-gov/cg-cli-tools@main - with: - cf_username: ${{ secrets.CF_USERNAME }} - cf_password: ${{ secrets.CF_PASSWORD }} - cf_org: gsa-tts-oros-fac - cf_space: ${{ env.space }} - command: cf run-task gsa-fac -k 7G -m 3G --name deploy_backup --command "./fac-backup-util.sh v0.1.5 deploy_backup" - - - name: Deploy Preview to cloud.gov - if: ${{ inputs.environment == 'preview' }} - uses: cloud-gov/cg-cli-tools@main - with: - cf_username: ${{ secrets.CF_USERNAME }} - cf_password: ${{ secrets.CF_PASSWORD }} - cf_org: gsa-tts-oros-fac - cf_space: ${{ env.space }} - cf_manifest: backend/manifests/manifest-preview.yml - cf_vars_file: backend/manifests/vars/vars-${{ env.space }}.yml - command: bin/ops/deploy_preview.sh - - - name: Deploy fac to cloud.gov - if: ${{ inputs.environment != 'preview' }} - uses: cloud-gov/cg-cli-tools@main - with: - cf_username: ${{ secrets.CF_USERNAME }} - cf_password: ${{ secrets.CF_PASSWORD }} - cf_org: gsa-tts-oros-fac - cf_space: ${{ env.space }} - cf_manifest: backend/manifests/manifest-fac.yml - cf_vars_file: backend/manifests/vars/vars-${{ env.space }}.yml - command: bin/ops/deploy.sh - - - name: Load historical data - uses: cloud-gov/cg-cli-tools@main - with: - cf_username: ${{ secrets.CF_USERNAME }} - cf_password: ${{ secrets.CF_PASSWORD }} - cf_org: gsa-tts-oros-fac - cf_space: ${{ env.space }} - command: cf run-task gsa-fac -k 6G -m 1G --name load_data --command "./load_data.sh" - - # This has to happen after an application deployment because the manifest (currently) is responsible - # for binding the "logdrain service" to the "gsa-fac application". This also needs to be done - # based on the suspicion that fluentbit cannot register the incoming logs when it is initially - # created, resulting in a 502. Restarting the application after everything is configured results - # in a 201, or, the expected status when transmitting logs. - - name: Restart the logshipper application - uses: cloud-gov/cg-cli-tools@main - with: - cf_username: ${{ secrets.CF_USERNAME }} - cf_password: ${{ secrets.CF_PASSWORD }} - cf_org: gsa-tts-oros-fac - cf_space: ${{ env.space }} - command: cf restart logshipper - +--- +name: Deploy application to cloud.gov +on: + workflow_call: + inputs: + environment: + required: true + type: string + +jobs: + push-with-creds: + name: Deploy to cloud.gov with updated credentials + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + env: + space: ${{ inputs.environment }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Restore npm cache + uses: actions/cache@v4 + id: cache-npm + with: + path: ~/.npm + key: fac-build-npm-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + fac-build-npm- + fac-build- + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Restore pip cache + uses: actions/cache@v4 + id: cache-pip + with: + path: | + ~/.cache/pip + /opt/hostedtoolcache/Python/ + key: fac-build-pip-${{ hashFiles('**/requirements.txt') }}-${{ hashFiles('**/dev-requirements.txt') }} + restore-keys: | + fac-build-pip-${{ hashFiles('**/requirements.txt') }}- + fac-build-pip- + fac-build- + + - name: Install npm dependencies + working-directory: ./backend + run: npm ci --production + + - name: Compile JS/CSS assets + working-directory: ./backend + run: npm run build + + - name: Grab sling for copying data from DB1 to DB2 + working-directory: ./backend + run: | + curl -L -O https://github.com/slingdata-io/sling-cli/releases/download/v1.2.20/sling_linux_amd64.tar.gz + tar xzf sling_linux_amd64.tar.gz + chmod 755 sling + + - name: Update service keys + uses: cloud-gov/cg-cli-tools@main + env: + SAM_API_KEY: ${{ secrets.SAM_API_KEY }} + DJANGO_SECRET_LOGIN_KEY: $${{ secrets.DJANGO_SECRET_LOGIN_KEY }} + LOGIN_CLIENT_ID: $${{ secrets.LOGIN_CLIENT_ID }} + SECRET_KEY: ${{ secrets.SECRET_KEY }} + with: + cf_username: ${{ secrets.CF_USERNAME }} + cf_password: ${{ secrets.CF_PASSWORD }} + cf_org: gsa-tts-oros-fac + cf_space: ${{ env.space }} + cf_command: update-user-provided-service fac-key-service -p '"{\"SAM_API_KEY\":\"${{ secrets.SAM_API_KEY }}\", \"DJANGO_SECRET_LOGIN_KEY\":\"${{ secrets.DJANGO_SECRET_LOGIN_KEY }}\", \"LOGIN_CLIENT_ID\":\"${{ secrets.LOGIN_CLIENT_ID }}\", \"SECRET_KEY\":\"${{ secrets.SECRET_KEY}}\"}"' + + - name: Backup the database + # if: startsWith(github.ref, 'refs/tags/v1.') + uses: cloud-gov/cg-cli-tools@main + with: + cf_username: ${{ secrets.CF_USERNAME }} + cf_password: ${{ secrets.CF_PASSWORD }} + cf_org: gsa-tts-oros-fac + cf_space: ${{ env.space }} + command: cf run-task gsa-fac -k 7G -m 3G --name deploy_backup --command "./fac-backup-util.sh v0.1.5 deploy_backup" + + - name: Deploy Preview to cloud.gov + if: ${{ inputs.environment == 'preview' }} + uses: cloud-gov/cg-cli-tools@main + with: + cf_username: ${{ secrets.CF_USERNAME }} + cf_password: ${{ secrets.CF_PASSWORD }} + cf_org: gsa-tts-oros-fac + cf_space: ${{ env.space }} + cf_manifest: backend/manifests/manifest-preview.yml + cf_vars_file: backend/manifests/vars/vars-${{ env.space }}.yml + command: bin/ops/deploy_preview.sh + + - name: Deploy fac to cloud.gov + if: ${{ inputs.environment != 'preview' }} + uses: cloud-gov/cg-cli-tools@main + with: + cf_username: ${{ secrets.CF_USERNAME }} + cf_password: ${{ secrets.CF_PASSWORD }} + cf_org: gsa-tts-oros-fac + cf_space: ${{ env.space }} + cf_manifest: backend/manifests/manifest-fac.yml + cf_vars_file: backend/manifests/vars/vars-${{ env.space }}.yml + command: bin/ops/deploy.sh + + - name: Load historical data + uses: cloud-gov/cg-cli-tools@main + with: + cf_username: ${{ secrets.CF_USERNAME }} + cf_password: ${{ secrets.CF_PASSWORD }} + cf_org: gsa-tts-oros-fac + cf_space: ${{ env.space }} + command: cf run-task gsa-fac -k 6G -m 1G --name load_data --command "./load_data.sh" + + # This has to happen after an application deployment because the manifest (currently) is responsible + # for binding the "logdrain service" to the "gsa-fac application". This also needs to be done + # based on the suspicion that fluentbit cannot register the incoming logs when it is initially + # created, resulting in a 502. Restarting the application after everything is configured results + # in a 201, or, the expected status when transmitting logs. + - name: Restart the logshipper application + uses: cloud-gov/cg-cli-tools@main + with: + cf_username: ${{ secrets.CF_USERNAME }} + cf_password: ${{ secrets.CF_PASSWORD }} + cf_org: gsa-tts-oros-fac + cf_space: ${{ env.space }} + command: cf restart logshipper + diff --git a/.github/workflows/fac-backup-and-sling-scheduled.yml b/.github/workflows/fac-backup-and-sling-scheduled.yml index cf4ca4aa78..ac4fe09c59 100644 --- a/.github/workflows/fac-backup-and-sling-scheduled.yml +++ b/.github/workflows/fac-backup-and-sling-scheduled.yml @@ -32,5 +32,5 @@ jobs: cf_password: ${{ secrets.CF_PASSWORD }} cf_org: gsa-tts-oros-fac cf_space: ${{ env.space }} - command: cf run-task gsa-fac -k 7G -m 3G --name backup_util_scheduled --command "./tools/fac-backup-and-sling.sh ${{ inputs.util_version }} ${{ inputs.backup_operation }}" + command: cf run-task gsa-fac -k 7G -m 3G --name backup_and_sling_scheduled --command "./util/fac-backup-and-sling-scheduled.sh ${{ inputs.util_version }} ${{ inputs.backup_operation }}" diff --git a/backend/.profile b/backend/.profile index 6c4c89023f..a8484f1c73 100644 --- a/backend/.profile +++ b/backend/.profile @@ -34,6 +34,13 @@ if [[ "$CF_INSTANCE_INDEX" == 0 ]]; then migrate_app_tables gonogo "migrate_app_tables" + ##### + # PREP API TABLES + # This runs sling and preps tables in the snapshot DB. + # Only runs if the tables are not present (e.g. first deploy) + sling_first_run + gonogo "sling_first_run" + ##### # API STANDUP # Standup the API, which may depend on migration changes diff --git a/backend/run.sh b/backend/run.sh index ce257309a8..ddc4bcb0e8 100755 --- a/backend/run.sh +++ b/backend/run.sh @@ -11,7 +11,6 @@ source tools/util_startup.sh source tools/setup_env.sh source tools/api_teardown.sh source tools/migrate_app_tables.sh -source tools/sling_support_functions.sh source tools/sling_first_run.sh source tools/api_standup.sh source tools/build_indexes.sh diff --git a/backend/tools/remove_api_from_fac_db.sh b/backend/tools/remove_api_from_fac_db.sh new file mode 100644 index 0000000000..8c93a92bd1 --- /dev/null +++ b/backend/tools/remove_api_from_fac_db.sh @@ -0,0 +1,3 @@ +# run this once in each env +# after we deploy the new API and everything +# is happy. diff --git a/backend/tools/sling_first_run.sh b/backend/tools/sling_first_run.sh index 98d9aa7411..b8e3dea08c 100644 --- a/backend/tools/sling_first_run.sh +++ b/backend/tools/sling_first_run.sh @@ -36,7 +36,7 @@ function sling_first_run() { $SLING_EXE run -r dissemination/sql/sling/public_data_v1_0_0/tribal_data_v1_0_0.yaml gonogo "sling tribal data for API tables" $SLING_EXE run -r dissemination/sql/sling/public_data_v1_0_0/public_metadata_v1_0_0.yaml - gonogo "sling tribal data for API tables" + gonogo "sling metadata table" startup_log "SLING_FIRST_RUN" "Successfully ran sling to generate tables." else startup_log "SLING_FIRST_RUN" "API tables exist; skipping sling." diff --git a/backend/tools/sling_support_functions.sh b/backend/tools/sling_support_functions.sh deleted file mode 100644 index 644a5f3434..0000000000 --- a/backend/tools/sling_support_functions.sh +++ /dev/null @@ -1,15 +0,0 @@ -source tools/util_startup.sh - -function sling_support_functions() { - startup_log "SLING_SUPPORT_FUNCTIONS" "Loading functions needed by sling" - - check_table_exists $FAC_SNAPSHOT_URI 'public_data_v1_0_0.general' - gonogo "sling_support_functions general exists" - - # We need to load some functions for sling to complete, because - # we use those functions as part of the metadata table generation. - local base_path='dissemination/sql' - local location='sling' - run_sql $FAC_SNAPSHOT_URI $base_path $location 'public_data_v1_0_0' 'sling_functions.sql' - return $? -} From 88bc8a5b36631bfdd8c4b4296189a7bc1eb6099b Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Wed, 9 Oct 2024 11:30:37 -0400 Subject: [PATCH 17/89] Source the script... Testing again --- backend/.profile | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/.profile b/backend/.profile index a8484f1c73..7d445d725e 100644 --- a/backend/.profile +++ b/backend/.profile @@ -10,6 +10,7 @@ source tools/util_startup.sh # for local envs (LOCAL or TESTING) and cloud.gov source tools/setup_env.sh source tools/api_teardown.sh +source tools/sling_first_run.sh source tools/build_indexes.sh source tools/migrate_app_tables.sh source tools/api_standup.sh From 0e4f2403e9d3208c97acdc13ca78b9aabd7ee49a Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Wed, 9 Oct 2024 12:04:49 -0400 Subject: [PATCH 18/89] Fix path to sling --- backend/tools/setup_cgov_env.sh | 1 + backend/tools/sling_first_run.sh | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/backend/tools/setup_cgov_env.sh b/backend/tools/setup_cgov_env.sh index 70bce2796b..d4ca2e1954 100644 --- a/backend/tools/setup_cgov_env.sh +++ b/backend/tools/setup_cgov_env.sh @@ -46,6 +46,7 @@ function setup_cgov_env { export FAC_SNAPSHOT_URI="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "fac-snapshot-db" ".[][] | select(.name == \$service_name) | .credentials.uri")" # https://stackoverflow.com/questions/37072245/check-return-status-of-psql-command-in-unix-shell-scripting export PSQL_EXE='/home/vcap/deps/0/apt/usr/lib/postgresql/*/bin/psql --single-transaction -v ON_ERROR_STOP=on' + export SLING_EXE='/home/vcap/app/sling' return 0 } diff --git a/backend/tools/sling_first_run.sh b/backend/tools/sling_first_run.sh index b8e3dea08c..7aa3f55925 100644 --- a/backend/tools/sling_first_run.sh +++ b/backend/tools/sling_first_run.sh @@ -3,9 +3,9 @@ source tools/util_startup.sh function sling_first_run() { startup_log "SLING_FIRST_RUN" "Slinging data to fac-snapshot if needed" - # FIXME. We could do a clean bring-up, and have no data. - # This would end up creating tables, and subsequent runs pass. - # So. + # The deploy action does a DB->DB backup with cgov-util. + # This should only ever actually run when testing locally. + # In the cloud environment, it will skip the actul run. check_table_exists $FAC_SNAPSHOT_URI 'public.dissemination_general' local is_general_table=$? if [ $is_general_table -ne 0 ]; then From a94598c9f4448e351e9c1a91c4b6f0361b70e27d Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Wed, 9 Oct 2024 12:42:46 -0400 Subject: [PATCH 19/89] Making the checks noisy. --- backend/tools/util_startup.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/tools/util_startup.sh b/backend/tools/util_startup.sh index 335737fa86..794395d12d 100644 --- a/backend/tools/util_startup.sh +++ b/backend/tools/util_startup.sh @@ -23,6 +23,7 @@ function check_table_exists() { local dbname="$2" $PSQL_EXE $db_uri -c "SELECT '$dbname'::regclass" >/dev/null 2>&1 result=$? + echo "CHECK_TABLE_EXISTS $dbname: $result" return $result } @@ -30,6 +31,7 @@ function check_schema_exists () { local db_uri="$1" local schema_name="$2" local result=$(psql $db_uri -qtAX -c "SELECT EXISTS(SELECT 1 FROM information_schema.schemata WHERE schema_name = '$schema_name');") + echo "CHECK_SCHEMA_EXISTS $schema_name: $result" if [ "$result" = "t" ]; then return 0; else From 2006d8afef8658bb455cd8ec937f0d40490828e2 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Wed, 9 Oct 2024 12:46:51 -0400 Subject: [PATCH 20/89] Noisyer. --- backend/tools/util_startup.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/tools/util_startup.sh b/backend/tools/util_startup.sh index 794395d12d..65fe7fbde9 100644 --- a/backend/tools/util_startup.sh +++ b/backend/tools/util_startup.sh @@ -21,7 +21,9 @@ function gonogo { function check_table_exists() { local db_uri="$1" local dbname="$2" - $PSQL_EXE $db_uri -c "SELECT '$dbname'::regclass" >/dev/null 2>&1 + echo "CHECK_TABLE_EXISTS: $dbname" + # >/dev/null 2>&1 + $PSQL_EXE $db_uri -c "SELECT '$dbname'::regclass" result=$? echo "CHECK_TABLE_EXISTS $dbname: $result" return $result @@ -30,7 +32,8 @@ function check_table_exists() { function check_schema_exists () { local db_uri="$1" local schema_name="$2" - local result=$(psql $db_uri -qtAX -c "SELECT EXISTS(SELECT 1 FROM information_schema.schemata WHERE schema_name = '$schema_name');") + echo "CHECK_SCHEMA_EXISTS $schema_name" + local result=$($PSQL_EXE $db_uri -qtAX -c "SELECT EXISTS(SELECT 1 FROM information_schema.schemata WHERE schema_name = '$schema_name');") echo "CHECK_SCHEMA_EXISTS $schema_name: $result" if [ "$result" = "t" ]; then return 0; From b95cb6164ec67ed4e9c119f1eea06cae6faff44b Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Wed, 9 Oct 2024 13:01:01 -0400 Subject: [PATCH 21/89] Don't exit so quickly... --- backend/tools/util_startup.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/tools/util_startup.sh b/backend/tools/util_startup.sh index 65fe7fbde9..22274fb5ea 100644 --- a/backend/tools/util_startup.sh +++ b/backend/tools/util_startup.sh @@ -21,20 +21,24 @@ function gonogo { function check_table_exists() { local db_uri="$1" local dbname="$2" + set +e echo "CHECK_TABLE_EXISTS: $dbname" # >/dev/null 2>&1 $PSQL_EXE $db_uri -c "SELECT '$dbname'::regclass" result=$? echo "CHECK_TABLE_EXISTS $dbname: $result" + set -e return $result } function check_schema_exists () { local db_uri="$1" local schema_name="$2" + set +e echo "CHECK_SCHEMA_EXISTS $schema_name" local result=$($PSQL_EXE $db_uri -qtAX -c "SELECT EXISTS(SELECT 1 FROM information_schema.schemata WHERE schema_name = '$schema_name');") echo "CHECK_SCHEMA_EXISTS $schema_name: $result" + set -e if [ "$result" = "t" ]; then return 0; else From 6f1d15310013f4035689d10467090d85075c6656 Mon Sep 17 00:00:00 2001 From: Alex Steel <130377221+asteel-gsa@users.noreply.github.com> Date: Wed, 9 Oct 2024 14:47:28 -0400 Subject: [PATCH 22/89] Try to catch the error and continue if found --- backend/tools/util_startup.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/backend/tools/util_startup.sh b/backend/tools/util_startup.sh index 22274fb5ea..66280260e0 100644 --- a/backend/tools/util_startup.sh +++ b/backend/tools/util_startup.sh @@ -21,13 +21,11 @@ function gonogo { function check_table_exists() { local db_uri="$1" local dbname="$2" - set +e echo "CHECK_TABLE_EXISTS: $dbname" # >/dev/null 2>&1 - $PSQL_EXE $db_uri -c "SELECT '$dbname'::regclass" + $PSQL_EXE $db_uri -c "SELECT '$dbname'::regclass" || true result=$? echo "CHECK_TABLE_EXISTS $dbname: $result" - set -e return $result } From 8c2a04710af2e09d8d42dad7e911b5159efdcbc3 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Thu, 10 Oct 2024 05:30:12 -0400 Subject: [PATCH 23/89] Before further renaming. --- .../{standup.sql => post.sql} | 0 .../{teardown.sql => pre.sql} | 0 .../{standup.sql => post.sql} | 0 .../{teardown.sql => pre.sql} | 0 .../{standup.sql => post.sql} | 0 .../{teardown.sql => pre.sql} | 0 .../api/api_v1_0_3/{standup.sql => post.sql} | 0 .../api/api_v1_0_3/{teardown.sql => pre.sql} | 0 .../api/api_v1_1_0/{standup.sql => post.sql} | 0 .../api/api_v1_1_0/{teardown.sql => pre.sql} | 0 .../api/finalize/{standup.sql => post.sql} | 0 .../api/finalize/{teardown.sql => pre.sql} | 0 .../api/permissions/{standup.sql => post.sql} | 0 .../{standup.sql => post.sql} | 0 .../{standup.sql => post.sql} | 0 .../{standup.sql => post.sql} | 0 .../sql/api/roles/{standup.sql => post.sql} | 0 .../sql/api/schemas/{standup.sql => post.sql} | 0 .../api/sequences/{standup.sql => post.sql} | 0 .../public_data_v1_0_0.yaml | 107 +++++++++++++++++- backend/tools/{api_standup.sh => sql_post.sh} | 0 backend/tools/{api_teardown.sh => sql_pre.sh} | 0 backend/tools/util_startup.sh | 7 +- backend/tools/variables.sh | 21 ++-- backend/util/load_public_dissem_data/Makefile | 2 +- .../load_public_data_locally.sh | 19 ++-- 26 files changed, 132 insertions(+), 24 deletions(-) rename backend/dissemination/sql/api/admin_api_access_tables/{standup.sql => post.sql} (100%) rename backend/dissemination/sql/api/admin_api_access_tables/{teardown.sql => pre.sql} (100%) rename backend/dissemination/sql/api/admin_api_v1_1_0/{standup.sql => post.sql} (100%) rename backend/dissemination/sql/api/admin_api_v1_1_0/{teardown.sql => pre.sql} (100%) rename backend/dissemination/sql/api/admin_api_v1_1_1/{standup.sql => post.sql} (100%) rename backend/dissemination/sql/api/admin_api_v1_1_1/{teardown.sql => pre.sql} (100%) rename backend/dissemination/sql/api/api_v1_0_3/{standup.sql => post.sql} (100%) rename backend/dissemination/sql/api/api_v1_0_3/{teardown.sql => pre.sql} (100%) rename backend/dissemination/sql/api/api_v1_1_0/{standup.sql => post.sql} (100%) rename backend/dissemination/sql/api/api_v1_1_0/{teardown.sql => pre.sql} (100%) rename backend/dissemination/sql/api/finalize/{standup.sql => post.sql} (100%) rename backend/dissemination/sql/api/finalize/{teardown.sql => pre.sql} (100%) rename backend/dissemination/sql/api/permissions/{standup.sql => post.sql} (100%) rename backend/dissemination/sql/api/prep_snapshot_db/{standup.sql => post.sql} (100%) rename backend/dissemination/sql/api/public_api_v1_0_0/{standup.sql => post.sql} (100%) rename backend/dissemination/sql/api/public_data_v1_0_0/{standup.sql => post.sql} (100%) rename backend/dissemination/sql/api/roles/{standup.sql => post.sql} (100%) rename backend/dissemination/sql/api/schemas/{standup.sql => post.sql} (100%) rename backend/dissemination/sql/api/sequences/{standup.sql => post.sql} (100%) rename backend/tools/{api_standup.sh => sql_post.sh} (100%) rename backend/tools/{api_teardown.sh => sql_pre.sh} (100%) diff --git a/backend/dissemination/sql/api/admin_api_access_tables/standup.sql b/backend/dissemination/sql/api/admin_api_access_tables/post.sql similarity index 100% rename from backend/dissemination/sql/api/admin_api_access_tables/standup.sql rename to backend/dissemination/sql/api/admin_api_access_tables/post.sql diff --git a/backend/dissemination/sql/api/admin_api_access_tables/teardown.sql b/backend/dissemination/sql/api/admin_api_access_tables/pre.sql similarity index 100% rename from backend/dissemination/sql/api/admin_api_access_tables/teardown.sql rename to backend/dissemination/sql/api/admin_api_access_tables/pre.sql diff --git a/backend/dissemination/sql/api/admin_api_v1_1_0/standup.sql b/backend/dissemination/sql/api/admin_api_v1_1_0/post.sql similarity index 100% rename from backend/dissemination/sql/api/admin_api_v1_1_0/standup.sql rename to backend/dissemination/sql/api/admin_api_v1_1_0/post.sql diff --git a/backend/dissemination/sql/api/admin_api_v1_1_0/teardown.sql b/backend/dissemination/sql/api/admin_api_v1_1_0/pre.sql similarity index 100% rename from backend/dissemination/sql/api/admin_api_v1_1_0/teardown.sql rename to backend/dissemination/sql/api/admin_api_v1_1_0/pre.sql diff --git a/backend/dissemination/sql/api/admin_api_v1_1_1/standup.sql b/backend/dissemination/sql/api/admin_api_v1_1_1/post.sql similarity index 100% rename from backend/dissemination/sql/api/admin_api_v1_1_1/standup.sql rename to backend/dissemination/sql/api/admin_api_v1_1_1/post.sql diff --git a/backend/dissemination/sql/api/admin_api_v1_1_1/teardown.sql b/backend/dissemination/sql/api/admin_api_v1_1_1/pre.sql similarity index 100% rename from backend/dissemination/sql/api/admin_api_v1_1_1/teardown.sql rename to backend/dissemination/sql/api/admin_api_v1_1_1/pre.sql diff --git a/backend/dissemination/sql/api/api_v1_0_3/standup.sql b/backend/dissemination/sql/api/api_v1_0_3/post.sql similarity index 100% rename from backend/dissemination/sql/api/api_v1_0_3/standup.sql rename to backend/dissemination/sql/api/api_v1_0_3/post.sql diff --git a/backend/dissemination/sql/api/api_v1_0_3/teardown.sql b/backend/dissemination/sql/api/api_v1_0_3/pre.sql similarity index 100% rename from backend/dissemination/sql/api/api_v1_0_3/teardown.sql rename to backend/dissemination/sql/api/api_v1_0_3/pre.sql diff --git a/backend/dissemination/sql/api/api_v1_1_0/standup.sql b/backend/dissemination/sql/api/api_v1_1_0/post.sql similarity index 100% rename from backend/dissemination/sql/api/api_v1_1_0/standup.sql rename to backend/dissemination/sql/api/api_v1_1_0/post.sql diff --git a/backend/dissemination/sql/api/api_v1_1_0/teardown.sql b/backend/dissemination/sql/api/api_v1_1_0/pre.sql similarity index 100% rename from backend/dissemination/sql/api/api_v1_1_0/teardown.sql rename to backend/dissemination/sql/api/api_v1_1_0/pre.sql diff --git a/backend/dissemination/sql/api/finalize/standup.sql b/backend/dissemination/sql/api/finalize/post.sql similarity index 100% rename from backend/dissemination/sql/api/finalize/standup.sql rename to backend/dissemination/sql/api/finalize/post.sql diff --git a/backend/dissemination/sql/api/finalize/teardown.sql b/backend/dissemination/sql/api/finalize/pre.sql similarity index 100% rename from backend/dissemination/sql/api/finalize/teardown.sql rename to backend/dissemination/sql/api/finalize/pre.sql diff --git a/backend/dissemination/sql/api/permissions/standup.sql b/backend/dissemination/sql/api/permissions/post.sql similarity index 100% rename from backend/dissemination/sql/api/permissions/standup.sql rename to backend/dissemination/sql/api/permissions/post.sql diff --git a/backend/dissemination/sql/api/prep_snapshot_db/standup.sql b/backend/dissemination/sql/api/prep_snapshot_db/post.sql similarity index 100% rename from backend/dissemination/sql/api/prep_snapshot_db/standup.sql rename to backend/dissemination/sql/api/prep_snapshot_db/post.sql diff --git a/backend/dissemination/sql/api/public_api_v1_0_0/standup.sql b/backend/dissemination/sql/api/public_api_v1_0_0/post.sql similarity index 100% rename from backend/dissemination/sql/api/public_api_v1_0_0/standup.sql rename to backend/dissemination/sql/api/public_api_v1_0_0/post.sql diff --git a/backend/dissemination/sql/api/public_data_v1_0_0/standup.sql b/backend/dissemination/sql/api/public_data_v1_0_0/post.sql similarity index 100% rename from backend/dissemination/sql/api/public_data_v1_0_0/standup.sql rename to backend/dissemination/sql/api/public_data_v1_0_0/post.sql diff --git a/backend/dissemination/sql/api/roles/standup.sql b/backend/dissemination/sql/api/roles/post.sql similarity index 100% rename from backend/dissemination/sql/api/roles/standup.sql rename to backend/dissemination/sql/api/roles/post.sql diff --git a/backend/dissemination/sql/api/schemas/standup.sql b/backend/dissemination/sql/api/schemas/post.sql similarity index 100% rename from backend/dissemination/sql/api/schemas/standup.sql rename to backend/dissemination/sql/api/schemas/post.sql diff --git a/backend/dissemination/sql/api/sequences/standup.sql b/backend/dissemination/sql/api/sequences/post.sql similarity index 100% rename from backend/dissemination/sql/api/sequences/standup.sql rename to backend/dissemination/sql/api/sequences/post.sql diff --git a/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml b/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml index 58ef7e1bcd..815583bbfd 100644 --- a/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml +++ b/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml @@ -22,7 +22,11 @@ streams: -- All of the general table is public data. SELECT gen.id as id, - div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_general'), 20000) as batch_number, + NEXTVAL('public_data_v1_0_0.seq_v1_0_0_general') AS seq, + -- Add in post + -- DIV(seq, 20000) AS batch_number, + -- What if we use this for partitioning, instead? + -- mod((gen.id - min(gen.id)), 20000) AS batch_number_2, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -110,6 +114,107 @@ streams: FROM public.dissemination_general gen ORDER BY gen.id + target_options: + pre_sql: | + -- DROP SCHEMA IF EXISTS public_data_v1_0_0 CASCADE; + DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_additional_eins; + DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_additional_ueis; + DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_combined; + DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_corrective_action_plans; + DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_federal_awards; + DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_findings_text; + DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_findings; + DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_general; + DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_notes_to_sefa; + DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_passthrough; + DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_secondary_auditors; + -- Now, create them. We get a start at 1 for every refresh this way. + CREATE SCHEMA IF NOT EXISTS public_data_v1_0_0; + CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_additional_eins START 1; + CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_additional_ueis START 1; + CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_combined START 1; + CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_corrective_action_plans START 1; + CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_federal_awards START 1; + CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_findings_text START 1; + CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_findings START 1; + CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_general START 1; + CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_notes_to_sefa START 1; + CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_passthrough START 1; + CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_secondary_auditors START 1; + post_sql: | + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.general + ADD COLUMN batch_number INTEGER; + -- Update that batch number as a modulus of 20K, our response + -- size limit in PostgREST. + UPDATE public_data_v1_0_0.general SET batch_number=DIV(seq, 20000); + -- -- Create a temporary table that is a copy of the general table, but partition it. + -- CREATE TABLE public_data_v1_0_0.tmp_general + -- (LIKE public_data_v1_0_0.general) + -- PARTITION BY hash(seq); + -- -- Drop our existing general table. + -- DROP TABLE public_data_v1_0_0.general; + -- SET search_path TO public_data_v1_0_0; --? + -- -- Rename that temporary table to the general table. Why? + -- -- We cannot create the table with a PARTITION, because it is a SELECT. + -- -- This works around that. + -- ALTER TABLE public_data_v1_0_0.tmp_general RENAME TO general; + -- -- Create the partition tables. + -- CREATE TABLE public_data_v1_0_0.part_general_0 + -- PARTITION OF public_data_v1_0_0.general + -- FOR VALUES WITH (modulus 5, remainder 0); + -- CREATE TABLE public_data_v1_0_0.part_general_1 + -- PARTITION OF public_data_v1_0_0.general + -- FOR VALUES WITH (modulus 5, remainder 1); + -- CREATE TABLE public_data_v1_0_0.part_general_2 + -- PARTITION OF public_data_v1_0_0.general + -- FOR VALUES WITH (modulus 5, remainder 2); + -- CREATE TABLE public_data_v1_0_0.part_general_3 + -- PARTITION OF public_data_v1_0_0.general + -- FOR VALUES WITH (modulus 5, remainder 3); + -- CREATE TABLE public_data_v1_0_0.part_general_4 + -- PARTITION OF public_data_v1_0_0.general + -- FOR VALUES WITH (modulus 5, remainder 4); + -- Create the indexes on the table. + CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_report_id_idx + on public_data_v1_0_0.general (report_id); + CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_report_id_fad + on public_data_v1_0_0.general (report_id, fac_accepted_date); + CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_report_id_fad + on public_data_v1_0_0.general (report_id, fac_accepted_date); + CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditee_certify_name_idx + ON public_data_v1_0_0.general + ((lower(auditee_certify_name))); + CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditee_name_idx + ON public_data_v1_0_0.general + ((lower(auditee_name))); + CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditor_certify_name_idx + ON public_data_v1_0_0.general + ((lower(auditor_certify_name))); + CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditor_contact_name_idx + ON public_data_v1_0_0.general + ((lower(auditor_contact_name))); + CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditor_firm_name_idx + ON public_data_v1_0_0.general + ((lower(auditor_firm_name))); + CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditee_email_idx + on public_data_v1_0_0.general ((lower(auditee_email))); + CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditor_email_idx + on public_data_v1_0_0.general ((lower(auditor_email))); + CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_start_date_idx + ON public_data_v1_0_0.general (fy_start_date); + CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_end_date_idx + ON public_data_v1_0_0.general (fy_end_date); + CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditee_uei_idx + ON public_data_v1_0_0.general (auditee_uei); + CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditee_ein_idx + ON public_data_v1_0_0.general (auditee_ein); + CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_audit_year_idx + on public_data_v1_0_0.general (audit_year); + CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_batch_general_idx + ON public_data_v1_0_0.general (batch_number); + + # All additional EINs are public. public.additional_eins: diff --git a/backend/tools/api_standup.sh b/backend/tools/sql_post.sh similarity index 100% rename from backend/tools/api_standup.sh rename to backend/tools/sql_post.sh diff --git a/backend/tools/api_teardown.sh b/backend/tools/sql_pre.sh similarity index 100% rename from backend/tools/api_teardown.sh rename to backend/tools/sql_pre.sh diff --git a/backend/tools/util_startup.sh b/backend/tools/util_startup.sh index 66280260e0..d027356c4b 100644 --- a/backend/tools/util_startup.sh +++ b/backend/tools/util_startup.sh @@ -23,8 +23,8 @@ function check_table_exists() { local dbname="$2" echo "CHECK_TABLE_EXISTS: $dbname" # >/dev/null 2>&1 - $PSQL_EXE $db_uri -c "SELECT '$dbname'::regclass" || true - result=$? + $PSQL_EXE $db_uri -c "SELECT '$dbname'::regclass" || false + local result=$? echo "CHECK_TABLE_EXISTS $dbname: $result" return $result } @@ -34,7 +34,8 @@ function check_schema_exists () { local schema_name="$2" set +e echo "CHECK_SCHEMA_EXISTS $schema_name" - local result=$($PSQL_EXE $db_uri -qtAX -c "SELECT EXISTS(SELECT 1 FROM information_schema.schemata WHERE schema_name = '$schema_name');") + $PSQL_EXE $db_uri -qtAX -c "SELECT EXISTS(SELECT 1 FROM information_schema.schemata WHERE schema_name = '$schema_name');" || false + local result=$? echo "CHECK_SCHEMA_EXISTS $schema_name: $result" set -e if [ "$result" = "t" ]; then diff --git a/backend/tools/variables.sh b/backend/tools/variables.sh index 0eae7301d7..903d168605 100644 --- a/backend/tools/variables.sh +++ b/backend/tools/variables.sh @@ -1,7 +1,7 @@ # These are our active APIs # admin_api_access_tables must run before # the admin API can be stood up. -declare -a standup_scripts=( +declare -a sql_pre_scripts=( "api_v1_0_3" "api_v1_1_0" "admin_api_access_tables" @@ -9,11 +9,17 @@ declare -a standup_scripts=( "admin_api_v1_1_1" "public_api_v1_0_0" "public_data_v1_0_0" + # We need some of this for the + # copying of data from DB1 to DB2. + # Specifically, schemas, roles, permissions, and sequences. + "schemas" + "roles" "permissions" + "sequences" "finalize" - ) +) -declare -a teardown_scripts=( +declare -a sql_post_scripts=( "api_v1_0_3" "api_v1_1_0" "admin_api_access_tables" @@ -21,15 +27,10 @@ declare -a teardown_scripts=( "admin_api_v1_1_1" "public_api_v1_0_0" "public_data_v1_0_0" - # We need some of this for the - # copying of data from DB1 to DB2. - # Specifically, schemas, roles, permissions, and sequences. - "schemas" - "roles" "permissions" - "sequences" "finalize" -) + ) + # # These are the tables that must be present # # in order to stand up that API. diff --git a/backend/util/load_public_dissem_data/Makefile b/backend/util/load_public_dissem_data/Makefile index 079d45b022..bd02fe9bc7 100644 --- a/backend/util/load_public_dissem_data/Makefile +++ b/backend/util/load_public_dissem_data/Makefile @@ -1,4 +1,4 @@ -NETWORK?=backend-web-1 +NETWORK?=backend-db2-1 build: docker build -t facloaddata . diff --git a/backend/util/load_public_dissem_data/load_public_data_locally.sh b/backend/util/load_public_dissem_data/load_public_data_locally.sh index 96be4bf546..f102204fc2 100755 --- a/backend/util/load_public_dissem_data/load_public_data_locally.sh +++ b/backend/util/load_public_dissem_data/load_public_data_locally.sh @@ -25,19 +25,20 @@ sleep 4 rm -f /app/data/db_dissem_dump rm -rf /app/data/__MACOSX -# Next, we drop the public_data schema. -# This is because we want to make sure it is -# regenerated fresh. -psql $FAC_SNAPSHOT_URI -c "DROP SCHEMA IF EXISTS public_data_v1_0_0 CASCADE" +# # Next, we drop the public_data schema. +# # This is because we want to make sure it is +# # regenerated fresh. +# psql $FAC_SNAPSHOT_URI -c "DROP SCHEMA IF EXISTS public_data_v1_0_0 CASCADE" -# Now, the schema for the public_data is -# created. This provies a place for the tables to -# land when we run sling -psql $FAC_SNAPSHOT_URI -c "CREATE SCHEMA IF NOT EXISTS public_data_v1_0_0" -psql $FAC_SNAPSHOT_URI -c "CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_combined START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE" +# # Now, the schema for the public_data is +# # created. This provies a place for the tables to +# # land when we run sling +# psql $FAC_SNAPSHOT_URI -c "CREATE SCHEMA IF NOT EXISTS public_data_v1_0_0" +# psql $FAC_SNAPSHOT_URI -c "CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_combined START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE" # Unzip the compressed historical data dump. pushd /app/data +echo "Unzipping data." unzip db_dissem_dump.zip popd From e3cc950f8875b1529579d8c3e38231c716722933 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Thu, 10 Oct 2024 10:45:53 -0400 Subject: [PATCH 24/89] Simplified. Also, added partitions. Partitions are 3x faster for downloading all the data (by year), but 10x slower than batches (via EXPLAIN). In the real, it is 2x slower to download than batches, but 3x faster than a straight download. So. Probably worth it, if we can document that people downloading all the data should either 1) do so year-by-year, or 2) use batches. --- .../create_partition_statements.py | 112 +++++++ .../fac-snapshot-db/post/001_partitioning.sql | 303 ++++++++++++++++++ .../post/010_api_v1_0_3.sql} | 0 .../post/020_api_v1_1_0.sql} | 0 .../post/030_admin_api_access_tables.sql} | 0 .../post/040_admin_api_v1_1_0.sql} | 0 .../post/050_admin_api_v1_1_1.sql} | 0 .../post/060_public_api_v1_0_0.sql} | 0 .../post/070_public_data_v1_0_0.sql} | 0 .../post/080_permissions.sql} | 0 .../post/950_indexes_additional_eins.sql} | 0 .../post/951_indexes_additional_ueis.sql} | 0 .../post/952_indexes_combined.sql} | 0 .../953_indexes_corrective_action_plans.sql} | 0 .../post/954_indexes_federal_awards.sql} | 0 .../post/955_indexes_findings.sql} | 0 .../post/956_indexes_findings_text.sql} | 0 .../post/957_indexes_general.sql} | 0 .../post/958_indexes_notes_to_sefa.sql} | 0 .../post/959_indexes_passthrough.sql} | 0 .../post/960_indexes_secondary_auditors.sql} | 0 .../post/999_finalize.sql} | 0 .../pre/000_api_v1_0_3.sql} | 0 .../pre/010_api_v1_1_0.sql} | 0 .../pre/020_admin_api_v1_1_0.sql} | 0 .../pre/025_admin_api_access_tables.sql} | 0 .../pre/030_admin_api_v1_1_1.sql} | 0 .../pre/040_public_api_v1_0_0.sql} | 0 .../pre/050_public_data_v1_0_0.sql} | 0 .../pre/060_schemas.sql} | 0 .../pre/070_schemas.sql} | 0 .../pre/080_permissions.sql} | 0 .../pre/090_sequences.sql} | 0 .../pre/999_finalize.sql} | 0 backend/dissemination/sql/public.rest | 53 +++ backend/tools/sql_pre.sh | 14 - backend/tools/sql_pre_post.sh | 37 +++ .../tools/{sql_post.sh => sql_pre_post.sql} | 0 38 files changed, 505 insertions(+), 14 deletions(-) create mode 100644 backend/dissemination/sql/fac-snapshot-db/create_partition_statements.py create mode 100644 backend/dissemination/sql/fac-snapshot-db/post/001_partitioning.sql rename backend/dissemination/sql/{api/api_v1_0_3/post.sql => fac-snapshot-db/post/010_api_v1_0_3.sql} (100%) rename backend/dissemination/sql/{api/api_v1_1_0/post.sql => fac-snapshot-db/post/020_api_v1_1_0.sql} (100%) rename backend/dissemination/sql/{api/admin_api_access_tables/post.sql => fac-snapshot-db/post/030_admin_api_access_tables.sql} (100%) rename backend/dissemination/sql/{api/admin_api_v1_1_0/post.sql => fac-snapshot-db/post/040_admin_api_v1_1_0.sql} (100%) rename backend/dissemination/sql/{api/admin_api_v1_1_1/post.sql => fac-snapshot-db/post/050_admin_api_v1_1_1.sql} (100%) rename backend/dissemination/sql/{api/public_api_v1_0_0/post.sql => fac-snapshot-db/post/060_public_api_v1_0_0.sql} (100%) rename backend/dissemination/sql/{api/public_data_v1_0_0/post.sql => fac-snapshot-db/post/070_public_data_v1_0_0.sql} (100%) rename backend/dissemination/sql/{api/permissions/post.sql => fac-snapshot-db/post/080_permissions.sql} (100%) rename backend/dissemination/sql/{indexes/fac-snapshot-db/additional_eins.sql => fac-snapshot-db/post/950_indexes_additional_eins.sql} (100%) rename backend/dissemination/sql/{indexes/fac-snapshot-db/additional_ueis.sql => fac-snapshot-db/post/951_indexes_additional_ueis.sql} (100%) rename backend/dissemination/sql/{indexes/fac-snapshot-db/combined.sql => fac-snapshot-db/post/952_indexes_combined.sql} (100%) rename backend/dissemination/sql/{indexes/fac-snapshot-db/corrective_action_plans.sql => fac-snapshot-db/post/953_indexes_corrective_action_plans.sql} (100%) rename backend/dissemination/sql/{indexes/fac-snapshot-db/federal_awards.sql => fac-snapshot-db/post/954_indexes_federal_awards.sql} (100%) rename backend/dissemination/sql/{indexes/fac-snapshot-db/findings.sql => fac-snapshot-db/post/955_indexes_findings.sql} (100%) rename backend/dissemination/sql/{indexes/fac-snapshot-db/findings_text.sql => fac-snapshot-db/post/956_indexes_findings_text.sql} (100%) rename backend/dissemination/sql/{indexes/fac-snapshot-db/general.sql => fac-snapshot-db/post/957_indexes_general.sql} (100%) rename backend/dissemination/sql/{indexes/fac-snapshot-db/notes_to_sefa.sql => fac-snapshot-db/post/958_indexes_notes_to_sefa.sql} (100%) rename backend/dissemination/sql/{indexes/fac-snapshot-db/passthrough.sql => fac-snapshot-db/post/959_indexes_passthrough.sql} (100%) rename backend/dissemination/sql/{indexes/fac-snapshot-db/secondary_auditors.sql => fac-snapshot-db/post/960_indexes_secondary_auditors.sql} (100%) rename backend/dissemination/sql/{api/finalize/post.sql => fac-snapshot-db/post/999_finalize.sql} (100%) rename backend/dissemination/sql/{api/api_v1_0_3/pre.sql => fac-snapshot-db/pre/000_api_v1_0_3.sql} (100%) rename backend/dissemination/sql/{api/api_v1_1_0/pre.sql => fac-snapshot-db/pre/010_api_v1_1_0.sql} (100%) rename backend/dissemination/sql/{api/admin_api_v1_1_0/pre.sql => fac-snapshot-db/pre/020_admin_api_v1_1_0.sql} (100%) rename backend/dissemination/sql/{api/admin_api_access_tables/pre.sql => fac-snapshot-db/pre/025_admin_api_access_tables.sql} (100%) rename backend/dissemination/sql/{api/admin_api_v1_1_1/pre.sql => fac-snapshot-db/pre/030_admin_api_v1_1_1.sql} (100%) rename backend/dissemination/sql/{api/public_api_v1_0_0/teardown.sql => fac-snapshot-db/pre/040_public_api_v1_0_0.sql} (100%) rename backend/dissemination/sql/{api/public_data_v1_0_0/teardown.sql => fac-snapshot-db/pre/050_public_data_v1_0_0.sql} (100%) rename backend/dissemination/sql/{api/schemas/teardown.sql => fac-snapshot-db/pre/060_schemas.sql} (100%) rename backend/dissemination/sql/{api/roles/teardown.sql => fac-snapshot-db/pre/070_schemas.sql} (100%) rename backend/dissemination/sql/{api/permissions/teardown.sql => fac-snapshot-db/pre/080_permissions.sql} (100%) rename backend/dissemination/sql/{api/sequences/teardown.sql => fac-snapshot-db/pre/090_sequences.sql} (100%) rename backend/dissemination/sql/{api/finalize/pre.sql => fac-snapshot-db/pre/999_finalize.sql} (100%) create mode 100644 backend/dissemination/sql/public.rest delete mode 100644 backend/tools/sql_pre.sh create mode 100644 backend/tools/sql_pre_post.sh rename backend/tools/{sql_post.sh => sql_pre_post.sql} (100%) diff --git a/backend/dissemination/sql/fac-snapshot-db/create_partition_statements.py b/backend/dissemination/sql/fac-snapshot-db/create_partition_statements.py new file mode 100644 index 0000000000..d998a8838b --- /dev/null +++ b/backend/dissemination/sql/fac-snapshot-db/create_partition_statements.py @@ -0,0 +1,112 @@ +import argparse +import sys + +# for i in general federal_awards combined ; do python create_partition_statements.py general 20 ; done + +# rm post/001_partitioning.sql ; for i in general federal_awards combined ; do python create_partition_statements.py $i >> post/001_partitioning.sql ; done + + +#################################### +# by hash (makes no sense?) +#################################### +def partition_by_sequence_hash(): + + parser = argparse.ArgumentParser("simple_example") + parser.add_argument( + "table_name", help="Table to create partition statements for", type=str + ) + parser.add_argument("number_of_partitions", help="number of partitions", type=int) + args = parser.parse_args() + table_name = args.table_name + number_of_partitions = args.number_of_partitions + + print( + f""" +------------------------------------------------------ +-- {table_name} - {number_of_partitions} partitions (hash seq) +------------------------------------------------------ +""" + ) + print("SET search_path TO public_data_v1_0_0;") + + print( + f"ALTER TABLE public_data_v1_0_0.{table_name} RENAME TO {table_name}_to_be_removed;" + ) + print( + f"""CREATE TABLE public_data_v1_0_0.{table_name} + (LIKE public_data_v1_0_0.{table_name}_to_be_removed) + PARTITION BY hash(seq); + """ + ) + + for ndx in range(number_of_partitions): + print( + f""" + DROP TABLE IF EXISTS public_data_v1_0_0.part_{table_name}_{ndx:02}; + CREATE TABLE public_data_v1_0_0.part_{table_name}_{ndx:02} + PARTITION OF public_data_v1_0_0.{table_name} + FOR VALUES WITH (modulus {number_of_partitions}, remainder {ndx}); + """ + ) + print( + f""" + INSERT INTO public_data_v1_0_0.{table_name} + SELECT * FROM public_data_v1_0_0.{table_name}_to_be_removed; + """ + ) + print(f"DROP TABLE public_data_v1_0_0.{table_name}_to_be_removed;") + + +#################################### +# by audit year +#################################### +def partition_by_audit_year(): + + parser = argparse.ArgumentParser("simple_example") + parser.add_argument( + "table_name", help="Table to create partition statements for", type=str + ) + args = parser.parse_args() + table_name = args.table_name + + print( + f""" +------------------------------------------------------ +-- {table_name} - partitions (audit_year) +------------------------------------------------------ +-- python {' '.join(sys.argv)} +""" + ) + print("SET search_path TO public_data_v1_0_0;") + + print( + f"ALTER TABLE public_data_v1_0_0.{table_name} RENAME TO {table_name}_to_be_removed;" + ) + print( + f"""CREATE TABLE public_data_v1_0_0.{table_name} + (LIKE public_data_v1_0_0.{table_name}_to_be_removed) + PARTITION BY list(audit_year); + """ + ) + + for ndx in range(16, 30): + print( + f""" + DROP TABLE IF EXISTS public_data_v1_0_0.part_{table_name}_20{ndx:02}; + CREATE TABLE public_data_v1_0_0.part_{table_name}_20{ndx:02} + PARTITION OF public_data_v1_0_0.{table_name} + FOR VALUES IN ('20{ndx:02}'); + """ + ) + print( + f""" + INSERT INTO public_data_v1_0_0.{table_name} + SELECT * FROM public_data_v1_0_0.{table_name}_to_be_removed; + """ + ) + print(f"DROP TABLE public_data_v1_0_0.{table_name}_to_be_removed;") + + +if __name__ == "__main__": + # partition_by_sequence_hash() + partition_by_audit_year() diff --git a/backend/dissemination/sql/fac-snapshot-db/post/001_partitioning.sql b/backend/dissemination/sql/fac-snapshot-db/post/001_partitioning.sql new file mode 100644 index 0000000000..026c10fd88 --- /dev/null +++ b/backend/dissemination/sql/fac-snapshot-db/post/001_partitioning.sql @@ -0,0 +1,303 @@ + +------------------------------------------------------ +-- general - partitions (audit_year) +------------------------------------------------------ +-- python create_partition_statements.py general + +SET search_path TO public_data_v1_0_0; +ALTER TABLE public_data_v1_0_0.general RENAME TO general_to_be_removed; +CREATE TABLE public_data_v1_0_0.general + (LIKE public_data_v1_0_0.general_to_be_removed) + PARTITION BY list(audit_year); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_general_2016; + CREATE TABLE public_data_v1_0_0.part_general_2016 + PARTITION OF public_data_v1_0_0.general + FOR VALUES IN ('2016'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_general_2017; + CREATE TABLE public_data_v1_0_0.part_general_2017 + PARTITION OF public_data_v1_0_0.general + FOR VALUES IN ('2017'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_general_2018; + CREATE TABLE public_data_v1_0_0.part_general_2018 + PARTITION OF public_data_v1_0_0.general + FOR VALUES IN ('2018'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_general_2019; + CREATE TABLE public_data_v1_0_0.part_general_2019 + PARTITION OF public_data_v1_0_0.general + FOR VALUES IN ('2019'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_general_2020; + CREATE TABLE public_data_v1_0_0.part_general_2020 + PARTITION OF public_data_v1_0_0.general + FOR VALUES IN ('2020'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_general_2021; + CREATE TABLE public_data_v1_0_0.part_general_2021 + PARTITION OF public_data_v1_0_0.general + FOR VALUES IN ('2021'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_general_2022; + CREATE TABLE public_data_v1_0_0.part_general_2022 + PARTITION OF public_data_v1_0_0.general + FOR VALUES IN ('2022'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_general_2023; + CREATE TABLE public_data_v1_0_0.part_general_2023 + PARTITION OF public_data_v1_0_0.general + FOR VALUES IN ('2023'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_general_2024; + CREATE TABLE public_data_v1_0_0.part_general_2024 + PARTITION OF public_data_v1_0_0.general + FOR VALUES IN ('2024'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_general_2025; + CREATE TABLE public_data_v1_0_0.part_general_2025 + PARTITION OF public_data_v1_0_0.general + FOR VALUES IN ('2025'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_general_2026; + CREATE TABLE public_data_v1_0_0.part_general_2026 + PARTITION OF public_data_v1_0_0.general + FOR VALUES IN ('2026'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_general_2027; + CREATE TABLE public_data_v1_0_0.part_general_2027 + PARTITION OF public_data_v1_0_0.general + FOR VALUES IN ('2027'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_general_2028; + CREATE TABLE public_data_v1_0_0.part_general_2028 + PARTITION OF public_data_v1_0_0.general + FOR VALUES IN ('2028'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_general_2029; + CREATE TABLE public_data_v1_0_0.part_general_2029 + PARTITION OF public_data_v1_0_0.general + FOR VALUES IN ('2029'); + + + INSERT INTO public_data_v1_0_0.general + SELECT * FROM public_data_v1_0_0.general_to_be_removed; + +DROP TABLE public_data_v1_0_0.general_to_be_removed; + +------------------------------------------------------ +-- federal_awards - partitions (audit_year) +------------------------------------------------------ +-- python create_partition_statements.py federal_awards + +SET search_path TO public_data_v1_0_0; +ALTER TABLE public_data_v1_0_0.federal_awards RENAME TO federal_awards_to_be_removed; +CREATE TABLE public_data_v1_0_0.federal_awards + (LIKE public_data_v1_0_0.federal_awards_to_be_removed) + PARTITION BY list(audit_year); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_federal_awards_2016; + CREATE TABLE public_data_v1_0_0.part_federal_awards_2016 + PARTITION OF public_data_v1_0_0.federal_awards + FOR VALUES IN ('2016'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_federal_awards_2017; + CREATE TABLE public_data_v1_0_0.part_federal_awards_2017 + PARTITION OF public_data_v1_0_0.federal_awards + FOR VALUES IN ('2017'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_federal_awards_2018; + CREATE TABLE public_data_v1_0_0.part_federal_awards_2018 + PARTITION OF public_data_v1_0_0.federal_awards + FOR VALUES IN ('2018'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_federal_awards_2019; + CREATE TABLE public_data_v1_0_0.part_federal_awards_2019 + PARTITION OF public_data_v1_0_0.federal_awards + FOR VALUES IN ('2019'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_federal_awards_2020; + CREATE TABLE public_data_v1_0_0.part_federal_awards_2020 + PARTITION OF public_data_v1_0_0.federal_awards + FOR VALUES IN ('2020'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_federal_awards_2021; + CREATE TABLE public_data_v1_0_0.part_federal_awards_2021 + PARTITION OF public_data_v1_0_0.federal_awards + FOR VALUES IN ('2021'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_federal_awards_2022; + CREATE TABLE public_data_v1_0_0.part_federal_awards_2022 + PARTITION OF public_data_v1_0_0.federal_awards + FOR VALUES IN ('2022'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_federal_awards_2023; + CREATE TABLE public_data_v1_0_0.part_federal_awards_2023 + PARTITION OF public_data_v1_0_0.federal_awards + FOR VALUES IN ('2023'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_federal_awards_2024; + CREATE TABLE public_data_v1_0_0.part_federal_awards_2024 + PARTITION OF public_data_v1_0_0.federal_awards + FOR VALUES IN ('2024'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_federal_awards_2025; + CREATE TABLE public_data_v1_0_0.part_federal_awards_2025 + PARTITION OF public_data_v1_0_0.federal_awards + FOR VALUES IN ('2025'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_federal_awards_2026; + CREATE TABLE public_data_v1_0_0.part_federal_awards_2026 + PARTITION OF public_data_v1_0_0.federal_awards + FOR VALUES IN ('2026'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_federal_awards_2027; + CREATE TABLE public_data_v1_0_0.part_federal_awards_2027 + PARTITION OF public_data_v1_0_0.federal_awards + FOR VALUES IN ('2027'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_federal_awards_2028; + CREATE TABLE public_data_v1_0_0.part_federal_awards_2028 + PARTITION OF public_data_v1_0_0.federal_awards + FOR VALUES IN ('2028'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_federal_awards_2029; + CREATE TABLE public_data_v1_0_0.part_federal_awards_2029 + PARTITION OF public_data_v1_0_0.federal_awards + FOR VALUES IN ('2029'); + + + INSERT INTO public_data_v1_0_0.federal_awards + SELECT * FROM public_data_v1_0_0.federal_awards_to_be_removed; + +DROP TABLE public_data_v1_0_0.federal_awards_to_be_removed; + +------------------------------------------------------ +-- combined - partitions (audit_year) +------------------------------------------------------ +-- python create_partition_statements.py combined + +SET search_path TO public_data_v1_0_0; +ALTER TABLE public_data_v1_0_0.combined RENAME TO combined_to_be_removed; +CREATE TABLE public_data_v1_0_0.combined + (LIKE public_data_v1_0_0.combined_to_be_removed) + PARTITION BY list(audit_year); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_combined_2016; + CREATE TABLE public_data_v1_0_0.part_combined_2016 + PARTITION OF public_data_v1_0_0.combined + FOR VALUES IN ('2016'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_combined_2017; + CREATE TABLE public_data_v1_0_0.part_combined_2017 + PARTITION OF public_data_v1_0_0.combined + FOR VALUES IN ('2017'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_combined_2018; + CREATE TABLE public_data_v1_0_0.part_combined_2018 + PARTITION OF public_data_v1_0_0.combined + FOR VALUES IN ('2018'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_combined_2019; + CREATE TABLE public_data_v1_0_0.part_combined_2019 + PARTITION OF public_data_v1_0_0.combined + FOR VALUES IN ('2019'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_combined_2020; + CREATE TABLE public_data_v1_0_0.part_combined_2020 + PARTITION OF public_data_v1_0_0.combined + FOR VALUES IN ('2020'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_combined_2021; + CREATE TABLE public_data_v1_0_0.part_combined_2021 + PARTITION OF public_data_v1_0_0.combined + FOR VALUES IN ('2021'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_combined_2022; + CREATE TABLE public_data_v1_0_0.part_combined_2022 + PARTITION OF public_data_v1_0_0.combined + FOR VALUES IN ('2022'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_combined_2023; + CREATE TABLE public_data_v1_0_0.part_combined_2023 + PARTITION OF public_data_v1_0_0.combined + FOR VALUES IN ('2023'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_combined_2024; + CREATE TABLE public_data_v1_0_0.part_combined_2024 + PARTITION OF public_data_v1_0_0.combined + FOR VALUES IN ('2024'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_combined_2025; + CREATE TABLE public_data_v1_0_0.part_combined_2025 + PARTITION OF public_data_v1_0_0.combined + FOR VALUES IN ('2025'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_combined_2026; + CREATE TABLE public_data_v1_0_0.part_combined_2026 + PARTITION OF public_data_v1_0_0.combined + FOR VALUES IN ('2026'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_combined_2027; + CREATE TABLE public_data_v1_0_0.part_combined_2027 + PARTITION OF public_data_v1_0_0.combined + FOR VALUES IN ('2027'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_combined_2028; + CREATE TABLE public_data_v1_0_0.part_combined_2028 + PARTITION OF public_data_v1_0_0.combined + FOR VALUES IN ('2028'); + + + DROP TABLE IF EXISTS public_data_v1_0_0.part_combined_2029; + CREATE TABLE public_data_v1_0_0.part_combined_2029 + PARTITION OF public_data_v1_0_0.combined + FOR VALUES IN ('2029'); + + + INSERT INTO public_data_v1_0_0.combined + SELECT * FROM public_data_v1_0_0.combined_to_be_removed; + +DROP TABLE public_data_v1_0_0.combined_to_be_removed; diff --git a/backend/dissemination/sql/api/api_v1_0_3/post.sql b/backend/dissemination/sql/fac-snapshot-db/post/010_api_v1_0_3.sql similarity index 100% rename from backend/dissemination/sql/api/api_v1_0_3/post.sql rename to backend/dissemination/sql/fac-snapshot-db/post/010_api_v1_0_3.sql diff --git a/backend/dissemination/sql/api/api_v1_1_0/post.sql b/backend/dissemination/sql/fac-snapshot-db/post/020_api_v1_1_0.sql similarity index 100% rename from backend/dissemination/sql/api/api_v1_1_0/post.sql rename to backend/dissemination/sql/fac-snapshot-db/post/020_api_v1_1_0.sql diff --git a/backend/dissemination/sql/api/admin_api_access_tables/post.sql b/backend/dissemination/sql/fac-snapshot-db/post/030_admin_api_access_tables.sql similarity index 100% rename from backend/dissemination/sql/api/admin_api_access_tables/post.sql rename to backend/dissemination/sql/fac-snapshot-db/post/030_admin_api_access_tables.sql diff --git a/backend/dissemination/sql/api/admin_api_v1_1_0/post.sql b/backend/dissemination/sql/fac-snapshot-db/post/040_admin_api_v1_1_0.sql similarity index 100% rename from backend/dissemination/sql/api/admin_api_v1_1_0/post.sql rename to backend/dissemination/sql/fac-snapshot-db/post/040_admin_api_v1_1_0.sql diff --git a/backend/dissemination/sql/api/admin_api_v1_1_1/post.sql b/backend/dissemination/sql/fac-snapshot-db/post/050_admin_api_v1_1_1.sql similarity index 100% rename from backend/dissemination/sql/api/admin_api_v1_1_1/post.sql rename to backend/dissemination/sql/fac-snapshot-db/post/050_admin_api_v1_1_1.sql diff --git a/backend/dissemination/sql/api/public_api_v1_0_0/post.sql b/backend/dissemination/sql/fac-snapshot-db/post/060_public_api_v1_0_0.sql similarity index 100% rename from backend/dissemination/sql/api/public_api_v1_0_0/post.sql rename to backend/dissemination/sql/fac-snapshot-db/post/060_public_api_v1_0_0.sql diff --git a/backend/dissemination/sql/api/public_data_v1_0_0/post.sql b/backend/dissemination/sql/fac-snapshot-db/post/070_public_data_v1_0_0.sql similarity index 100% rename from backend/dissemination/sql/api/public_data_v1_0_0/post.sql rename to backend/dissemination/sql/fac-snapshot-db/post/070_public_data_v1_0_0.sql diff --git a/backend/dissemination/sql/api/permissions/post.sql b/backend/dissemination/sql/fac-snapshot-db/post/080_permissions.sql similarity index 100% rename from backend/dissemination/sql/api/permissions/post.sql rename to backend/dissemination/sql/fac-snapshot-db/post/080_permissions.sql diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/additional_eins.sql b/backend/dissemination/sql/fac-snapshot-db/post/950_indexes_additional_eins.sql similarity index 100% rename from backend/dissemination/sql/indexes/fac-snapshot-db/additional_eins.sql rename to backend/dissemination/sql/fac-snapshot-db/post/950_indexes_additional_eins.sql diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/additional_ueis.sql b/backend/dissemination/sql/fac-snapshot-db/post/951_indexes_additional_ueis.sql similarity index 100% rename from backend/dissemination/sql/indexes/fac-snapshot-db/additional_ueis.sql rename to backend/dissemination/sql/fac-snapshot-db/post/951_indexes_additional_ueis.sql diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/combined.sql b/backend/dissemination/sql/fac-snapshot-db/post/952_indexes_combined.sql similarity index 100% rename from backend/dissemination/sql/indexes/fac-snapshot-db/combined.sql rename to backend/dissemination/sql/fac-snapshot-db/post/952_indexes_combined.sql diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/corrective_action_plans.sql b/backend/dissemination/sql/fac-snapshot-db/post/953_indexes_corrective_action_plans.sql similarity index 100% rename from backend/dissemination/sql/indexes/fac-snapshot-db/corrective_action_plans.sql rename to backend/dissemination/sql/fac-snapshot-db/post/953_indexes_corrective_action_plans.sql diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/federal_awards.sql b/backend/dissemination/sql/fac-snapshot-db/post/954_indexes_federal_awards.sql similarity index 100% rename from backend/dissemination/sql/indexes/fac-snapshot-db/federal_awards.sql rename to backend/dissemination/sql/fac-snapshot-db/post/954_indexes_federal_awards.sql diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/findings.sql b/backend/dissemination/sql/fac-snapshot-db/post/955_indexes_findings.sql similarity index 100% rename from backend/dissemination/sql/indexes/fac-snapshot-db/findings.sql rename to backend/dissemination/sql/fac-snapshot-db/post/955_indexes_findings.sql diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/findings_text.sql b/backend/dissemination/sql/fac-snapshot-db/post/956_indexes_findings_text.sql similarity index 100% rename from backend/dissemination/sql/indexes/fac-snapshot-db/findings_text.sql rename to backend/dissemination/sql/fac-snapshot-db/post/956_indexes_findings_text.sql diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/general.sql b/backend/dissemination/sql/fac-snapshot-db/post/957_indexes_general.sql similarity index 100% rename from backend/dissemination/sql/indexes/fac-snapshot-db/general.sql rename to backend/dissemination/sql/fac-snapshot-db/post/957_indexes_general.sql diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/notes_to_sefa.sql b/backend/dissemination/sql/fac-snapshot-db/post/958_indexes_notes_to_sefa.sql similarity index 100% rename from backend/dissemination/sql/indexes/fac-snapshot-db/notes_to_sefa.sql rename to backend/dissemination/sql/fac-snapshot-db/post/958_indexes_notes_to_sefa.sql diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/passthrough.sql b/backend/dissemination/sql/fac-snapshot-db/post/959_indexes_passthrough.sql similarity index 100% rename from backend/dissemination/sql/indexes/fac-snapshot-db/passthrough.sql rename to backend/dissemination/sql/fac-snapshot-db/post/959_indexes_passthrough.sql diff --git a/backend/dissemination/sql/indexes/fac-snapshot-db/secondary_auditors.sql b/backend/dissemination/sql/fac-snapshot-db/post/960_indexes_secondary_auditors.sql similarity index 100% rename from backend/dissemination/sql/indexes/fac-snapshot-db/secondary_auditors.sql rename to backend/dissemination/sql/fac-snapshot-db/post/960_indexes_secondary_auditors.sql diff --git a/backend/dissemination/sql/api/finalize/post.sql b/backend/dissemination/sql/fac-snapshot-db/post/999_finalize.sql similarity index 100% rename from backend/dissemination/sql/api/finalize/post.sql rename to backend/dissemination/sql/fac-snapshot-db/post/999_finalize.sql diff --git a/backend/dissemination/sql/api/api_v1_0_3/pre.sql b/backend/dissemination/sql/fac-snapshot-db/pre/000_api_v1_0_3.sql similarity index 100% rename from backend/dissemination/sql/api/api_v1_0_3/pre.sql rename to backend/dissemination/sql/fac-snapshot-db/pre/000_api_v1_0_3.sql diff --git a/backend/dissemination/sql/api/api_v1_1_0/pre.sql b/backend/dissemination/sql/fac-snapshot-db/pre/010_api_v1_1_0.sql similarity index 100% rename from backend/dissemination/sql/api/api_v1_1_0/pre.sql rename to backend/dissemination/sql/fac-snapshot-db/pre/010_api_v1_1_0.sql diff --git a/backend/dissemination/sql/api/admin_api_v1_1_0/pre.sql b/backend/dissemination/sql/fac-snapshot-db/pre/020_admin_api_v1_1_0.sql similarity index 100% rename from backend/dissemination/sql/api/admin_api_v1_1_0/pre.sql rename to backend/dissemination/sql/fac-snapshot-db/pre/020_admin_api_v1_1_0.sql diff --git a/backend/dissemination/sql/api/admin_api_access_tables/pre.sql b/backend/dissemination/sql/fac-snapshot-db/pre/025_admin_api_access_tables.sql similarity index 100% rename from backend/dissemination/sql/api/admin_api_access_tables/pre.sql rename to backend/dissemination/sql/fac-snapshot-db/pre/025_admin_api_access_tables.sql diff --git a/backend/dissemination/sql/api/admin_api_v1_1_1/pre.sql b/backend/dissemination/sql/fac-snapshot-db/pre/030_admin_api_v1_1_1.sql similarity index 100% rename from backend/dissemination/sql/api/admin_api_v1_1_1/pre.sql rename to backend/dissemination/sql/fac-snapshot-db/pre/030_admin_api_v1_1_1.sql diff --git a/backend/dissemination/sql/api/public_api_v1_0_0/teardown.sql b/backend/dissemination/sql/fac-snapshot-db/pre/040_public_api_v1_0_0.sql similarity index 100% rename from backend/dissemination/sql/api/public_api_v1_0_0/teardown.sql rename to backend/dissemination/sql/fac-snapshot-db/pre/040_public_api_v1_0_0.sql diff --git a/backend/dissemination/sql/api/public_data_v1_0_0/teardown.sql b/backend/dissemination/sql/fac-snapshot-db/pre/050_public_data_v1_0_0.sql similarity index 100% rename from backend/dissemination/sql/api/public_data_v1_0_0/teardown.sql rename to backend/dissemination/sql/fac-snapshot-db/pre/050_public_data_v1_0_0.sql diff --git a/backend/dissemination/sql/api/schemas/teardown.sql b/backend/dissemination/sql/fac-snapshot-db/pre/060_schemas.sql similarity index 100% rename from backend/dissemination/sql/api/schemas/teardown.sql rename to backend/dissemination/sql/fac-snapshot-db/pre/060_schemas.sql diff --git a/backend/dissemination/sql/api/roles/teardown.sql b/backend/dissemination/sql/fac-snapshot-db/pre/070_schemas.sql similarity index 100% rename from backend/dissemination/sql/api/roles/teardown.sql rename to backend/dissemination/sql/fac-snapshot-db/pre/070_schemas.sql diff --git a/backend/dissemination/sql/api/permissions/teardown.sql b/backend/dissemination/sql/fac-snapshot-db/pre/080_permissions.sql similarity index 100% rename from backend/dissemination/sql/api/permissions/teardown.sql rename to backend/dissemination/sql/fac-snapshot-db/pre/080_permissions.sql diff --git a/backend/dissemination/sql/api/sequences/teardown.sql b/backend/dissemination/sql/fac-snapshot-db/pre/090_sequences.sql similarity index 100% rename from backend/dissemination/sql/api/sequences/teardown.sql rename to backend/dissemination/sql/fac-snapshot-db/pre/090_sequences.sql diff --git a/backend/dissemination/sql/api/finalize/pre.sql b/backend/dissemination/sql/fac-snapshot-db/pre/999_finalize.sql similarity index 100% rename from backend/dissemination/sql/api/finalize/pre.sql rename to backend/dissemination/sql/fac-snapshot-db/pre/999_finalize.sql diff --git a/backend/dissemination/sql/public.rest b/backend/dissemination/sql/public.rest new file mode 100644 index 0000000000..adf0246f88 --- /dev/null +++ b/backend/dissemination/sql/public.rest @@ -0,0 +1,53 @@ +### +GET {{scheme}}://{{apiUrl}}/general?limit=20000&offset=100000 +authorization: {{authorization}} +x-api-user-id: {{xApiUserId}} +accept-profile: {{acceptProfile}} +Accept: application/vnd.pgrst.plan + +### +GET {{scheme}}://{{apiUrl}}/general?report_id=eq.2021-12-CENSUS-0000250449 +authorization: {{authorization}} +x-api-user-id: {{xApiUserId}} +accept-profile: api_v1_1_0 +Accept: application/vnd.pgrst.plan + +### +GET {{scheme}}://{{apiUrl}}/general?report_id=eq.2021-12-CENSUS-0000250449 +authorization: {{authorization}} +x-api-user-id: {{xApiUserId}} +accept-profile: public_api_v1_0_0 +Accept: application/vnd.pgrst.plan + +### +GET {{scheme}}://{{apiUrl}}/federal_awards?batch_number=eq.200 +authorization: {{authorization}} +x-api-user-id: {{xApiUserId}} +accept-profile: public_api_v1_0_0 +Accept: application/vnd.pgrst.plan + + +### +GET {{scheme}}://{{apiUrl}}/federal_awards?batch_number=eq.200&limit=1 +authorization: {{authorization}} +x-api-user-id: {{xApiUserId}} +accept-profile: public_api_v1_0_0 + +### +GET {{scheme}}://{{apiUrl}}/combined?report_id=eq.2021-12-CENSUS-0000250449 +authorization: {{authorization}} +x-api-user-id: {{xApiUserId}} +accept-profile: public_api_v1_0_0 +Accept: application/vnd.pgrst.plan + +### +GET {{scheme}}://{{apiUrl}}/rpc/get_batch_federal_awards?_batch=3&limit=10 +authorization: {{authorization}} +x-api-user-id: {{xApiUserId}} +accept-profile: public_api_v1_0_0 + +### +GET {{scheme}}://{{apiUrl}}/metadata +authorization: {{authorization}} +x-api-user-id: {{xApiUserId}} +accept-profile: public_api_v1_0_0 diff --git a/backend/tools/sql_pre.sh b/backend/tools/sql_pre.sh deleted file mode 100644 index 0ee3046054..0000000000 --- a/backend/tools/sql_pre.sh +++ /dev/null @@ -1,14 +0,0 @@ -source tools/util_startup.sh - -function api_teardown { - startup_log "API_TEARDOWN" "BEGIN" - # Loop by index. - for index in "${!teardown_scripts[@]}" - do - local subdir="${teardown_scripts[index]}" - local path="dissemination/sql/api/${subdir}/teardown.sql" - run_sql $FAC_SNAPSHOT_URI $path - done - startup_log "API_TEARDOWN" "END" - return 0 -} diff --git a/backend/tools/sql_pre_post.sh b/backend/tools/sql_pre_post.sh new file mode 100644 index 0000000000..af01f14219 --- /dev/null +++ b/backend/tools/sql_pre_post.sh @@ -0,0 +1,37 @@ +source tools/util_startup.sh + +function run_sql_files { + local db="$1" + local tag="$2" + local pre_post="$3" + local which_db="NO_DB_SELECTED" + + if [ $db == $FAC_DB_URI ]; then + which_db="fac-db" + fi; + if [ $db == $FAC_SNAPSHOT_URI ]; then + which_db="fac-snapshot-db" + fi; + + startup_log $tag "BEGIN" + # Loop by index. + for file in `ls dissemination/sql/$which_db/$pre_post/*.sql`; + do + # run_sql has an explicit go/no-go built-in. + run_sql $db $file + done + startup_log $tag "END" + return 0 +} + + +function sql_pre { + run_sql_files $FAC_SNAPSHOT_URI "SQL_PRE" "pre" +} + +function sql_post { + run_sql_files $FAC_SNAPSHOT_URI "SQL_POST" "post" + # Vacuum things when we're done. + # Cannot run inside a transaction. + $PSQL_EXE_NO_TXN $FAC_SNAPSHOT_URI -c "VACUUM (FULL, VERBOSE, ANALYZE);" +} diff --git a/backend/tools/sql_post.sh b/backend/tools/sql_pre_post.sql similarity index 100% rename from backend/tools/sql_post.sh rename to backend/tools/sql_pre_post.sql From eb12badd6a14a2165b667d917948d7eee635f17b Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Thu, 10 Oct 2024 11:03:48 -0400 Subject: [PATCH 25/89] Forgot to delete a bunch of things... And, redeploying... --- backend/.profile | 35 +- .../sql/api/admin_api_v1_1_0/base.sql | 29 -- .../admin_api_v1_1_0/create_access_tables.sql | 43 -- .../api/admin_api_v1_1_0/create_functions.sql | 438 ------------------ .../api/admin_api_v1_1_0/create_schema.sql | 56 --- .../sql/api/admin_api_v1_1_0/create_views.sql | 92 ---- .../sql/api/admin_api_v1_1_0/drop_schema.sql | 9 - .../sql/api/admin_api_v1_1_0/drop_views.sql | 8 - .../sql/api/admin_api_v1_1_0/test.rest | 114 ----- .../sql/api/prep_snapshot_db/post.sql | 6 - .../sql/api/prep_snapshot_db/teardown.sql | 0 .../sql/api/public_api_v1_0_0/public.rest | 53 --- backend/dissemination/sql/api/roles/post.sql | 7 - .../dissemination/sql/api/schemas/post.sql | 0 .../dissemination/sql/api/sequences/post.sql | 1 - .../sql/create_materialized_views.sql | 231 --------- .../sql/drop_materialized_views.sql | 7 - .../post/954_indexes_federal_awards.sql | 3 + .../sql/fac-snapshot-db/post/999_finalize.sql | 1 + .../fac-snapshot-db/pre/000_api_v1_0_3.sql | 5 +- .../pre/030_admin_api_v1_1_1.sql | 5 - .../pre/040_public_api_v1_0_0.sql | 14 +- .../sql/refresh_materialized_views.sql | 3 - .../public_data_v1_0_0.yaml | 29 -- backend/docker-compose.yml | 2 + backend/run.sh | 28 +- backend/tools/setup_cgov_env.sh | 2 + backend/tools/setup_local_env.sh | 2 + backend/tools/sql_pre_post.sql | 15 - backend/tools/variables.sh | 104 ++--- backend/util/api_perf_test/perf.py | 140 +++++- 31 files changed, 210 insertions(+), 1272 deletions(-) delete mode 100644 backend/dissemination/sql/api/admin_api_v1_1_0/base.sql delete mode 100644 backend/dissemination/sql/api/admin_api_v1_1_0/create_access_tables.sql delete mode 100644 backend/dissemination/sql/api/admin_api_v1_1_0/create_functions.sql delete mode 100644 backend/dissemination/sql/api/admin_api_v1_1_0/create_schema.sql delete mode 100644 backend/dissemination/sql/api/admin_api_v1_1_0/create_views.sql delete mode 100644 backend/dissemination/sql/api/admin_api_v1_1_0/drop_schema.sql delete mode 100644 backend/dissemination/sql/api/admin_api_v1_1_0/drop_views.sql delete mode 100644 backend/dissemination/sql/api/admin_api_v1_1_0/test.rest delete mode 100644 backend/dissemination/sql/api/prep_snapshot_db/post.sql delete mode 100644 backend/dissemination/sql/api/prep_snapshot_db/teardown.sql delete mode 100644 backend/dissemination/sql/api/public_api_v1_0_0/public.rest delete mode 100644 backend/dissemination/sql/api/roles/post.sql delete mode 100644 backend/dissemination/sql/api/schemas/post.sql delete mode 100644 backend/dissemination/sql/api/sequences/post.sql delete mode 100644 backend/dissemination/sql/create_materialized_views.sql delete mode 100644 backend/dissemination/sql/drop_materialized_views.sql delete mode 100644 backend/dissemination/sql/refresh_materialized_views.sql delete mode 100644 backend/tools/sql_pre_post.sql diff --git a/backend/.profile b/backend/.profile index 7d445d725e..b92feba65c 100644 --- a/backend/.profile +++ b/backend/.profile @@ -9,11 +9,9 @@ source tools/util_startup.sh # This will choose the correct environment # for local envs (LOCAL or TESTING) and cloud.gov source tools/setup_env.sh -source tools/api_teardown.sh +source tools/sql_pre_post.sh source tools/sling_first_run.sh -source tools/build_indexes.sh source tools/migrate_app_tables.sh -source tools/api_standup.sh source tools/seed_cog_baseline.sh ##### @@ -24,11 +22,11 @@ gonogo "setup_env" if [[ "$CF_INSTANCE_INDEX" == 0 ]]; then ##### - # API TEARDOWN - # API has to be deprecated/removed before migration, because - # of tight coupling between schema/views and the dissemination tables - api_teardown - gonogo "api_teardown" + # SQL PRE + # We have SQL that we want to run before the migrations and sling are run. + # This tears down things that would conflict with migrations, etc. + sql_pre + gonogo "sql_pre" ##### # MIGRATE APP TABLES @@ -43,22 +41,11 @@ if [[ "$CF_INSTANCE_INDEX" == 0 ]]; then gonogo "sling_first_run" ##### - # API STANDUP - # Standup the API, which may depend on migration changes - api_standup - gonogo "api_standup" - - ##### - # BUILD INDEXES - # Builds indexes on the API tables in fac-snapshot-db - build_indexes - gonogo "build_indexes" - - ##### - # COLLECT STATIC - # Do Django things with static files. - run_collectstatic - gonogo "run_collectstatic" + # SQL POST + # Rebuild the API and prepare the system for execution. + # Runs after migrations. + sql_post + gonogo "sql_post" ##### # SEED COG/OVER TABLES diff --git a/backend/dissemination/sql/api/admin_api_v1_1_0/base.sql b/backend/dissemination/sql/api/admin_api_v1_1_0/base.sql deleted file mode 100644 index dedabe0cb7..0000000000 --- a/backend/dissemination/sql/api/admin_api_v1_1_0/base.sql +++ /dev/null @@ -1,29 +0,0 @@ -DO -$do$ -BEGIN - IF EXISTS ( - SELECT FROM pg_catalog.pg_roles - WHERE rolname = 'authenticator') THEN - RAISE NOTICE 'Role "authenticator" already exists. Skipping.'; - ELSE - CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; - END IF; -END -$do$; - -DO -$do$ -BEGIN - IF EXISTS ( - SELECT FROM pg_catalog.pg_roles - WHERE rolname = 'api_fac_gov') THEN - RAISE NOTICE 'Role "api_fac_gov" already exists. Skipping.'; - ELSE - CREATE ROLE api_fac_gov NOLOGIN; - END IF; -END -$do$; - -GRANT api_fac_gov TO authenticator; - -NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/api/admin_api_v1_1_0/create_access_tables.sql b/backend/dissemination/sql/api/admin_api_v1_1_0/create_access_tables.sql deleted file mode 100644 index 18db91956d..0000000000 --- a/backend/dissemination/sql/api/admin_api_v1_1_0/create_access_tables.sql +++ /dev/null @@ -1,43 +0,0 @@ --- This is explicitly not a Django managed table. --- In order to have an administrative key added, --- it must be added via a Github commit, and a PR --- must be performed to merge the key into the tree. - --- This is because administrative keys can read/write --- to some tables in the database. They can read internal and --- in-flight data. - -DROP TABLE IF EXISTS support_administrative_key_uuids; - -CREATE TABLE support_administrative_key_uuids - ( - id BIGSERIAL PRIMARY KEY, - email TEXT, - uuid TEXT, - permissions TEXT, - added DATE - ); - -INSERT INTO support_administrative_key_uuids - (email, uuid, permissions, added) - VALUES - ( - 'matthew.jadud@gsa.gov', - '61ba59b2-f545-4c2f-9b24-9655c706a06c', - 'CREATE,READ,DELETE', - '2023-12-04' - ), - ( - 'daniel.swick@gsa.gov', - 'b6e08808-ecb2-4b6a-b928-46d4205497ff', - 'CREATE,READ,DELETE', - '2023-12-08' - ), - ( - 'fac-gov-test-users+api-tester-admin@gsa.gov', - 'dd60c3f9-053d-4d82-a309-c89da53559f4', - 'CREATE,READ,DELETE', - '2024-07-10' - ) - ; - diff --git a/backend/dissemination/sql/api/admin_api_v1_1_0/create_functions.sql b/backend/dissemination/sql/api/admin_api_v1_1_0/create_functions.sql deleted file mode 100644 index b89b001186..0000000000 --- a/backend/dissemination/sql/api/admin_api_v1_1_0/create_functions.sql +++ /dev/null @@ -1,438 +0,0 @@ --- WARNING --- Under PostgreSQL 12, the functions below work. --- Under PostgreSQL 14, these will break. --- --- Note the differences: --- --- raise info 'Works under PostgreSQL 12'; --- raise info 'request.header.x-magic %', (SELECT current_setting('request.header.x-magic', true)); --- raise info 'request.jwt.claim.expires %', (SELECT current_setting('request.jwt.claim.expires', true)); --- raise info 'Works under PostgreSQL 14'; --- raise info 'request.headers::json->>x-magic %', (SELECT current_setting('request.headers', true)::json->>'x-magic'); --- raise info 'request.jwt.claims::json->expires %', (SELECT current_setting('request.jwt.claims', true)::json->>'expires'); --- --- To quote the work of Dav Pilkey, "remember this now." - -begin; - - -CREATE OR REPLACE FUNCTION admin_api_v1_1_0_functions.get_header(item text) RETURNS text - AS $get_header$ - declare res text; - begin - SELECT (current_setting('request.headers', true)::json)->>item into res; - return res; - end; -$get_header$ LANGUAGE plpgsql; - -create or replace function admin_api_v1_1_0_functions.get_api_key_uuid() returns TEXT -as $gaku$ -declare uuid text; -begin - select admin_api_v1_1_0_functions.get_header('x-api-user-id') into uuid; - return uuid; -end; -$gaku$ LANGUAGE plpgsql; - --- log_api_event --- Maintain an internal table of administrative API events. --- Also RAISE INFO so that NR gets a copy. -create or replace function admin_api_v1_1_0_functions.log_admin_api_event(event TEXT, meta JSON) -returns boolean -as $log_admin_api_event$ -DECLARE - uuid_header text; -BEGIN - SELECT admin_api_v1_1_0_functions.get_api_key_uuid() INTO uuid_header; - - INSERT INTO public.support_adminapievent - (api_key_uuid, event, event_data, "timestamp") - VALUES (uuid_header, event, meta, NOW()); - - RAISE INFO 'ADMIN_API % % %', uuid_header, event, meta; - RETURN 1; -END; -$log_admin_api_event$ LANGUAGE plpgsql; - - --- has_admin_data_access :: permission -> bool --- The permissions (insert, select, delete) allow us to have users who can --- read administrative data in addition to users who can (say) update --- select tables like the tribal access lists. -create or replace function admin_api_v1_1_0_functions.has_admin_data_access(perm TEXT) returns boolean -as $has_admin_data_access$ -DECLARE - uuid_header text; - key_exists boolean; - has_permission boolean; -BEGIN - SELECT admin_api_v1_1_0_functions.get_api_key_uuid() INTO uuid_header; - - SELECT - CASE WHEN EXISTS ( - SELECT uuid - FROM public.support_administrative_key_uuids aku - WHERE aku.uuid = uuid_header) - THEN 1::BOOLEAN - ELSE 0::BOOLEAN - END - INTO key_exists; - - SELECT - CASE WHEN EXISTS ( - SELECT permissions - FROM public.support_administrative_key_uuids aku - WHERE aku.uuid = uuid_header - AND aku.permissions like '%' || perm || '%') - THEN 1::BOOLEAN - ELSE 0::BOOLEAN - END - INTO has_permission; - - -- This log event is an INSERT. When called from a VIEW (a SELECT-only context), - -- a call to log_admin_api_event() fails. So, we'll RAISE INFO right here, so we can - -- see the resultse of access checks in the log. We might later comment this out if - -- it becomes too noisy. - RAISE INFO 'ADMIN_API has_access_check % % %', uuid_header, key_exists, has_permission; - - RETURN key_exists AND has_permission; -END; -$has_admin_data_access$ LANGUAGE plpgsql; - --- Takes an email address and, if that address is not in the access table, --- inserts it. If the address already exists, the insert is skipped. --- --- ### Example from REST client --- POST http://localhost:3000/rpc/add_tribal_access_email --- authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} --- content-profile: admin_api_v1_1_0 --- content-type: application/json --- Prefer: params=single-object --- // Not actually a key UUID. --- X-Api-User-Id: 18ef0e72-8976-11ee-ad35-3f80b454d3cc --- { --- "email": "darth.vader@deathstar.org" --- } -create or replace function admin_api_v1_1_0.add_tribal_access_email(params JSON) -returns BOOLEAN -as $add_tribal_access_email$ -DECLARE - already_exists INTEGER; - read_tribal_id INTEGER; -BEGIN - -- If the API user has insert permissions, give it a go - IF admin_api_v1_1_0_functions.has_admin_data_access('CREATE') - THEN - -- Are they already in the table? - SELECT count(up.email) - FROM public.users_userpermission as up - WHERE LOWER(email) = LOWER(params->>'email') INTO already_exists; - - -- If they are, we're going to exit. - IF already_exists <> 0 - THEN - RETURN 0; - END IF; - - -- Grab the permission ID that we need for the insert below. - -- We want the 'read-tribal' permission, which has a human-readable - -- slug. But, we need it's ID, because that is the PK. - SELECT up.id INTO read_tribal_id - FROM public.users_permission AS up - WHERE up.slug = 'read-tribal'; - - IF already_exists = 0 - THEN - -- Can we make the 1 not magic... do a select into. - INSERT INTO public.users_userpermission - (email, permission_id, user_id) - VALUES (LOWER(params->>'email'), read_tribal_id, null); - - RAISE INFO 'ADMIN_API add_tribal_access_email OK %', LOWER(params->>'email'); - RETURN admin_api_v1_1_0_functions.log_admin_api_event('tribal-access-email-added', - json_build_object('email', LOWER(params->>'email'))); - END IF; - ELSE - RETURN 0; - END IF; -end; -$add_tribal_access_email$ LANGUAGE plpgsql; - --- Adds many email addresses. Calls `add_tribal_access_email` for each address. --- --- ### Example from REST client --- POST http://localhost:3000/rpc/add_tribal_access_emails --- authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} --- content-profile: admin_api_v1_1_0 --- content-type: application/json --- Prefer: params=single-object --- // Not actually a key UUID. --- X-Api-User-Id: 18ef0e72-8976-11ee-ad35-3f80b454d3cc --- { --- "emails": [ --- "darth.vader@deathstar.org", --- "bob.darth.vader@deathstar.org", --- "darthy.vader@deathstar.org", --- "bob@deathstar.org" --- ] --- } -CREATE OR REPLACE FUNCTION admin_api_v1_1_0.add_tribal_access_emails(params JSON) -returns BOOLEAN -as $add_tribal_access_emails$ -DECLARE - ele TEXT; - em record; -BEGIN - IF admin_api_v1_1_0_functions.has_admin_data_access('CREATE') - THEN - -- This is a FOR loop over a JSON array in plPgSQL - FOR em IN (SELECT json_array_elements_text((params->>'emails')::JSON) ele) - LOOP - -- PERFORM is how to execute code that does not return anything. - -- If a SELECT was used here, the SQL compiler would complain. - PERFORM admin_api_v1_1_0.add_tribal_access_email(json_build_object('email', em.ele)::JSON); - END LOOP; - RETURN 1; - END IF; - RETURN 0; -END; -$add_tribal_access_emails$ LANGUAGE plpgsql; - --- Removes the email. Will remove multiple rows. That shouldn't happen, but still. --- --- ### Example from REST client --- POST http://localhost:3000/rpc/remove_tribal_access_email --- authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} --- content-profile: admin_api_v1_1_0 --- content-type: application/json --- Prefer: params=single-object --- // Not actually a key UUID. --- X-Api-User-Id: 18ef0e72-8976-11ee-ad35-3f80b454d3cc --- { --- "email": "darth.vader@deathstar.org" --- } -CREATE OR REPLACE FUNCTION admin_api_v1_1_0.remove_tribal_access_email(params JSON) -returns BOOLEAN -as $remove_tribal_access_email$ -DECLARE - affected_rows INTEGER; -BEGIN - - IF admin_api_v1_1_0_functions.has_admin_data_access('DELETE') - THEN - -- Delete rows where the email address matches - DELETE FROM public.users_userpermission as up - WHERE LOWER(up.email) = LOWER(params->>'email'); - -- This is the Postgres way to find out how many rows - -- were affected by a DELETE. - GET DIAGNOSTICS affected_rows = ROW_COUNT; - -- If that is greater than zero, we were successful. - IF affected_rows > 0 - THEN - RETURN admin_api_v1_1_0_functions.log_admin_api_event('tribal-access-email-removed', - json_build_object('email', LOWER(params->>'email'))); - ELSE - RETURN 0; - END IF; - ELSE - -- If we did not have permission, consider it a failure. - RETURN 0; - END IF; -end; -$remove_tribal_access_email$ LANGUAGE plpgsql; - --- Removes many email addresses. Calls `remove_tribal_access_email` for each address. --- --- ### Example from REST client --- POST http://localhost:3000/rpc/remove_tribal_access_emails --- authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} --- content-profile: admin_api_v1_1_0 --- content-type: application/json --- Prefer: params=single-object --- // Not actually a key UUID. --- X-Api-User-Id: 18ef0e72-8976-11ee-ad35-3f80b454d3cc --- { --- "emails": [ --- "darth.vader@deathstar.org", --- "bob.darth.vader@deathstar.org", --- "darthy.vader@deathstar.org", --- "bob@deathstar.org" --- ] --- } -CREATE OR REPLACE FUNCTION admin_api_v1_1_0.remove_tribal_access_emails(params JSON) -returns BOOLEAN -as $remove_tribal_access_emails$ -DECLARE - ele TEXT; - em record; -BEGIN - IF admin_api_v1_1_0_functions.has_admin_data_access('DELETE') - THEN - FOR em IN (SELECT json_array_elements_text((params->>'emails')::JSON) ele) - LOOP - PERFORM admin_api_v1_1_0.remove_tribal_access_email(json_build_object('email', em.ele)::JSON); - END LOOP; - RETURN 1; - END IF; - RETURN 0; -END; -$remove_tribal_access_emails$ LANGUAGE plpgsql; - - - - ---The function below add_tribal_api_key_access adds read access to a tribal API for a specified email. ---It checks if the API user has read permissions. ---Otherwise, it adds the email with 'read-tribal' permission, logs the event, and returns true. - -CREATE OR REPLACE FUNCTION admin_api_v1_1_0.add_tribal_api_key_access(params JSON) -RETURNS JSON -AS $add_tribal_api_key_access$ -DECLARE - user_exists BOOLEAN; -BEGIN - -- If the API user has create permissions, we can proceed - IF admin_api_v1_1_0_functions.has_admin_data_access('CREATE') THEN - -- Check if the user with the given email - SELECT EXISTS ( - SELECT 1 - FROM public.dissemination_TribalApiAccessKeyIds - WHERE LOWER(email) = LOWER(params->>'email') - ) - INTO user_exists; - - -- If the user already exists, it means they have access. - -- For purposes of this function, lets call that "succses", and return true. - IF user_exists THEN - RAISE INFO 'ADMIN_API add_tribal_api_key_access ALREADY_EXISTS %', LOWER(params->>'email'); - RETURN json_build_object( - 'result', 'success', - 'message', 'User with this key already exists')::JSON; - - END IF; - - -- If the user does not exist, add a new record - INSERT INTO public.dissemination_TribalApiAccessKeyIds (email, key_id, date_added) - VALUES (LOWER(params->>'email'), params->>'key_id', CURRENT_TIMESTAMP); - RAISE INFO 'ADMIN_API add_tribal_api_key_access ACCESS_GRANTED % %', LOWER(params->>'email'), params->>'key_id'; - RETURN json_build_object( - 'result', 'success', - 'message', 'User access granted')::JSON; - ELSE - -- If the user does not have CREATE permissions, then we should return a message to that effect. - -- It is a permissions error, but still, we need to know this failed. - RAISE INFO 'ADMIN_API add_tribal_api_key_access ADMIN_LACKS_CREATE'; - RETURN json_build_object( - 'result', 'failure', - 'message', 'Admin user lacks CREATE permissions')::JSON; - END IF; - - -- Return false by default. - RAISE INFO 'ADMIN_API add_tribal_api_key_access WAT %', LOWER(params->>'email'); - RETURN json_build_object( - 'result', 'failure', - 'message', 'Unknown error in access addition')::JSON; -END; -$add_tribal_api_key_access$ LANGUAGE plpgsql; - --- The function below removes tribal API key access for a specified email. --- It checks if the API user has read permissions. --- If the email exists in the database with 'read-tribal' permission, it removes the entry, logs the removal event, and returns true. --- If the email doesn't exist or the user lacks proper permissions, the function returns false. - -CREATE OR REPLACE FUNCTION admin_api_v1_1_0.remove_tribal_api_key_access(params JSON) -RETURNS JSON -AS $remove_tribal_api_key_access$ -DECLARE - user_exists BOOLEAN; -BEGIN - -- If the API user has read permissions, give it a go - IF admin_api_v1_1_0_functions.has_admin_data_access('DELETE') THEN - -- Check if the user with the given email exists - SELECT EXISTS ( - SELECT 1 - FROM public.dissemination_TribalApiAccessKeyIds - WHERE LOWER(email) = LOWER(params->>'email') - ) - INTO user_exists; - - -- If the user exists, remove the record - IF user_exists THEN - DELETE FROM public.dissemination_TribalApiAccessKeyIds - WHERE LOWER(email) = LOWER(params->>'email'); - RAISE INFO 'ADMIN_API remove_tribal_api_key_access ACCESS_REMOVED %', LOWER(params->>'email'); - RETURN json_build_object( - 'result', 'success', - 'message', 'Removed record')::JSON; - ELSE - RAISE INFO 'ADMIN_API remove_tribal_api_key_access DID_NOT_EXIST %', LOWER(params->>'email'); - RETURN json_build_object( - 'result', 'failure', - 'message', 'User did not exist in table')::JSON; - END IF; - ELSE - RAISE INFO 'ADMIN_API remove_tribal_api_key_access ADMIN_LACKS_DELETE'; - RETURN json_build_object( - 'result', 'failure', - 'message', 'Admin user lacks DELETE permissions')::JSON; -- Return false if the API user doesn't have read permissions - END IF; - RAISE INFO 'ADMIN_API add_tribal_api_key_access WAT %', LOWER(params->>'email'); - RETURN json_build_object( - 'result', 'failure', - 'message', 'Uknown error in access removal')::JSON; -END; -$remove_tribal_api_key_access$ LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION admin_api_v1_1_0.request_file_access( - report_id TEXT -) RETURNS JSON LANGUAGE plpgsql AS -$$ -DECLARE - v_uuid_header TEXT; - v_access_uuid VARCHAR(200); - v_key_exists BOOLEAN; - v_key_added_date DATE; -BEGIN - - SELECT admin_api_v1_1_0_functions.get_api_key_uuid() INTO v_uuid_header; - - -- Check if the provided API key exists in public.dissemination_TribalApiAccessKeyIds - SELECT - EXISTS( - SELECT 1 - FROM public.dissemination_TribalApiAccessKeyIds - WHERE key_id = v_uuid_header - ) INTO v_key_exists; - - - -- Get the added date of the key from public.dissemination_TribalApiAccessKeyIds - SELECT date_added - INTO v_key_added_date - FROM public.dissemination_TribalApiAccessKeyIds - WHERE key_id = v_uuid_header; - - - -- Check if the key is less than 6 months old - IF v_uuid_header IS NOT NULL AND v_key_exists AND v_key_added_date >= CURRENT_DATE - INTERVAL '6 months' THEN - -- Generate UUID (using PostgreSQL's gen_random_uuid function) - SELECT gen_random_uuid() INTO v_access_uuid; - - -- Inserting data into the one_time_access table - INSERT INTO public.dissemination_onetimeaccess (uuid, api_key_id, timestamp, report_id) - VALUES (v_access_uuid::UUID, v_uuid_header, CURRENT_TIMESTAMP, report_id); - - -- Return the UUID to the user - RETURN json_build_object('access_uuid', v_access_uuid); - ELSE - -- Return an error for unauthorized access - RETURN json_build_object('error', 'Unauthorized access or key older than 6 months')::JSON; - END IF; -END; -$$; - - - -commit; - -NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/api/admin_api_v1_1_0/create_schema.sql b/backend/dissemination/sql/api/admin_api_v1_1_0/create_schema.sql deleted file mode 100644 index 25f913689d..0000000000 --- a/backend/dissemination/sql/api/admin_api_v1_1_0/create_schema.sql +++ /dev/null @@ -1,56 +0,0 @@ -begin; - -do -$$ -begin - DROP SCHEMA IF EXISTS admin_api_v1_1_0 CASCADE; - DROP SCHEMA IF EXISTS admin_api_v1_1_0_functions CASCADE; - - if not exists (select schema_name from information_schema.schemata where schema_name = 'admin_api_v1_1_0') then - create schema admin_api_v1_1_0; - create schema admin_api_v1_1_0_functions; - - grant usage on schema admin_api_v1_1_0_functions to api_fac_gov; - - -- Grant access to tables and views - alter default privileges - in schema admin_api_v1_1_0 - grant select - -- this includes views - on tables - to api_fac_gov; - - -- Grant access to sequences, if we have them - grant usage on schema admin_api_v1_1_0 to api_fac_gov; - grant select, usage on all sequences in schema admin_api_v1_1_0 to api_fac_gov; - alter default privileges - in schema admin_api_v1_1_0 - grant select, usage - on sequences - to api_fac_gov; - - -- The admin API needs to be able to write user permissions. - -- This is so we can add and remove people who will have tribal data access - -- via the administrative API. - GRANT INSERT, SELECT, DELETE on public.users_userpermission to api_fac_gov; - -- We need to be able to look up slugs and turn them into permission IDs. - GRANT SELECT on public.users_permission to api_fac_gov; - -- It also needs to be able to log events. - GRANT INSERT on public.support_adminapievent to api_fac_gov; - -- And, it wants to read the UUIDs of administrative keys - GRANT SELECT ON public.support_administrative_key_uuids TO api_fac_gov; - -- We want to see data in flight as admins. - GRANT SELECT ON public.audit_singleauditchecklist TO api_fac_gov; - - GRANT INSERT, SELECT, DELETE on public.dissemination_tribalapiaccesskeyids to api_fac_gov; - GRANT INSERT on public.dissemination_onetimeaccess to api_fac_gov; - end if; -end -$$ -; - -commit; - -notify pgrst, - 'reload schema'; - diff --git a/backend/dissemination/sql/api/admin_api_v1_1_0/create_views.sql b/backend/dissemination/sql/api/admin_api_v1_1_0/create_views.sql deleted file mode 100644 index 04b747662f..0000000000 --- a/backend/dissemination/sql/api/admin_api_v1_1_0/create_views.sql +++ /dev/null @@ -1,92 +0,0 @@ - -begin; - - ---------------------------------------- --- accesses ---------------------------------------- --- public.audit_access definition - --- Drop table - --- DROP TABLE public.audit_access; - -CREATE OR REPLACE VIEW admin_api_v1_1_0.audit_access AS - SELECT - aa.role, - aa.fullname, - aa.email, - aa.sac_id, - aa.user_id - FROM - public.audit_access aa - WHERE - admin_api_v1_1_0_functions.has_admin_data_access('READ') - ORDER BY aa.id -; - -CREATE OR REPLACE VIEW admin_api_v1_1_0.singleauditchecklist AS - SELECT - sac.id, - sac.date_created, - sac.submission_status, - sac.data_source, - sac.transition_name, - sac.transition_date, - sac.report_id, - sac.audit_type, - sac.general_information, - sac.audit_information, - sac.federal_awards, - sac.corrective_action_plan, - sac.findings_text, - sac.findings_uniform_guidance, - sac.additional_ueis, - sac.additional_eins, - sac.secondary_auditors, - sac.notes_to_sefa, - sac.auditor_certification, - sac.auditee_certification, - sac.tribal_data_consent, - sac.cognizant_agency, - sac.oversight_agency, - sac.submitted_by_id - from - public.audit_singleauditchecklist sac - where - admin_api_v1_1_0_functions.has_admin_data_access('READ') - order by sac.id -; - -CREATE OR REPLACE VIEW admin_api_v1_1_0.tribal_access AS - SELECT - uup.email, - up.slug as permission - FROM - users_userpermission uup, - users_permission up - WHERE - (uup.permission_id = up.id) - AND (up.slug = 'read-tribal') - AND admin_api_v1_1_0_functions.has_admin_data_access('READ') - ORDER BY uup.id -; - -CREATE OR REPLACE VIEW admin_api_v1_1_0.admin_api_events AS - SELECT - ae.timestamp, - ae.api_key_uuid, - ae.event, - ae.event_data - FROM - public.support_adminapievent ae - WHERE - admin_api_v1_1_0_functions.has_admin_data_access('READ') - ORDER BY ae.id -; - - -commit; - -notify pgrst, - 'reload schema'; diff --git a/backend/dissemination/sql/api/admin_api_v1_1_0/drop_schema.sql b/backend/dissemination/sql/api/admin_api_v1_1_0/drop_schema.sql deleted file mode 100644 index b0d4cc1109..0000000000 --- a/backend/dissemination/sql/api/admin_api_v1_1_0/drop_schema.sql +++ /dev/null @@ -1,9 +0,0 @@ - -begin; - -DROP SCHEMA IF EXISTS admin_api_v1_1_0 CASCADE; - -commit; - -notify pgrst, - 'reload schema'; diff --git a/backend/dissemination/sql/api/admin_api_v1_1_0/drop_views.sql b/backend/dissemination/sql/api/admin_api_v1_1_0/drop_views.sql deleted file mode 100644 index 41236e55d7..0000000000 --- a/backend/dissemination/sql/api/admin_api_v1_1_0/drop_views.sql +++ /dev/null @@ -1,8 +0,0 @@ -begin; - - drop view if exists admin_api_v1_1_0.audit_access; - -commit; - -notify pgrst, - 'reload schema'; diff --git a/backend/dissemination/sql/api/admin_api_v1_1_0/test.rest b/backend/dissemination/sql/api/admin_api_v1_1_0/test.rest deleted file mode 100644 index 16e6ab054c..0000000000 --- a/backend/dissemination/sql/api/admin_api_v1_1_0/test.rest +++ /dev/null @@ -1,114 +0,0 @@ -@api_user_id = 61ba59b2-f545-4c2f-9b24-9655c706a06c -# @api_user_id = 1e2845a0-c844-4a6f-84ac-f398b58ce7c9 -@scheme = http -@api_url = localhost:3000 -# @scheme = https -# @api_url = api-preview.fac.gov - -### -GET {{scheme}}://{{api_url}}/general?limit=1 -// Have to use the JWT when testing locally. -authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} -x-api-key: {{$processEnv API_GOV_KEY}} - - -### -GET {{scheme}}://{{api_url}}/audit_access -// Have to use the JWT and API UUID when testing locally. -authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} -x-api-user-id: {{api_user_id}} -x-api-key: {{$processEnv API_GOV_KEY}} -accept-profile: admin_api_v1_1_0 - -### -GET {{scheme}}://{{api_url}}/singleauditchecklist -authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} -x-api-user-id: {{api_user_id}} -accept-profile: admin_api_v1_1_0 -x-api-key: {{$processEnv API_GOV_KEY}} - -### -// This should list everything using a view. -GET {{scheme}}://{{api_url}}/tribal_access -authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} -x-api-user-id: {{api_user_id}} -accept-profile: admin_api_v1_1_0 -x-api-key: {{$processEnv API_GOV_KEY}} - - -### -// Call the RPC to add access -POST {{scheme}}://{{api_url}}/rpc/add_tribal_access_email -authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} -x-api-user-id: {{api_user_id}} -content-profile: admin_api_v1_1_0 -content-type: application/json -Prefer: params=single-object -x-api-key: {{$processEnv API_GOV_KEY}} - -{ - "email": "darth.vader@deathstar.gsa.gov" -} - -### -// Call the RPC to remove access -POST {{scheme}}://{{api_url}}/rpc/add_tribal_access_emails -authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} -x-api-user-id: {{api_user_id}} -content-profile: admin_api_v1_1_0 -content-type: application/json -Prefer: params=single-object -x-api-key: {{$processEnv API_GOV_KEY}} - -{ - "emails": [ - "darth.vader@deathstar.gsa.gov", - "bob.darth.vader@deathstar.gsa.gov", - "darthy.vader@deathstar.gsa.gov", - "bob@deathstar.gsa.gov" - ] -} - -### -// Call the RPC to remove access -POST {{scheme}}://{{api_url}}/rpc/remove_tribal_access_email -authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} -x-api-user-id: {{api_user_id}} -content-profile: admin_api_v1_1_0 -content-type: application/json -Prefer: params=single-object -x-api-key: {{$processEnv API_GOV_KEY}} - -{ - "email": "darth.vader@deathstar.gsa.gov" -} - - -### -// Call the RPC to remove access -POST {{scheme}}://{{api_url}}/rpc/remove_tribal_access_emails -authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} -x-api-user-id: {{api_user_id}} -content-profile: admin_api_v1_1_0 -content-type: application/json -Prefer: params=single-object -x-api-key: {{$processEnv API_GOV_KEY}} - -{ - "emails": [ - "darth.vader@deathstar.gsa.gov", - "bob.darth.vader@deathstar.gsa.gov", - "darthy.vader@deathstar.gsa.gov", - "bob@deathstar.gsa.gov" - ] -} - - -### -// This should list everything using a view. -GET {{scheme}}://{{api_url}}/admin_api_events -authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} -x-api-user-id: {{api_user_id}} -accept-profile: admin_api_v1_1_0 -x-api-key: {{$processEnv API_GOV_KEY}} - diff --git a/backend/dissemination/sql/api/prep_snapshot_db/post.sql b/backend/dissemination/sql/api/prep_snapshot_db/post.sql deleted file mode 100644 index 26f6fb448e..0000000000 --- a/backend/dissemination/sql/api/prep_snapshot_db/post.sql +++ /dev/null @@ -1,6 +0,0 @@ -DO LANGUAGE plpgsql -$NOTE$ - BEGIN - RAISE info 'PREP_SNAPSHOT_DB standup no-op.'; - END; -$NOTE$; diff --git a/backend/dissemination/sql/api/prep_snapshot_db/teardown.sql b/backend/dissemination/sql/api/prep_snapshot_db/teardown.sql deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/backend/dissemination/sql/api/public_api_v1_0_0/public.rest b/backend/dissemination/sql/api/public_api_v1_0_0/public.rest deleted file mode 100644 index adf0246f88..0000000000 --- a/backend/dissemination/sql/api/public_api_v1_0_0/public.rest +++ /dev/null @@ -1,53 +0,0 @@ -### -GET {{scheme}}://{{apiUrl}}/general?limit=20000&offset=100000 -authorization: {{authorization}} -x-api-user-id: {{xApiUserId}} -accept-profile: {{acceptProfile}} -Accept: application/vnd.pgrst.plan - -### -GET {{scheme}}://{{apiUrl}}/general?report_id=eq.2021-12-CENSUS-0000250449 -authorization: {{authorization}} -x-api-user-id: {{xApiUserId}} -accept-profile: api_v1_1_0 -Accept: application/vnd.pgrst.plan - -### -GET {{scheme}}://{{apiUrl}}/general?report_id=eq.2021-12-CENSUS-0000250449 -authorization: {{authorization}} -x-api-user-id: {{xApiUserId}} -accept-profile: public_api_v1_0_0 -Accept: application/vnd.pgrst.plan - -### -GET {{scheme}}://{{apiUrl}}/federal_awards?batch_number=eq.200 -authorization: {{authorization}} -x-api-user-id: {{xApiUserId}} -accept-profile: public_api_v1_0_0 -Accept: application/vnd.pgrst.plan - - -### -GET {{scheme}}://{{apiUrl}}/federal_awards?batch_number=eq.200&limit=1 -authorization: {{authorization}} -x-api-user-id: {{xApiUserId}} -accept-profile: public_api_v1_0_0 - -### -GET {{scheme}}://{{apiUrl}}/combined?report_id=eq.2021-12-CENSUS-0000250449 -authorization: {{authorization}} -x-api-user-id: {{xApiUserId}} -accept-profile: public_api_v1_0_0 -Accept: application/vnd.pgrst.plan - -### -GET {{scheme}}://{{apiUrl}}/rpc/get_batch_federal_awards?_batch=3&limit=10 -authorization: {{authorization}} -x-api-user-id: {{xApiUserId}} -accept-profile: public_api_v1_0_0 - -### -GET {{scheme}}://{{apiUrl}}/metadata -authorization: {{authorization}} -x-api-user-id: {{xApiUserId}} -accept-profile: public_api_v1_0_0 diff --git a/backend/dissemination/sql/api/roles/post.sql b/backend/dissemination/sql/api/roles/post.sql deleted file mode 100644 index 3ab5e55e39..0000000000 --- a/backend/dissemination/sql/api/roles/post.sql +++ /dev/null @@ -1,7 +0,0 @@ -DO LANGUAGE plpgsql -$NOTE$ - BEGIN - RAISE info 'ROLES teardown no-op.'; - END; -$NOTE$; - diff --git a/backend/dissemination/sql/api/schemas/post.sql b/backend/dissemination/sql/api/schemas/post.sql deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/backend/dissemination/sql/api/sequences/post.sql b/backend/dissemination/sql/api/sequences/post.sql deleted file mode 100644 index 224514cb5b..0000000000 --- a/backend/dissemination/sql/api/sequences/post.sql +++ /dev/null @@ -1 +0,0 @@ -RAISE NOTICE 'SEQUENCES standup no-op.'; diff --git a/backend/dissemination/sql/create_materialized_views.sql b/backend/dissemination/sql/create_materialized_views.sql deleted file mode 100644 index 36844cc0c0..0000000000 --- a/backend/dissemination/sql/create_materialized_views.sql +++ /dev/null @@ -1,231 +0,0 @@ -CREATE SEQUENCE IF NOT EXISTS dissemination_combined_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE; - ------------------------ --- dissemination_combined --- This table is used primarily by search. -CREATE MATERIALIZED VIEW IF NOT EXISTS - dissemination_combined_temp AS - SELECT - nextval('dissemination_combined_id_seq') AS id, - dg.report_id, - dfa.award_reference, - df.reference_number, - -- Build a composite ALN in case we want/need it - concat(dfa.federal_agency_prefix,'.',dfa.federal_award_extension) as aln, - -- All of diss_general as of 20240313 - dg.agencies_with_prior_findings, - dg.audit_period_covered, - dg.audit_type, - dg.audit_year, - dg.auditee_address_line_1, - dg.auditee_certified_date, - dg.auditee_certify_name, - dg.auditee_certify_title, - dg.auditee_city, - dg.auditee_contact_name, - dg.auditee_contact_title, - dg.auditee_ein, - dg.auditee_email, - dg.auditee_name, - dg.auditee_phone, - dg.auditee_state, - dg.auditee_uei, - dg.auditee_zip, - dg.auditor_address_line_1, - dg.auditor_certified_date, - dg.auditor_certify_name, - dg.auditor_certify_title, - dg.auditor_city, - dg.auditor_contact_name, - dg.auditor_contact_title, - dg.auditor_country, - dg.auditor_ein, - dg.auditor_email, - dg.auditor_firm_name, - dg.auditor_foreign_address, - dg.auditor_phone, - dg.auditor_state, - dg.auditor_zip, - dg.cognizant_agency, - dg.data_source, - dg.date_created, - dg.dollar_threshold, - dg.entity_type, - dg.fac_accepted_date, - dg.fy_end_date, - dg.fy_start_date, - dg.gaap_results, - dg.is_additional_ueis, - dg.is_aicpa_audit_guide_included, - dg.is_going_concern_included, - dg.is_internal_control_deficiency_disclosed, - dg.is_internal_control_material_weakness_disclosed, - dg.is_low_risk_auditee, - dg.is_material_noncompliance_disclosed, - dg.is_public, - dg.is_sp_framework_required, - dg.number_months, - dg.oversight_agency, - dg.ready_for_certification_date, - dg.sp_framework_basis, - dg.sp_framework_opinions, - dg.submitted_date, - dg.total_amount_expended, - dg.type_audit_code, - -- All of diss_federalaward - dfa.additional_award_identification, - dfa.amount_expended, - dfa.cluster_name, - dfa.cluster_total, - dfa.federal_agency_prefix, - dfa.federal_award_extension, - dfa.federal_program_name, - dfa.federal_program_total, - dfa.findings_count, - dfa.is_direct, - dfa.is_loan, - dfa.is_major, - dfa.is_passthrough_award, - dfa.loan_balance, - dfa.audit_report_type, - dfa.other_cluster_name, - dfa.passthrough_amount, - dfa.state_cluster_name, - -- All of diss_finding - df.is_material_weakness, - df.is_modified_opinion, - df.is_other_findings, - df.is_other_matters, - df.is_questioned_costs, - df.is_repeat_finding, - df.is_significant_deficiency, - df.prior_finding_ref_numbers, - df.type_requirement, - -- ALL of Passthrough - dp.passthrough_name, - dp.passthrough_id - FROM - dissemination_federalaward dfa - LEFT JOIN dissemination_general dg - ON dfa.report_id = dg.report_id - LEFT JOIN dissemination_finding df - ON dfa.report_id = df.report_id - AND dfa.award_reference = df.award_reference - LEFT JOIN dissemination_passthrough dp - ON dfa.report_id = dp.report_id - AND dfa.award_reference = dp.award_reference - ; - - -DROP MATERIALIZED VIEW IF EXISTS dissemination_combined; -ALTER SEQUENCE dissemination_combined_id_seq RESTART; -ALTER MATERIALIZED VIEW dissemination_combined_temp RENAME TO dissemination_combined; - -CREATE INDEX IF NOT EXISTS dc_report_id_idx - on dissemination_combined (report_id); - -CREATE INDEX IF NOT EXISTS dc_auditee_certify_name_idx - ON dissemination_combined - ((lower(auditee_certify_name))); - -CREATE INDEX IF NOT EXISTS dc_auditee_name_idx - ON dissemination_combined - ((lower(auditee_name))); - -CREATE INDEX IF NOT EXISTS dc_auditor_certify_name_idx - ON dissemination_combined - ((lower(auditor_certify_name))); - -CREATE INDEX IF NOT EXISTS dc_auditor_contact_name_idx - ON dissemination_combined - ((lower(auditor_contact_name))); - -CREATE INDEX IF NOT EXISTS dc_auditor_firm_name_idx - ON dissemination_combined - ((lower(auditor_firm_name))); - -CREATE INDEX IF NOT EXISTS dc_auditee_email_idx - on dissemination_combined ((lower(auditee_email))); - -CREATE INDEX IF NOT EXISTS dc_auditor_email_idx - on dissemination_combined ((lower(auditor_email))); - -CREATE INDEX IF NOT EXISTS dc_start_date_idx - ON dissemination_combined (fy_start_date); - -CREATE INDEX IF NOT EXISTS dc_end_date_idx - ON dissemination_combined (fy_end_date); - -CREATE INDEX IF NOT EXISTS dc_auditee_uei_idx - ON dissemination_combined (auditee_uei); - -CREATE INDEX IF NOT EXISTS dc_auditee_ein_idx - ON dissemination_combined (auditee_ein); - -CREATE INDEX IF NOT EXISTS dc_federal_agency_prefix_idx - on dissemination_combined (federal_agency_prefix); - -CREATE INDEX IF NOT EXISTS dc_federal_award_extension_idx - on dissemination_combined (federal_award_extension); - -CREATE INDEX IF NOT EXISTS dc_audit_year_idx - on dissemination_combined (audit_year); - -CREATE INDEX IF NOT EXISTS dc_aln_idx - on dissemination_combined (aln); - - --- Census to GSA Crosswalk -CREATE SEQUENCE IF NOT EXISTS census_gsa_crosswalk_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE; - -CREATE MATERIALIZED VIEW IF NOT EXISTS - census_gsa_crosswalk_temp AS - WITH dissemination_general_data_sub_temp AS - (SELECT - dg.report_id, - dg.auditee_uei, - dg.auditee_ein, - substring(dg.report_id, 1, 4) as audit_year, - ltrim(substring(dg.report_id, 16, 25), '0') as dbkey - FROM dissemination_general dg - WHERE dg.report_id ilike '%CENSUS%') - SELECT DISTINCT - nextval('census_gsa_crosswalk_id_seq') AS id, - chmr.audit_year, - chmr.dbkey, - gd.report_id, - gd.auditee_uei, - gd.auditee_ein, - chmr.run_datetime, - chmr.migration_status - FROM census_historical_migration_reportmigrationstatus chmr - INNER JOIN dissemination_general_data_sub_temp gd - ON chmr.audit_year = gd.audit_year AND chmr.dbkey = gd.dbkey - WHERE chmr.migration_status = 'SUCCESS'; - -DROP MATERIALIZED VIEW IF EXISTS census_gsa_crosswalk; -ALTER SEQUENCE census_gsa_crosswalk_id_seq RESTART; -ALTER MATERIALIZED VIEW census_gsa_crosswalk_temp RENAME TO census_gsa_crosswalk; - -CREATE INDEX IF NOT EXISTS cg_crosswalk_report_id_idx - on census_gsa_crosswalk (report_id); - -CREATE INDEX IF NOT EXISTS cg_crosswalk_audit_year_idx - on census_gsa_crosswalk (audit_year); - -CREATE INDEX IF NOT EXISTS cg_crosswalk_dbkey_idx - on census_gsa_crosswalk (dbkey); - -CREATE INDEX IF NOT EXISTS cg_crosswalk_auditee_uei_idx - on census_gsa_crosswalk (auditee_uei); - -CREATE INDEX IF NOT EXISTS cg_crosswalk_auditee_ein_idx - on census_gsa_crosswalk (auditee_ein); diff --git a/backend/dissemination/sql/drop_materialized_views.sql b/backend/dissemination/sql/drop_materialized_views.sql deleted file mode 100644 index 83c1e16867..0000000000 --- a/backend/dissemination/sql/drop_materialized_views.sql +++ /dev/null @@ -1,7 +0,0 @@ -DROP MATERIALIZED VIEW IF EXISTS dissemination_combined; - -DROP SEQUENCE IF EXISTS dissemination_combined_id_seq; - -DROP MATERIALIZED VIEW IF EXISTS census_gsa_crosswalk; - -DROP SEQUENCE IF EXISTS census_gsa_crosswalk_id_seq; \ No newline at end of file diff --git a/backend/dissemination/sql/fac-snapshot-db/post/954_indexes_federal_awards.sql b/backend/dissemination/sql/fac-snapshot-db/post/954_indexes_federal_awards.sql index f59f983c50..a01526bd6f 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/954_indexes_federal_awards.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/954_indexes_federal_awards.sql @@ -33,3 +33,6 @@ CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_awards_aln_cnt CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_batch_fa ON public_data_v1_0_0.federal_awards (batch_number); + +CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_ay + ON public_data_v1_0_0.federal_awards (audit_year); diff --git a/backend/dissemination/sql/fac-snapshot-db/post/999_finalize.sql b/backend/dissemination/sql/fac-snapshot-db/post/999_finalize.sql index d1168e7a39..9261088b61 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/999_finalize.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/999_finalize.sql @@ -1,3 +1,4 @@ -- PostgREST likes to know when the schemas and things -- attached to them change. NOTIFY pgrst, 'reload schema'; + diff --git a/backend/dissemination/sql/fac-snapshot-db/pre/000_api_v1_0_3.sql b/backend/dissemination/sql/fac-snapshot-db/pre/000_api_v1_0_3.sql index 358ecd172d..5025af5fb4 100644 --- a/backend/dissemination/sql/fac-snapshot-db/pre/000_api_v1_0_3.sql +++ b/backend/dissemination/sql/fac-snapshot-db/pre/000_api_v1_0_3.sql @@ -1,5 +1,2 @@ -BEGIN; - DROP SCHEMA IF EXISTS api_v1_0_3 CASCADE; -COMMIT; - +DROP SCHEMA IF EXISTS api_v1_0_3 CASCADE; NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/fac-snapshot-db/pre/030_admin_api_v1_1_1.sql b/backend/dissemination/sql/fac-snapshot-db/pre/030_admin_api_v1_1_1.sql index 63138d39c9..6cbdce2d39 100644 --- a/backend/dissemination/sql/fac-snapshot-db/pre/030_admin_api_v1_1_1.sql +++ b/backend/dissemination/sql/fac-snapshot-db/pre/030_admin_api_v1_1_1.sql @@ -1,8 +1,3 @@ -BEGIN; - DROP SCHEMA IF EXISTS admin_api_v1_1_1 CASCADE; - -COMMIT; - notify pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/fac-snapshot-db/pre/040_public_api_v1_0_0.sql b/backend/dissemination/sql/fac-snapshot-db/pre/040_public_api_v1_0_0.sql index e5feac30d5..c601de73f2 100644 --- a/backend/dissemination/sql/fac-snapshot-db/pre/040_public_api_v1_0_0.sql +++ b/backend/dissemination/sql/fac-snapshot-db/pre/040_public_api_v1_0_0.sql @@ -1,12 +1,8 @@ -BEGIN; - -- - -- This drops all the views, too. - --- - DROP SCHEMA IF EXISTS public_api_v1_0_0 CASCADE; - DROP SCHEMA IF EXISTS public_api_v1_0_0_functions CASCADE; - -COMMIT; - +-- +-- This drops all the views, too. +--- +DROP SCHEMA IF EXISTS public_api_v1_0_0 CASCADE; +DROP SCHEMA IF EXISTS public_api_v1_0_0_functions CASCADE; notify pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/refresh_materialized_views.sql b/backend/dissemination/sql/refresh_materialized_views.sql deleted file mode 100644 index 4911e13ed0..0000000000 --- a/backend/dissemination/sql/refresh_materialized_views.sql +++ /dev/null @@ -1,3 +0,0 @@ -REFRESH MATERIALIZED VIEW dissemination_combined; - -REFRESH MATERIALIZED VIEW census_gsa_crosswalk; \ No newline at end of file diff --git a/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml b/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml index 815583bbfd..224b75516e 100644 --- a/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml +++ b/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml @@ -116,7 +116,6 @@ streams: ORDER BY gen.id target_options: pre_sql: | - -- DROP SCHEMA IF EXISTS public_data_v1_0_0 CASCADE; DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_additional_eins; DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_additional_ueis; DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_combined; @@ -129,7 +128,6 @@ streams: DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_passthrough; DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_secondary_auditors; -- Now, create them. We get a start at 1 for every refresh this way. - CREATE SCHEMA IF NOT EXISTS public_data_v1_0_0; CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_additional_eins START 1; CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_additional_ueis START 1; CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_combined START 1; @@ -148,33 +146,6 @@ streams: -- Update that batch number as a modulus of 20K, our response -- size limit in PostgREST. UPDATE public_data_v1_0_0.general SET batch_number=DIV(seq, 20000); - -- -- Create a temporary table that is a copy of the general table, but partition it. - -- CREATE TABLE public_data_v1_0_0.tmp_general - -- (LIKE public_data_v1_0_0.general) - -- PARTITION BY hash(seq); - -- -- Drop our existing general table. - -- DROP TABLE public_data_v1_0_0.general; - -- SET search_path TO public_data_v1_0_0; --? - -- -- Rename that temporary table to the general table. Why? - -- -- We cannot create the table with a PARTITION, because it is a SELECT. - -- -- This works around that. - -- ALTER TABLE public_data_v1_0_0.tmp_general RENAME TO general; - -- -- Create the partition tables. - -- CREATE TABLE public_data_v1_0_0.part_general_0 - -- PARTITION OF public_data_v1_0_0.general - -- FOR VALUES WITH (modulus 5, remainder 0); - -- CREATE TABLE public_data_v1_0_0.part_general_1 - -- PARTITION OF public_data_v1_0_0.general - -- FOR VALUES WITH (modulus 5, remainder 1); - -- CREATE TABLE public_data_v1_0_0.part_general_2 - -- PARTITION OF public_data_v1_0_0.general - -- FOR VALUES WITH (modulus 5, remainder 2); - -- CREATE TABLE public_data_v1_0_0.part_general_3 - -- PARTITION OF public_data_v1_0_0.general - -- FOR VALUES WITH (modulus 5, remainder 3); - -- CREATE TABLE public_data_v1_0_0.part_general_4 - -- PARTITION OF public_data_v1_0_0.general - -- FOR VALUES WITH (modulus 5, remainder 4); -- Create the indexes on the table. CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_report_id_idx on public_data_v1_0_0.general (report_id); diff --git a/backend/docker-compose.yml b/backend/docker-compose.yml index fe3b29d28f..0bb665ae60 100644 --- a/backend/docker-compose.yml +++ b/backend/docker-compose.yml @@ -4,6 +4,7 @@ services: #--------------------------------------------- db: image: "postgres:15" + shm_size: 1g environment: POSTGRES_HOST_AUTH_METHOD: trust volumes: @@ -17,6 +18,7 @@ services: retries: 10 db2: image: "postgres:15" + shm_size: 1g environment: POSTGRES_HOST_AUTH_METHOD: "trust" volumes: diff --git a/backend/run.sh b/backend/run.sh index ddc4bcb0e8..45646815e3 100755 --- a/backend/run.sh +++ b/backend/run.sh @@ -9,10 +9,9 @@ source tools/util_startup.sh # This will choose the correct environment # for local envs (LOCAL or TESTING) and cloud.gov source tools/setup_env.sh -source tools/api_teardown.sh +source tools/sql_pre_post.sh source tools/migrate_app_tables.sh source tools/sling_first_run.sh -source tools/api_standup.sh source tools/build_indexes.sh source tools/seed_cog_baseline.sh @@ -22,11 +21,11 @@ setup_env gonogo "setup_env" ##### -# API TEARDOWN -# API has to be deprecated/removed before migration, because -# of tight coupling between schema/views and the dissemination tables -api_teardown -gonogo "api_teardown" +# SQL PRE +# We have SQL that we want to run before the migrations and sling are run. +# This tears down things that would conflict with migrations, etc. +sql_pre +gonogo "sql_pre" ##### # MIGRATE APP TABLES @@ -41,16 +40,11 @@ sling_first_run gonogo "sling_first_run" ##### -# API STANDUP -# Standup the API, which may depend on migration changes -api_standup -gonogo "api_standup" - -##### -# BUILD INDEXES -# Builds indexes on the API tables in fac-snapshot-db -build_indexes -gonogo "build_indexes" +# SQL POST +# Rebuild the API and prepare the system for execution. +# Runs after migrations. +sql_post +gonogo "sql_post" ##### # SEED COG/OVER TABLES diff --git a/backend/tools/setup_cgov_env.sh b/backend/tools/setup_cgov_env.sh index d4ca2e1954..266d8349f8 100644 --- a/backend/tools/setup_cgov_env.sh +++ b/backend/tools/setup_cgov_env.sh @@ -46,6 +46,8 @@ function setup_cgov_env { export FAC_SNAPSHOT_URI="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "fac-snapshot-db" ".[][] | select(.name == \$service_name) | .credentials.uri")" # https://stackoverflow.com/questions/37072245/check-return-status-of-psql-command-in-unix-shell-scripting export PSQL_EXE='/home/vcap/deps/0/apt/usr/lib/postgresql/*/bin/psql --single-transaction -v ON_ERROR_STOP=on' + export PSQL_EXE_NO_TXN='/home/vcap/deps/0/apt/usr/lib/postgresql/*/bin/psql -v ON_ERROR_STOP=on' + export SLING_EXE='/home/vcap/app/sling' return 0 diff --git a/backend/tools/setup_local_env.sh b/backend/tools/setup_local_env.sh index 672eaa8108..57e332acfc 100644 --- a/backend/tools/setup_local_env.sh +++ b/backend/tools/setup_local_env.sh @@ -17,6 +17,8 @@ function setup_local_env { export FAC_DB_URI=${DATABASE_URL}?sslmode=disable export FAC_SNAPSHOT_URI=${SNAPSHOT_URL}?sslmode=disable export PSQL_EXE='psql --single-transaction -v ON_ERROR_STOP=on' + export PSQL_EXE_NO_TXN='psql -v ON_ERROR_STOP=on' + export SLING_EXE='/bin/sling' export CGOV_UTIL_EXE='/bin/cgov-util' diff --git a/backend/tools/sql_pre_post.sql b/backend/tools/sql_pre_post.sql deleted file mode 100644 index 0bc36ade83..0000000000 --- a/backend/tools/sql_pre_post.sql +++ /dev/null @@ -1,15 +0,0 @@ -source tools/util_startup.sh - -function api_standup { - startup_log "API_STANDUP" "BEGIN" - # Loop by index. - for index in "${!standup_scripts[@]}" - do - local subdir="${standup_scripts[index]}" - local path="dissemination/sql/api/${subdir}/standup.sql" - run_sql $FAC_SNAPSHOT_URI $path - done - startup_log "API_STANDUP" "END" - return 0 -} - diff --git a/backend/tools/variables.sh b/backend/tools/variables.sh index 903d168605..a480603f2b 100644 --- a/backend/tools/variables.sh +++ b/backend/tools/variables.sh @@ -1,62 +1,62 @@ # These are our active APIs # admin_api_access_tables must run before # the admin API can be stood up. -declare -a sql_pre_scripts=( - "api_v1_0_3" - "api_v1_1_0" - "admin_api_access_tables" - "admin_api_v1_1_0" - "admin_api_v1_1_1" - "public_api_v1_0_0" - "public_data_v1_0_0" - # We need some of this for the - # copying of data from DB1 to DB2. - # Specifically, schemas, roles, permissions, and sequences. - "schemas" - "roles" - "permissions" - "sequences" - "finalize" -) +# declare -a sql_pre_scripts=( +# "api_v1_0_3" +# "api_v1_1_0" +# "admin_api_access_tables" +# "admin_api_v1_1_0" +# "admin_api_v1_1_1" +# "public_api_v1_0_0" +# "public_data_v1_0_0" +# # We need some of this for the +# # copying of data from DB1 to DB2. +# # Specifically, schemas, roles, permissions, and sequences. +# "schemas" +# "roles" +# "permissions" +# "sequences" +# "finalize" +# ) -declare -a sql_post_scripts=( - "api_v1_0_3" - "api_v1_1_0" - "admin_api_access_tables" - "admin_api_v1_1_0" - "admin_api_v1_1_1" - "public_api_v1_0_0" - "public_data_v1_0_0" - "permissions" - "finalize" - ) +# declare -a sql_post_scripts=( +# "api_v1_0_3" +# "api_v1_1_0" +# "admin_api_access_tables" +# "admin_api_v1_1_0" +# "admin_api_v1_1_1" +# "public_api_v1_0_0" +# "public_data_v1_0_0" +# "permissions" +# "finalize" +# ) -# # These are the tables that must be present -# # in order to stand up that API. -# declare -a api_required_tables=( -# "public.dissemination_general" -# "public.dissemination_general" -# "public.support_adminapievent" -# "public.support_administrative_key_uuids" -# "public.support_administrative_key_uuids" -# "public_data_v1_0_0.general" -# "public.dissemination_general" -# ) +# # # These are the tables that must be present +# # # in order to stand up that API. +# # declare -a api_required_tables=( +# # "public.dissemination_general" +# # "public.dissemination_general" +# # "public.support_adminapievent" +# # "public.support_administrative_key_uuids" +# # "public.support_administrative_key_uuids" +# # "public_data_v1_0_0.general" +# # "public.dissemination_general" +# # ) -declare -a indexes=( - "additional_eins" - "additional_ueis" - "combined" - "corrective_action_plans" - "federal_awards" - "findings" - "findings_text" - "general" - "notes_to_sefa" - "passthrough" - "secondary_auditors" -) +# declare -a indexes=( +# "additional_eins" +# "additional_ueis" +# "combined" +# "corrective_action_plans" +# "federal_awards" +# "findings" +# "findings_text" +# "general" +# "notes_to_sefa" +# "passthrough" +# "secondary_auditors" +# ) # declare -a db2_indexes_required_tables=( # "public_data_v1_0_0.additional_eins" diff --git a/backend/util/api_perf_test/perf.py b/backend/util/api_perf_test/perf.py index 75cd3f10a8..db1e8954ad 100644 --- a/backend/util/api_perf_test/perf.py +++ b/backend/util/api_perf_test/perf.py @@ -3,6 +3,7 @@ import time from pprint import pprint import math +import json URI = "http://localhost:3000" @@ -16,6 +17,7 @@ def fetch_fa_exp(api_version): total_cost = 0 for offset in range(0, 4000000, 20000): + print(f"fetch_fa_exp api {api_version} offset {offset}") query = f"{URI}/federal_awards?limit=20000&offset={offset}" headers = { "accept-profile": api_version, @@ -30,10 +32,31 @@ def fetch_fa_exp(api_version): return math.floor(total_cost) +def fetch_fa_by_year_exp(api_version): + total_cost = 0 + for year in range(16, 24): + audit_year = f"20{year:02}" + for offset in range(0, 1000000, 20000): + print( + f"fetch_fa_by_year_exp api {api_version} ay {audit_year} offset {offset}" + ) + query = f"{URI}/federal_awards?audit_year=eq.{audit_year}&limit=20000&offset={offset}" + headers = { + "accept-profile": api_version, + "accept": "application/vnd.pgrst.plan+json", + "x-api-user-id": os.getenv("API_KEY_ID"), + "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", + } + + resp = requests.get(query, headers=headers) + # We get back a list of one plan, and we want the total cost. + total_cost += resp.json()[0]["Plan"]["Total Cost"] + return math.floor(total_cost) + + def fetch_fa_time(api_version): total_cost = 0 for offset in range(0, 4000000, 20000): - print(f"api {api_version} offset {offset}") query = f"{URI}/federal_awards?limit=20000&offset={offset}" headers = { "accept-profile": api_version, @@ -45,6 +68,29 @@ def fetch_fa_time(api_version): t1 = time.time() # We get back a list of one plan, and we want the total cost. total_cost += t1 - t0 + print(f"fetch_fa_time api {api_version} offset {offset} time {t1-t0}") + return math.floor(total_cost) + + +def fetch_fa_time_by_year(api_version): + total_cost = 0 + for year in range(16, 24): + for offset in range(0, 1000000, 20000): + audit_year = f"20{year:02}" + query = f"{URI}/federal_awards?audit_year=eq.{audit_year}&limit=20000&offset={offset}" + headers = { + "accept-profile": api_version, + "x-api-user-id": os.getenv("API_KEY_ID"), + "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", + } + t0 = time.time() + resp = requests.get(query, headers=headers) + t1 = time.time() + # We get back a list of one plan, and we want the total cost. + total_cost += t1 - t0 + print( + f"fetch_fa_time_by_year api {api_version} ay {audit_year} offset {offset} time {t1-t0}" + ) return math.floor(total_cost) @@ -83,36 +129,41 @@ def fetch_fa_batches_time(): return math.floor(total_cost) +def make_ratios(d1, d2): + min = math.inf + for k, v in d1.items(): + if v < min: + min = v + for k, v in d1.items(): + d2[k] = round(v / min, 2) + + if __name__ == "__main__": results1 = {} results2 = {} results3 = {} results4 = {} + results1["api110_by_year"] = fetch_fa_by_year_exp("api_v1_1_0") results1["ap110"] = fetch_fa_exp("api_v1_1_0") - results1["public200"] = fetch_fa_exp("public_api_v1_0_0") - results1["public200_batches"] = fetch_fa_batches_exp() - - min = math.inf - for k, v in results1.items(): - if v < min: - min = v - for k, v in results1.items(): - results2[k] = math.floor(v / min) + results1["public100"] = fetch_fa_exp("public_api_v1_0_0") + results1["public100_batches"] = fetch_fa_batches_exp() + results1["public100_by_year"] = fetch_fa_by_year_exp("public_api_v1_0_0") print("Running timing tests... ~5m") + results3["public100_by_year"] = fetch_fa_time_by_year("public_api_v1_0_0") + results3["public100"] = fetch_fa_time("public_api_v1_0_0") + results3["public100_batches"] = fetch_fa_batches_time() results3["ap110"] = fetch_fa_time("api_v1_1_0") - results3["public200"] = fetch_fa_time("public_api_v1_0_0") - results3["public200_batches"] = fetch_fa_batches_time() - - min = math.inf - for k, v in results3.items(): - if v < min: - min = v - for k, v in results3.items(): - results4[k] = math.floor(v / min) - + results3["ap110_by_year"] = fetch_fa_time_by_year("api_v1_1_0") + + make_ratios(results1, results2) + make_ratios(results3, results4) + results1["desc"] = "EXPLAIN raw" + results2["desc"] = "EXPLAIN ratio" + results3["desc"] = "TIME raw" + results4["desc"] = "TIME ratio" # results1 is the raw EXPLAIN cost of downloading all of federal_awards pprint(results1) # results2 is the ratio @@ -121,3 +172,52 @@ def fetch_fa_batches_time(): pprint(results3) # results4 is the ratio for timings pprint(results4) + +# Where there is no index on the audit_year column. +# by_year is worse. +# {'ap110': 36737858, +# 'api110_by_year': 87880072, +# 'public100': 36305424, +# 'public100_batches': 1176467, +# 'public100_by_year': 11407901} +# {'ap110': 31, +# 'api110_by_year': 74, +# 'public100': 30, +# 'public100_batches': 1, +# 'public100_by_year': 9} +# {'ap110': 188, +# 'ap110_by_year': 304, +# 'public100': 62, +# 'public100_batches': 29, +# 'public100_by_year': 40} +# {'ap110': 6, +# 'ap110_by_year': 10, +# 'public100': 2, +# 'public100_batches': 1, +# 'public100_by_year': 1} + + +# {'ap110': 36737858, +# 'api110_by_year': 87880072, +# 'desc': 'EXPLAIN raw', +# 'public100': 36304899, +# 'public100_batches': 1179458, +# 'public100_by_year': 11407650} +# {'ap110': 31.15, +# 'api110_by_year': 74.51, +# 'desc': 'EXPLAIN ratio', +# 'public100': 30.78, +# 'public100_batches': 1.0, +# 'public100_by_year': 9.67} +# {'ap110': 193, +# 'ap110_by_year': 312, +# 'desc': 'TIME raw', +# 'public100': 63, +# 'public100_batches': 29, +# 'public100_by_year': 45} +# {'ap110': 6.66, +# 'ap110_by_year': 10.76, +# 'desc': 'TIME ratio', +# 'public100': 2.17, +# 'public100_batches': 1.0, +# 'public100_by_year': 1.55} From d5d4915b127dc04d5621cf572f43614b3d7f9a23 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Thu, 10 Oct 2024 11:33:32 -0400 Subject: [PATCH 26/89] Set +/-e around check... --- backend/tools/util_startup.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/tools/util_startup.sh b/backend/tools/util_startup.sh index d027356c4b..65c446611c 100644 --- a/backend/tools/util_startup.sh +++ b/backend/tools/util_startup.sh @@ -23,9 +23,11 @@ function check_table_exists() { local dbname="$2" echo "CHECK_TABLE_EXISTS: $dbname" # >/dev/null 2>&1 + set +e $PSQL_EXE $db_uri -c "SELECT '$dbname'::regclass" || false local result=$? echo "CHECK_TABLE_EXISTS $dbname: $result" + set -e return $result } From 6b45c492812029182e02a728c4c0525532afb4c1 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Thu, 10 Oct 2024 11:52:05 -0400 Subject: [PATCH 27/89] Trying to get past error... Need to just return a non-zero exit code... --- backend/tools/util_startup.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/tools/util_startup.sh b/backend/tools/util_startup.sh index 65c446611c..52f720f586 100644 --- a/backend/tools/util_startup.sh +++ b/backend/tools/util_startup.sh @@ -24,8 +24,7 @@ function check_table_exists() { echo "CHECK_TABLE_EXISTS: $dbname" # >/dev/null 2>&1 set +e - $PSQL_EXE $db_uri -c "SELECT '$dbname'::regclass" || false - local result=$? + $PSQL_EXE $db_uri -c "SELECT '$dbname'::regclass" || result=1 echo "CHECK_TABLE_EXISTS $dbname: $result" set -e return $result From 328e9b9392d494a7a50295a94967e7f25750506b Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Thu, 10 Oct 2024 12:14:12 -0400 Subject: [PATCH 28/89] Fixing SQL checks Need to *always* succeed... --- backend/tools/sling_first_run.sh | 4 +-- backend/tools/sql_pre_post.sh | 3 +++ backend/tools/util_startup.sh | 26 +++++++++++-------- .../load_public_data_locally.sh | 2 +- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/backend/tools/sling_first_run.sh b/backend/tools/sling_first_run.sh index 7aa3f55925..d24036922a 100644 --- a/backend/tools/sling_first_run.sh +++ b/backend/tools/sling_first_run.sh @@ -6,7 +6,7 @@ function sling_first_run() { # The deploy action does a DB->DB backup with cgov-util. # This should only ever actually run when testing locally. # In the cloud environment, it will skip the actul run. - check_table_exists $FAC_SNAPSHOT_URI 'public.dissemination_general' + check_table_exists $FAC_SNAPSHOT_URI 'public' 'dissemination_general' local is_general_table=$? if [ $is_general_table -ne 0 ]; then # This is the first run. @@ -18,7 +18,7 @@ function sling_first_run() { fi # If the metadata table exists, it means sling has run to completion. - check_table_exists $FAC_SNAPSHOT_URI 'public_data_v1_0_0.metadata' + check_table_exists $FAC_SNAPSHOT_URI 'public_data_v1_0_0' 'metadata' local is_metadata_table=$? echo "check_table_exists public_data_v1_0_0.metadata $is_metadata_table" diff --git a/backend/tools/sql_pre_post.sh b/backend/tools/sql_pre_post.sh index af01f14219..999dfb6a71 100644 --- a/backend/tools/sql_pre_post.sh +++ b/backend/tools/sql_pre_post.sh @@ -31,6 +31,9 @@ function sql_pre { function sql_post { run_sql_files $FAC_SNAPSHOT_URI "SQL_POST" "post" +} + +function vacuum_snapshot_db { # Vacuum things when we're done. # Cannot run inside a transaction. $PSQL_EXE_NO_TXN $FAC_SNAPSHOT_URI -c "VACUUM (FULL, VERBOSE, ANALYZE);" diff --git a/backend/tools/util_startup.sh b/backend/tools/util_startup.sh index 52f720f586..4f06091242 100644 --- a/backend/tools/util_startup.sh +++ b/backend/tools/util_startup.sh @@ -20,25 +20,29 @@ function gonogo { function check_table_exists() { local db_uri="$1" - local dbname="$2" - echo "CHECK_TABLE_EXISTS: $dbname" + local schema="$2" + local table="$3" + + echo "CHECK_TABLE_EXISTS: $schema.$table" # >/dev/null 2>&1 - set +e - $PSQL_EXE $db_uri -c "SELECT '$dbname'::regclass" || result=1 - echo "CHECK_TABLE_EXISTS $dbname: $result" - set -e - return $result + # The qtAX incantation lets us pass the PSQL result value back to bash. + result=`$PSQL_EXE $db_uri -qtAX -c "SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = '$schema' AND tablename = '$table');"` + echo "CHECK_TABLE_EXISTS $schema.$table: $result" + # Flip TRUE to a 0, because UNIX considers a 0 exit code to be good. + if [ "$result" = "t" ]; then + return 0; + else + return 1; + fi; } function check_schema_exists () { local db_uri="$1" local schema_name="$2" - set +e echo "CHECK_SCHEMA_EXISTS $schema_name" - $PSQL_EXE $db_uri -qtAX -c "SELECT EXISTS(SELECT 1 FROM information_schema.schemata WHERE schema_name = '$schema_name');" || false - local result=$? + result=`$PSQL_EXE $db_uri -qtAX -c "SELECT EXISTS(SELECT 1 FROM information_schema.schemata WHERE schema_name = '$schema_name');"` echo "CHECK_SCHEMA_EXISTS $schema_name: $result" - set -e + # Flip TRUE to a 0, because UNIX considers a 0 exit code to be good. if [ "$result" = "t" ]; then return 0; else diff --git a/backend/util/load_public_dissem_data/load_public_data_locally.sh b/backend/util/load_public_dissem_data/load_public_data_locally.sh index f102204fc2..8919253330 100755 --- a/backend/util/load_public_dissem_data/load_public_data_locally.sh +++ b/backend/util/load_public_dissem_data/load_public_data_locally.sh @@ -71,7 +71,7 @@ fi # Run the backup of the dissemination_ tables from # fac-db to fac-snapshot-db. -check_table_exists $FAC_SNAPSHOT_URI "public.dissemination_general" +check_table_exists $FAC_SNAPSHOT_URI "public" "dissemination_general" result=$? # cgov-util wants to know the environment it is in. export ENV="LOCAL" From 3539ffcfa01fd38c496d871a8a5aafe3fa66a4f9 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Thu, 10 Oct 2024 12:34:25 -0400 Subject: [PATCH 29/89] Still... fixing return values... --- backend/tools/sling_first_run.sh | 2 +- backend/tools/util_startup.sh | 58 ++++++++++++++++++-------------- 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/backend/tools/sling_first_run.sh b/backend/tools/sling_first_run.sh index d24036922a..239e9eb3f5 100644 --- a/backend/tools/sling_first_run.sh +++ b/backend/tools/sling_first_run.sh @@ -20,7 +20,7 @@ function sling_first_run() { # If the metadata table exists, it means sling has run to completion. check_table_exists $FAC_SNAPSHOT_URI 'public_data_v1_0_0' 'metadata' local is_metadata_table=$? - echo "check_table_exists public_data_v1_0_0.metadata $is_metadata_table" + echo "SLING_FIRST_RUN public_data_v1_0_0.metadata = $is_metadata_table" # We need to load some functions for sling to complete, because # we use those functions as part of the metadata table generation. diff --git a/backend/tools/util_startup.sh b/backend/tools/util_startup.sh index 4f06091242..41379b2eb5 100644 --- a/backend/tools/util_startup.sh +++ b/backend/tools/util_startup.sh @@ -18,36 +18,44 @@ function gonogo { fi } +# 2024-10-10T12:28:29.61-0400 [APP/PROC/WEB/0] OUT CHECK_TABLE_EXISTS START: public.dissemination_general +# 2024-10-10T12:28:29.65-0400 [APP/PROC/WEB/0] OUT CHECK_TABLE_EXISTS END: public.dissemination_general = t +# 2024-10-10T12:28:29.65-0400 [APP/PROC/WEB/0] OUT CHECK_TABLE_EXISTS: public_data_v1_0_0.metadata +# 2024-10-10T12:28:29.68-0400 [APP/PROC/WEB/0] OUT CHECK_TABLE_EXISTS public_data_v1_0_0.metadata: f +# 2024-10-10T12:28:29.68-0400 [APP/PROC/WEB/0] OUT Exit status 1 +# 2024-10-10T12:28:29.68-0400 [CELL/SSHD/0] OUT Exit status 0 + function check_table_exists() { - local db_uri="$1" - local schema="$2" - local table="$3" + local db_uri="$1" + local schema="$2" + local table="$3" - echo "CHECK_TABLE_EXISTS: $schema.$table" - # >/dev/null 2>&1 - # The qtAX incantation lets us pass the PSQL result value back to bash. - result=`$PSQL_EXE $db_uri -qtAX -c "SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = '$schema' AND tablename = '$table');"` - echo "CHECK_TABLE_EXISTS $schema.$table: $result" - # Flip TRUE to a 0, because UNIX considers a 0 exit code to be good. - if [ "$result" = "t" ]; then - return 0; - else - return 1; - fi; + echo "CHECK_TABLE_EXISTS START: $schema.$table" + # >/dev/null 2>&1 + # The qtAX incantation lets us pass the PSQL result value back to bash. + result=`$PSQL_EXE $db_uri -qtAX -c "SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = '$schema' AND tablename = '$table');"` + # Flip TRUE to a 0, because UNIX considers a 0 exit code to be good. + if [ "$result" = "t" ]; then + echo "CHECK_TABLE_EXISTS END: $schema.$table = 0" + return 0 + else + echo "CHECK_TABLE_EXISTS END: $schema.$table = 1" + return 1 + fi } function check_schema_exists () { - local db_uri="$1" - local schema_name="$2" - echo "CHECK_SCHEMA_EXISTS $schema_name" - result=`$PSQL_EXE $db_uri -qtAX -c "SELECT EXISTS(SELECT 1 FROM information_schema.schemata WHERE schema_name = '$schema_name');"` - echo "CHECK_SCHEMA_EXISTS $schema_name: $result" - # Flip TRUE to a 0, because UNIX considers a 0 exit code to be good. - if [ "$result" = "t" ]; then - return 0; - else - return 1; - fi + local db_uri="$1" + local schema_name="$2" + echo "CHECK_SCHEMA_EXISTS $schema_name" + result=`$PSQL_EXE $db_uri -qtAX -c "SELECT EXISTS(SELECT 1 FROM information_schema.schemata WHERE schema_name = '$schema_name');"` + echo "CHECK_SCHEMA_EXISTS $schema_name: $result" + # Flip TRUE to a 0, because UNIX considers a 0 exit code to be good. + if [ "$result" = "t" ]; then + return 0 + else + return 1 + fi } function run_sql () { From 1093b0f345c8e07459caf39b9f5b1c5a15efd067 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Thu, 10 Oct 2024 12:37:00 -0400 Subject: [PATCH 30/89] Make the sources the same --- backend/.profile | 8 ++++---- backend/run.sh | 11 ++++------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/backend/.profile b/backend/.profile index b92feba65c..8964589ea4 100644 --- a/backend/.profile +++ b/backend/.profile @@ -5,14 +5,14 @@ # so a typo in a function name will fail silently. Similarly, # bash has horrible scoping, so use of `local` in functions is # critical for cleanliness in the startup script. -source tools/util_startup.sh # This will choose the correct environment # for local envs (LOCAL or TESTING) and cloud.gov -source tools/setup_env.sh -source tools/sql_pre_post.sh -source tools/sling_first_run.sh source tools/migrate_app_tables.sh source tools/seed_cog_baseline.sh +source tools/setup_env.sh +source tools/sling_first_run.sh +source tools/sql_pre_post.sh +source tools/util_startup.sh ##### # SETUP THE CGOV ENVIRONMENT diff --git a/backend/run.sh b/backend/run.sh index 45646815e3..7e33169f9b 100755 --- a/backend/run.sh +++ b/backend/run.sh @@ -5,15 +5,12 @@ # so a typo in a function name will fail silently. Similarly, # bash has horrible scoping, so use of `local` in functions is # critical for cleanliness in the startup script. -source tools/util_startup.sh -# This will choose the correct environment -# for local envs (LOCAL or TESTING) and cloud.gov -source tools/setup_env.sh -source tools/sql_pre_post.sh source tools/migrate_app_tables.sh -source tools/sling_first_run.sh -source tools/build_indexes.sh source tools/seed_cog_baseline.sh +source tools/setup_env.sh +source tools/sling_first_run.sh +source tools/sql_pre_post.sh +source tools/util_startup.sh ##### # SETUP THE LOCAL ENVIRONMENT From cd525cb0b5ad09e8bc21128f99dc6da3422ee60f Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Thu, 10 Oct 2024 12:39:36 -0400 Subject: [PATCH 31/89] Fix echo statements --- backend/tools/util_startup.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/tools/util_startup.sh b/backend/tools/util_startup.sh index 41379b2eb5..106d4b0161 100644 --- a/backend/tools/util_startup.sh +++ b/backend/tools/util_startup.sh @@ -47,13 +47,14 @@ function check_table_exists() { function check_schema_exists () { local db_uri="$1" local schema_name="$2" - echo "CHECK_SCHEMA_EXISTS $schema_name" + echo "CHECK_SCHEMA_EXISTS START: $schema_name" result=`$PSQL_EXE $db_uri -qtAX -c "SELECT EXISTS(SELECT 1 FROM information_schema.schemata WHERE schema_name = '$schema_name');"` - echo "CHECK_SCHEMA_EXISTS $schema_name: $result" # Flip TRUE to a 0, because UNIX considers a 0 exit code to be good. if [ "$result" = "t" ]; then + echo "CHECK_SCHEMA_EXISTS END: $schema_name = 0" return 0 else + echo "CHECK_SCHEMA_EXISTS END: $schema_name = 1" return 1 fi } From cbadf2ca0f862f0ec1ba1fa9cfe442b746da3558 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Thu, 10 Oct 2024 12:51:16 -0400 Subject: [PATCH 32/89] Try set -e? --- backend/tools/sling_first_run.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/tools/sling_first_run.sh b/backend/tools/sling_first_run.sh index 239e9eb3f5..70be1a8826 100644 --- a/backend/tools/sling_first_run.sh +++ b/backend/tools/sling_first_run.sh @@ -2,7 +2,10 @@ source tools/util_startup.sh function sling_first_run() { startup_log "SLING_FIRST_RUN" "Slinging data to fac-snapshot if needed" - + # check_table_exists might return 1. + # It is possible set -e is kicking us out when the function + # returns an error? So, try set +e. + set +e # The deploy action does a DB->DB backup with cgov-util. # This should only ever actually run when testing locally. # In the cloud environment, it will skip the actul run. From 2b332860be8c2f8a724d9c101ae2fa9d18521b94 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Thu, 10 Oct 2024 15:49:38 -0400 Subject: [PATCH 33/89] What about set +e... We don't really want this, but will it allow functions to return non-zero? --- backend/.profile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/.profile b/backend/.profile index 8964589ea4..fd881aaf09 100644 --- a/backend/.profile +++ b/backend/.profile @@ -1,5 +1,7 @@ #!/bin/bash +set +e + # Source everything; everything is now a function. # Remember: bash has no idea if a function exists, # so a typo in a function name will fail silently. Similarly, From cf6b5c909251b45f08ed96e677d53c88337c328e Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Thu, 10 Oct 2024 17:07:20 -0400 Subject: [PATCH 34/89] Working around bash... Setting a global function response variable... --- backend/tools/sling_first_run.sh | 4 +- backend/tools/util_startup.sh | 16 ++++--- backend/tools/variables.sh | 74 +------------------------------- 3 files changed, 12 insertions(+), 82 deletions(-) diff --git a/backend/tools/sling_first_run.sh b/backend/tools/sling_first_run.sh index 70be1a8826..aea89cc842 100644 --- a/backend/tools/sling_first_run.sh +++ b/backend/tools/sling_first_run.sh @@ -10,7 +10,7 @@ function sling_first_run() { # This should only ever actually run when testing locally. # In the cloud environment, it will skip the actul run. check_table_exists $FAC_SNAPSHOT_URI 'public' 'dissemination_general' - local is_general_table=$? + local is_general_table=$FUNCTION_RESULT if [ $is_general_table -ne 0 ]; then # This is the first run. startup_log "SLING_FIRST_RUN" "Running cgov-util INITIAL." @@ -22,7 +22,7 @@ function sling_first_run() { # If the metadata table exists, it means sling has run to completion. check_table_exists $FAC_SNAPSHOT_URI 'public_data_v1_0_0' 'metadata' - local is_metadata_table=$? + local is_metadata_table=$FUNCTION_RESULT echo "SLING_FIRST_RUN public_data_v1_0_0.metadata = $is_metadata_table" # We need to load some functions for sling to complete, because diff --git a/backend/tools/util_startup.sh b/backend/tools/util_startup.sh index 106d4b0161..a5ba18cc5d 100644 --- a/backend/tools/util_startup.sh +++ b/backend/tools/util_startup.sh @@ -3,7 +3,7 @@ source tools/variables.sh function startup_log { local tag="$1" local msg="$2" - echo STARTUP $tag $msg + echo "STARTUP" "$tag" "$msg" } # gonogo @@ -14,7 +14,7 @@ function gonogo { return 0 else startup_log "STARTUP_CHECK" "$1 FAIL" - exit -1 + exit 1 fi } @@ -33,15 +33,16 @@ function check_table_exists() { echo "CHECK_TABLE_EXISTS START: $schema.$table" # >/dev/null 2>&1 # The qtAX incantation lets us pass the PSQL result value back to bash. - result=`$PSQL_EXE $db_uri -qtAX -c "SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = '$schema' AND tablename = '$table');"` + result=`$PSQL_EXE "$db_uri" -qtAX -c "SELECT EXISTS (SELECT FROM pg_tables WHERE schemaname = '$schema' AND tablename = '$table');"` # Flip TRUE to a 0, because UNIX considers a 0 exit code to be good. if [ "$result" = "t" ]; then echo "CHECK_TABLE_EXISTS END: $schema.$table = 0" - return 0 + FUNCTION_RESULT=0 else echo "CHECK_TABLE_EXISTS END: $schema.$table = 1" - return 1 + FUNCTION_RESULT=1 fi + return 0 } function check_schema_exists () { @@ -52,11 +53,12 @@ function check_schema_exists () { # Flip TRUE to a 0, because UNIX considers a 0 exit code to be good. if [ "$result" = "t" ]; then echo "CHECK_SCHEMA_EXISTS END: $schema_name = 0" - return 0 + FUNCTION_RESULT=0 else echo "CHECK_SCHEMA_EXISTS END: $schema_name = 1" - return 1 + FUNCTION_RESULT=1 fi + return 0 } function run_sql () { diff --git a/backend/tools/variables.sh b/backend/tools/variables.sh index a480603f2b..f28b2d998c 100644 --- a/backend/tools/variables.sh +++ b/backend/tools/variables.sh @@ -1,73 +1 @@ -# These are our active APIs -# admin_api_access_tables must run before -# the admin API can be stood up. -# declare -a sql_pre_scripts=( -# "api_v1_0_3" -# "api_v1_1_0" -# "admin_api_access_tables" -# "admin_api_v1_1_0" -# "admin_api_v1_1_1" -# "public_api_v1_0_0" -# "public_data_v1_0_0" -# # We need some of this for the -# # copying of data from DB1 to DB2. -# # Specifically, schemas, roles, permissions, and sequences. -# "schemas" -# "roles" -# "permissions" -# "sequences" -# "finalize" -# ) - -# declare -a sql_post_scripts=( -# "api_v1_0_3" -# "api_v1_1_0" -# "admin_api_access_tables" -# "admin_api_v1_1_0" -# "admin_api_v1_1_1" -# "public_api_v1_0_0" -# "public_data_v1_0_0" -# "permissions" -# "finalize" -# ) - - -# # # These are the tables that must be present -# # # in order to stand up that API. -# # declare -a api_required_tables=( -# # "public.dissemination_general" -# # "public.dissemination_general" -# # "public.support_adminapievent" -# # "public.support_administrative_key_uuids" -# # "public.support_administrative_key_uuids" -# # "public_data_v1_0_0.general" -# # "public.dissemination_general" -# # ) - -# declare -a indexes=( -# "additional_eins" -# "additional_ueis" -# "combined" -# "corrective_action_plans" -# "federal_awards" -# "findings" -# "findings_text" -# "general" -# "notes_to_sefa" -# "passthrough" -# "secondary_auditors" -# ) - -# declare -a db2_indexes_required_tables=( -# "public_data_v1_0_0.additional_eins" -# "public_data_v1_0_0.additional_ueis" -# "public_data_v1_0_0.combined" -# "public_data_v1_0_0.corrective_action_plans" -# "public_data_v1_0_0.federal_awards" -# "public_data_v1_0_0.findings" -# "public_data_v1_0_0.findings_text" -# "public_data_v1_0_0.general" -# "public_data_v1_0_0.notes_to_sefa" -# "public_data_v1_0_0.passthrough" -# "public_data_v1_0_0.secondary_auditors" -# ) +export FUNCTION_RESULT="DAS DEFAULT" From a511d9cbc7dda8cd3ae8c6ca69cd3cae5b2e3761 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Tue, 22 Oct 2024 16:03:10 -0400 Subject: [PATCH 35/89] Incremental on removal of Tribal/admin API. This was work-in-flight. Instead of pushing further, I'm committing the in-between state. This moved some Admin API stuff around, which I'll remove in the next commit. The config change is for local dev only. We had a bucket called `gsa-fac-private-s3`, which was legacy naming that has been superceded in all other environments. The bucket is now called `fac-private-s3`, mirroring the production environments. This should not impact any operations except for local/Docker execution. (And, it should be transparent) The next commit should clean this up further. --- backend/census_historical_migration/README.md | 8 +- backend/config/settings.py | 3 +- .../support/dissemination-table-via-pdf.js | 114 ++++ backend/dissemination/sql/SQL_README.md | 49 ++ ...partitioning.sql => 001_partitioning.SKIP} | 7 + .../post/030_admin_api_access_tables.sql | 43 -- .../post/040_admin_api_v1_1_0.sql | 615 ------------------ .../post/050_admin_api_v1_1_1.sql | 572 ---------------- .../pre/020_admin_api_v1_1_0.sql | 17 - .../pre/025_admin_api_access_tables.sql | 2 - backend/dissemination/test_views.py | 2 +- backend/docker-compose.yml | 5 +- backend/run.sh | 3 +- backend/tools/example_vcap.json | 201 ++++++ backend/tools/setup_cgov_env.sh | 43 +- backend/tools/setup_local_env.sh | 31 +- backend/util/api_perf_test/perf.py | 75 ++- 17 files changed, 509 insertions(+), 1281 deletions(-) create mode 100644 backend/cypress/support/dissemination-table-via-pdf.js create mode 100644 backend/dissemination/sql/SQL_README.md rename backend/dissemination/sql/fac-snapshot-db/post/{001_partitioning.sql => 001_partitioning.SKIP} (97%) delete mode 100644 backend/dissemination/sql/fac-snapshot-db/post/030_admin_api_access_tables.sql delete mode 100644 backend/dissemination/sql/fac-snapshot-db/post/040_admin_api_v1_1_0.sql delete mode 100644 backend/dissemination/sql/fac-snapshot-db/post/050_admin_api_v1_1_1.sql delete mode 100644 backend/dissemination/sql/fac-snapshot-db/pre/020_admin_api_v1_1_0.sql delete mode 100644 backend/dissemination/sql/fac-snapshot-db/pre/025_admin_api_access_tables.sql create mode 100644 backend/tools/example_vcap.json diff --git a/backend/census_historical_migration/README.md b/backend/census_historical_migration/README.md index dafe732121..fd55fbfe27 100644 --- a/backend/census_historical_migration/README.md +++ b/backend/census_historical_migration/README.md @@ -20,7 +20,7 @@ This is implemented as a Django app to leverage existing management commands and - fac_s3.py - Uploads folders or files to an S3 bucket. ```bash -python manage.py fac_s3 gsa-fac-private-s3 --upload --src census_historical_migration/data +python manage.py fac_s3 fac-private-s3 --upload --src census_historical_migration/data ``` - csv_to_postgres.py - Inserts data into Postgres tables using the contents of the CSV files in the S3 bucket. The first row of each file is assumed to have the column names (we convert to lowercase). The name of the table is determined by examining the name of the file. The sample source files do not have delimters for empty fields at the end of a line - so we assume these are nulls. @@ -44,16 +44,16 @@ python manage.py csv_to_postgres --clean True 1. Download test Census data from https://drive.google.com/drive/folders/1TY-7yWsMd8DsVEXvwrEe_oWW1iR2sGoy into census_historical_migration/data folder. NOTE: Never check in the census_historical_migration/data folder into GitHub. -2. In the FAC/backend folder, run the following to load CSV files from census_historical_migration/data folder into gsa-fac-private-s3 bucket. +2. In the FAC/backend folder, run the following to load CSV files from census_historical_migration/data folder into fac-private-s3 bucket. ```bash docker compose run --rm web python manage.py fac_s3 \ - gsa-fac-private-s3 \ + fac-private-s3 \ --upload \ --src census_historical_migration/data ``` -3. In the FAC/backend folder, run the following to read the CSV files from gsa-fac-private-s3 bucket and load into Postgres. +3. In the FAC/backend folder, run the following to read the CSV files from fac-private-s3 bucket and load into Postgres. ```bash docker compose run --rm web python manage.py \ diff --git a/backend/config/settings.py b/backend/config/settings.py index 482f98c77b..cb37c79640 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -254,7 +254,8 @@ CORS_ALLOWED_ORIGINS += ["http://0.0.0.0:8000", "http://127.0.0.1:8000"] # Private bucket - AWS_PRIVATE_STORAGE_BUCKET_NAME = "gsa-fac-private-s3" + AWS_PRIVATE_STORAGE_BUCKET_NAME = "fac-private-s3" + AWS_PUBLIC_STORAGE_BUCKET_NAME = "fac-public-s3" AWS_S3_PRIVATE_REGION_NAME = os.environ.get( "AWS_S3_PRIVATE_REGION_NAME", "us-east-1" diff --git a/backend/cypress/support/dissemination-table-via-pdf.js b/backend/cypress/support/dissemination-table-via-pdf.js new file mode 100644 index 0000000000..e331b61efa --- /dev/null +++ b/backend/cypress/support/dissemination-table-via-pdf.js @@ -0,0 +1,114 @@ +/* + Re-useable code for testing the dissemination table. +*/ + +const API_GOV_JWT = Cypress.env('API_GOV_JWT') || ''; +const API_GOV_KEY = Cypress.env('API_GOV_KEY') || ''; +const API_GOV_USER_ID = Cypress.env('API_GOV_USER_ID'); +const API_GOV_KEY_ADMIN = Cypress.env('API_GOV_KEY_ADMIN'); +const API_GOV_USER_ID_ADMIN = Cypress.env('API_GOV_USER_ID_ADMIN'); +const API_GOV_URL = Cypress.env('API_GOV_URL'); +const API_VERSION = Cypress.env('API_VERSION'); +const ADMIN_API_VERSION = Cypress.env('ADMIN_API_VERSION'); + +const API_GOV_USER_EMAIL = `${API_GOV_USER_ID}@example.com`; + +// We're testing a 2x2. Actually, this would be better as a table, as this is a 3D test. +// is tribal is not tribal +// ┌────────────────────────────────┬────────────────────────────────┐ +// │ │ │ +// │ │ │ +// │ │ │ +// │ │ │ +// public │ UNPRIVILEGED KEY OK │ UNPRIVILEGED KEY OK │ +// │ │ │ +// │ │ │ +// │ │ │ +// │ │ │ +// ├────────────────────────────────┼────────────────────────────────┤ +// │ │ │ +// │ │ │ +// │ │ │ +// │ │ │ +// private │ PRIV KEY OK / UNPRIV NO │ DOES NOT COMPUTE │ +// │ │ │ +// │ │ │ +// │ │ │ +// │ │ │ +// └────────────────────────────────┴────────────────────────────────┘ + +// This could be reworked into a table test. +// However, the tests below should be clear +// enough to not require a full re-working. + +// Where +// T = Tribal(1)/Not Tribal(0) +// P = Private(1)/Public(0) +// U = Privileged(1)/Unprivileged(0) +// L = Length expected + +// T P U L +// ------- +// 0 0 0 1 +// 0 0 1 1 +// 0 1 0 - +// 0 1 1 - +// 1 0 0 1 +// 1 0 1 1 +// 1 1 0 0 +// 1 1 1 1 + + + +export function testSubmissionAccessViaPDF(reportId, isTribal, isPublic) { + console.log(`reportId: ${reportId}, isTribal: ${isTribal}, isPublic: ${isPublic}`); + + // The audit IS tribal and IS public + //////////////////////////////////////// + if (isTribal && isPublic) { + // When it is Tribal and public, we should always + // find the report id in the public and private endpoints + expect(isTribal).to.be.true + expect(isPublic).to.be.true + // We should be able to grab the PDF by URL + // https://app.fac.gov/dissemination/report/pdf/2023-04-GSAFAC-0000050825 + cy.request('HEAD', '/dissemination/report/pdf/' + reportId); + + + } + //////////////////////////////////////// + // The audit IS tribal and IS NOT public + //////////////////////////////////////// + else if (isTribal && !isPublic) { + expect(isTribal).to.be.true + expect(isPublic).to.be.false + cy.request('HEAD', '/dissemination/report/pdf/' + reportId); + } + //////////////////////////////////////// + // The audit IS NOT tribal and IS public + //////////////////////////////////////// + else if (!isTribal && isPublic) { + // This is a standard audit. + expect(isTribal).to.be.false + expect(isPublic).to.be.true + // We should always find it in all endpoints, priv or unpriv. + cy.request('HEAD', '/dissemination/report/pdf/' + reportId); + } + //////////////////////////////////////// + // The audit IS NOT tribal and IS NOT public + // (This is not possible.) + //////////////////////////////////////// + else if (!isTribal && !isPublic) { + console.log("Unreachable test case in testTribalAccess"); + expect(true).to.be.false; + } + //////////////////////////////////////// + // The audit somehow is none of the above. + // (This is not possible.) + //////////////////////////////////////// + else { + // We really should never be here. + console.log("The universe broke in testTribalAccess"); + expect(false).to.be.true; + }; +}; diff --git a/backend/dissemination/sql/SQL_README.md b/backend/dissemination/sql/SQL_README.md new file mode 100644 index 0000000000..97f082bff6 --- /dev/null +++ b/backend/dissemination/sql/SQL_README.md @@ -0,0 +1,49 @@ +# sql folder + +From the perspective of the application, there are both managed and unamanged tables. + +However, we have data and processes that are outside of the application, but inside the databases. + +The SQL folder provides a mechanism for: + +1. Running SQL against the databases *before* we run migrations. +2. Running SQL against the databases *afetr* we run migrations. + +We have two databases. + +1. **fac-db** is the production database. The app talks to this database for all live operations. +2. **fac-snapshot-db**, which began life (and continutes to be used for) snapshots of the production database right before a deploy. Because it wasn't doing anything else, we have co-opted this database to also serve as a "read replica" of the dissemination tables, and host the API. + +## folder organization + +* `fac-db` holds code that operates **on** fac-db before/after migrations. +* `fac-snapshot-db` holds code that operates **on** `fac-snapshot-db` before/after migrations. +* `sling` holds files used by the `sling` application (for moving data between the databases). + +# pre/post + +The `pre` and `post` folders contain SQL files in execution order. That means that the way the files are named matters. + +If the following files are in the `pre` folder: + +`000_first.sql` +`010_nope.SKIP` +`020_second.sql` + +then they will execute in the lexigraphical order as shown. *However*, only files ending in `.sql` will be executed. This means that `000_first.sql` will be executed, `010_nope.SKIP` will be skipped, and `020_second.sql` will be run second. (Although it encourages a dirty tree, we *might* want to keep a file in the tree, but not have it execute.) + +## in practice + +The API is torn down every time we deploy (`pre`). This is because the API has `VIEW`s on the database tables that interfere with migrations. Then, everything in `post` is run, which stands up the API and performs operations that optimize or otherwise improve upon the DB's health/performance. + +# in case it isn't confusing + +* The basic search (in the application) talks to fac-db. +* The advanced search (in the application) talks to fac-snapshot-db. +* The public/read API talks to fac-snapshot-db. +* The *write* portion of tribal access (adding/removing keys) talks to fac-db. +* The *read* portion of tribal access (reading files) talks to fac-snapshot-db. +* The admin API only talks to fac-db. + + + diff --git a/backend/dissemination/sql/fac-snapshot-db/post/001_partitioning.sql b/backend/dissemination/sql/fac-snapshot-db/post/001_partitioning.SKIP similarity index 97% rename from backend/dissemination/sql/fac-snapshot-db/post/001_partitioning.sql rename to backend/dissemination/sql/fac-snapshot-db/post/001_partitioning.SKIP index 026c10fd88..939c45fe1f 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/001_partitioning.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/001_partitioning.SKIP @@ -4,6 +4,13 @@ ------------------------------------------------------ -- python create_partition_statements.py general +-- 20241016 SKIPPED +-- This code demonstrates how to partition the tables that are slung over for +-- use in the API. We do this *post* because it happens after migration, but +-- before we sling data in. +-- This represents a large change, and is not strictly needed right now. +-- But, it serves as a good example of how to do it. + SET search_path TO public_data_v1_0_0; ALTER TABLE public_data_v1_0_0.general RENAME TO general_to_be_removed; CREATE TABLE public_data_v1_0_0.general diff --git a/backend/dissemination/sql/fac-snapshot-db/post/030_admin_api_access_tables.sql b/backend/dissemination/sql/fac-snapshot-db/post/030_admin_api_access_tables.sql deleted file mode 100644 index 18db91956d..0000000000 --- a/backend/dissemination/sql/fac-snapshot-db/post/030_admin_api_access_tables.sql +++ /dev/null @@ -1,43 +0,0 @@ --- This is explicitly not a Django managed table. --- In order to have an administrative key added, --- it must be added via a Github commit, and a PR --- must be performed to merge the key into the tree. - --- This is because administrative keys can read/write --- to some tables in the database. They can read internal and --- in-flight data. - -DROP TABLE IF EXISTS support_administrative_key_uuids; - -CREATE TABLE support_administrative_key_uuids - ( - id BIGSERIAL PRIMARY KEY, - email TEXT, - uuid TEXT, - permissions TEXT, - added DATE - ); - -INSERT INTO support_administrative_key_uuids - (email, uuid, permissions, added) - VALUES - ( - 'matthew.jadud@gsa.gov', - '61ba59b2-f545-4c2f-9b24-9655c706a06c', - 'CREATE,READ,DELETE', - '2023-12-04' - ), - ( - 'daniel.swick@gsa.gov', - 'b6e08808-ecb2-4b6a-b928-46d4205497ff', - 'CREATE,READ,DELETE', - '2023-12-08' - ), - ( - 'fac-gov-test-users+api-tester-admin@gsa.gov', - 'dd60c3f9-053d-4d82-a309-c89da53559f4', - 'CREATE,READ,DELETE', - '2024-07-10' - ) - ; - diff --git a/backend/dissemination/sql/fac-snapshot-db/post/040_admin_api_v1_1_0.sql b/backend/dissemination/sql/fac-snapshot-db/post/040_admin_api_v1_1_0.sql deleted file mode 100644 index 471069cd21..0000000000 --- a/backend/dissemination/sql/fac-snapshot-db/post/040_admin_api_v1_1_0.sql +++ /dev/null @@ -1,615 +0,0 @@ -DO -$do$ -BEGIN - IF EXISTS ( - SELECT FROM pg_catalog.pg_roles - WHERE rolname = 'authenticator') THEN - RAISE NOTICE 'Role "authenticator" already exists. Skipping.'; - ELSE - CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; - END IF; -END -$do$; - -DO -$do$ -BEGIN - IF EXISTS ( - SELECT FROM pg_catalog.pg_roles - WHERE rolname = 'api_fac_gov') THEN - RAISE NOTICE 'Role "api_fac_gov" already exists. Skipping.'; - ELSE - CREATE ROLE api_fac_gov NOLOGIN; - END IF; -END -$do$; - -GRANT api_fac_gov TO authenticator; - -NOTIFY pgrst, 'reload schema'; -begin; - -do -$$ -begin - DROP SCHEMA IF EXISTS admin_api_v1_1_0 CASCADE; - DROP SCHEMA IF EXISTS admin_api_v1_1_0_functions CASCADE; - - if not exists (select schema_name from information_schema.schemata where schema_name = 'admin_api_v1_1_0') then - create schema admin_api_v1_1_0; - create schema admin_api_v1_1_0_functions; - - grant usage on schema admin_api_v1_1_0_functions to api_fac_gov; - - -- Grant access to tables and views - alter default privileges - in schema admin_api_v1_1_0 - grant select - -- this includes views - on tables - to api_fac_gov; - - -- Grant access to sequences, if we have them - grant usage on schema admin_api_v1_1_0 to api_fac_gov; - grant select, usage on all sequences in schema admin_api_v1_1_0 to api_fac_gov; - alter default privileges - in schema admin_api_v1_1_0 - grant select, usage - on sequences - to api_fac_gov; - - -- The admin API needs to be able to write user permissions. - -- This is so we can add and remove people who will have tribal data access - -- via the administrative API. - GRANT INSERT, SELECT, DELETE on public.users_userpermission to api_fac_gov; - -- We need to be able to look up slugs and turn them into permission IDs. - GRANT SELECT on public.users_permission to api_fac_gov; - -- It also needs to be able to log events. - GRANT INSERT on public.support_adminapievent to api_fac_gov; - -- And, it wants to read the UUIDs of administrative keys - GRANT SELECT ON public.support_administrative_key_uuids TO api_fac_gov; - -- We want to see data in flight as admins. - GRANT SELECT ON public.audit_singleauditchecklist TO api_fac_gov; - - GRANT INSERT, SELECT, DELETE on public.dissemination_tribalapiaccesskeyids to api_fac_gov; - GRANT INSERT on public.dissemination_onetimeaccess to api_fac_gov; - end if; -end -$$ -; - -commit; - -notify pgrst, - 'reload schema'; - --- WARNING --- Under PostgreSQL 12, the functions below work. --- Under PostgreSQL 14, these will break. --- --- Note the differences: --- --- raise info 'Works under PostgreSQL 12'; --- raise info 'request.header.x-magic %', (SELECT current_setting('request.header.x-magic', true)); --- raise info 'request.jwt.claim.expires %', (SELECT current_setting('request.jwt.claim.expires', true)); --- raise info 'Works under PostgreSQL 14'; --- raise info 'request.headers::json->>x-magic %', (SELECT current_setting('request.headers', true)::json->>'x-magic'); --- raise info 'request.jwt.claims::json->expires %', (SELECT current_setting('request.jwt.claims', true)::json->>'expires'); --- --- To quote the work of Dav Pilkey, "remember this now." - -begin; - - -CREATE OR REPLACE FUNCTION admin_api_v1_1_0_functions.get_header(item text) RETURNS text - AS $get_header$ - declare res text; - begin - SELECT (current_setting('request.headers', true)::json)->>item into res; - return res; - end; -$get_header$ LANGUAGE plpgsql; - -create or replace function admin_api_v1_1_0_functions.get_api_key_uuid() returns TEXT -as $gaku$ -declare uuid text; -begin - select admin_api_v1_1_0_functions.get_header('x-api-user-id') into uuid; - return uuid; -end; -$gaku$ LANGUAGE plpgsql; - --- log_api_event --- Maintain an internal table of administrative API events. --- Also RAISE INFO so that NR gets a copy. -create or replace function admin_api_v1_1_0_functions.log_admin_api_event(event TEXT, meta JSON) -returns boolean -as $log_admin_api_event$ -DECLARE - uuid_header text; -BEGIN - SELECT admin_api_v1_1_0_functions.get_api_key_uuid() INTO uuid_header; - - INSERT INTO public.support_adminapievent - (api_key_uuid, event, event_data, "timestamp") - VALUES (uuid_header, event, meta, NOW()); - - RAISE INFO 'ADMIN_API % % %', uuid_header, event, meta; - RETURN 1; -END; -$log_admin_api_event$ LANGUAGE plpgsql; - - --- has_admin_data_access :: permission -> bool --- The permissions (insert, select, delete) allow us to have users who can --- read administrative data in addition to users who can (say) update --- select tables like the tribal access lists. -create or replace function admin_api_v1_1_0_functions.has_admin_data_access(perm TEXT) returns boolean -as $has_admin_data_access$ -DECLARE - uuid_header text; - key_exists boolean; - has_permission boolean; -BEGIN - SELECT admin_api_v1_1_0_functions.get_api_key_uuid() INTO uuid_header; - - SELECT - CASE WHEN EXISTS ( - SELECT uuid - FROM public.support_administrative_key_uuids aku - WHERE aku.uuid = uuid_header) - THEN 1::BOOLEAN - ELSE 0::BOOLEAN - END - INTO key_exists; - - SELECT - CASE WHEN EXISTS ( - SELECT permissions - FROM public.support_administrative_key_uuids aku - WHERE aku.uuid = uuid_header - AND aku.permissions like '%' || perm || '%') - THEN 1::BOOLEAN - ELSE 0::BOOLEAN - END - INTO has_permission; - - -- This log event is an INSERT. When called from a VIEW (a SELECT-only context), - -- a call to log_admin_api_event() fails. So, we'll RAISE INFO right here, so we can - -- see the resultse of access checks in the log. We might later comment this out if - -- it becomes too noisy. - RAISE INFO 'ADMIN_API has_access_check % % %', uuid_header, key_exists, has_permission; - - RETURN key_exists AND has_permission; -END; -$has_admin_data_access$ LANGUAGE plpgsql; - --- Takes an email address and, if that address is not in the access table, --- inserts it. If the address already exists, the insert is skipped. --- --- ### Example from REST client --- POST http://localhost:3000/rpc/add_tribal_access_email --- authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} --- content-profile: admin_api_v1_1_0 --- content-type: application/json --- Prefer: params=single-object --- // Not actually a key UUID. --- X-Api-User-Id: 18ef0e72-8976-11ee-ad35-3f80b454d3cc --- { --- "email": "darth.vader@deathstar.org" --- } -create or replace function admin_api_v1_1_0.add_tribal_access_email(params JSON) -returns BOOLEAN -as $add_tribal_access_email$ -DECLARE - already_exists INTEGER; - read_tribal_id INTEGER; -BEGIN - -- If the API user has insert permissions, give it a go - IF admin_api_v1_1_0_functions.has_admin_data_access('CREATE') - THEN - -- Are they already in the table? - SELECT count(up.email) - FROM public.users_userpermission as up - WHERE LOWER(email) = LOWER(params->>'email') INTO already_exists; - - -- If they are, we're going to exit. - IF already_exists <> 0 - THEN - RETURN 0; - END IF; - - -- Grab the permission ID that we need for the insert below. - -- We want the 'read-tribal' permission, which has a human-readable - -- slug. But, we need it's ID, because that is the PK. - SELECT up.id INTO read_tribal_id - FROM public.users_permission AS up - WHERE up.slug = 'read-tribal'; - - IF already_exists = 0 - THEN - -- Can we make the 1 not magic... do a select into. - INSERT INTO public.users_userpermission - (email, permission_id, user_id) - VALUES (LOWER(params->>'email'), read_tribal_id, null); - - RAISE INFO 'ADMIN_API add_tribal_access_email OK %', LOWER(params->>'email'); - RETURN admin_api_v1_1_0_functions.log_admin_api_event('tribal-access-email-added', - json_build_object('email', LOWER(params->>'email'))); - END IF; - ELSE - RETURN 0; - END IF; -end; -$add_tribal_access_email$ LANGUAGE plpgsql; - --- Adds many email addresses. Calls `add_tribal_access_email` for each address. --- --- ### Example from REST client --- POST http://localhost:3000/rpc/add_tribal_access_emails --- authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} --- content-profile: admin_api_v1_1_0 --- content-type: application/json --- Prefer: params=single-object --- // Not actually a key UUID. --- X-Api-User-Id: 18ef0e72-8976-11ee-ad35-3f80b454d3cc --- { --- "emails": [ --- "darth.vader@deathstar.org", --- "bob.darth.vader@deathstar.org", --- "darthy.vader@deathstar.org", --- "bob@deathstar.org" --- ] --- } -CREATE OR REPLACE FUNCTION admin_api_v1_1_0.add_tribal_access_emails(params JSON) -returns BOOLEAN -as $add_tribal_access_emails$ -DECLARE - ele TEXT; - em record; -BEGIN - IF admin_api_v1_1_0_functions.has_admin_data_access('CREATE') - THEN - -- This is a FOR loop over a JSON array in plPgSQL - FOR em IN (SELECT json_array_elements_text((params->>'emails')::JSON) ele) - LOOP - -- PERFORM is how to execute code that does not return anything. - -- If a SELECT was used here, the SQL compiler would complain. - PERFORM admin_api_v1_1_0.add_tribal_access_email(json_build_object('email', em.ele)::JSON); - END LOOP; - RETURN 1; - END IF; - RETURN 0; -END; -$add_tribal_access_emails$ LANGUAGE plpgsql; - --- Removes the email. Will remove multiple rows. That shouldn't happen, but still. --- --- ### Example from REST client --- POST http://localhost:3000/rpc/remove_tribal_access_email --- authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} --- content-profile: admin_api_v1_1_0 --- content-type: application/json --- Prefer: params=single-object --- // Not actually a key UUID. --- X-Api-User-Id: 18ef0e72-8976-11ee-ad35-3f80b454d3cc --- { --- "email": "darth.vader@deathstar.org" --- } -CREATE OR REPLACE FUNCTION admin_api_v1_1_0.remove_tribal_access_email(params JSON) -returns BOOLEAN -as $remove_tribal_access_email$ -DECLARE - affected_rows INTEGER; -BEGIN - - IF admin_api_v1_1_0_functions.has_admin_data_access('DELETE') - THEN - -- Delete rows where the email address matches - DELETE FROM public.users_userpermission as up - WHERE LOWER(up.email) = LOWER(params->>'email'); - -- This is the Postgres way to find out how many rows - -- were affected by a DELETE. - GET DIAGNOSTICS affected_rows = ROW_COUNT; - -- If that is greater than zero, we were successful. - IF affected_rows > 0 - THEN - RETURN admin_api_v1_1_0_functions.log_admin_api_event('tribal-access-email-removed', - json_build_object('email', LOWER(params->>'email'))); - ELSE - RETURN 0; - END IF; - ELSE - -- If we did not have permission, consider it a failure. - RETURN 0; - END IF; -end; -$remove_tribal_access_email$ LANGUAGE plpgsql; - --- Removes many email addresses. Calls `remove_tribal_access_email` for each address. --- --- ### Example from REST client --- POST http://localhost:3000/rpc/remove_tribal_access_emails --- authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} --- content-profile: admin_api_v1_1_0 --- content-type: application/json --- Prefer: params=single-object --- // Not actually a key UUID. --- X-Api-User-Id: 18ef0e72-8976-11ee-ad35-3f80b454d3cc --- { --- "emails": [ --- "darth.vader@deathstar.org", --- "bob.darth.vader@deathstar.org", --- "darthy.vader@deathstar.org", --- "bob@deathstar.org" --- ] --- } -CREATE OR REPLACE FUNCTION admin_api_v1_1_0.remove_tribal_access_emails(params JSON) -returns BOOLEAN -as $remove_tribal_access_emails$ -DECLARE - ele TEXT; - em record; -BEGIN - IF admin_api_v1_1_0_functions.has_admin_data_access('DELETE') - THEN - FOR em IN (SELECT json_array_elements_text((params->>'emails')::JSON) ele) - LOOP - PERFORM admin_api_v1_1_0.remove_tribal_access_email(json_build_object('email', em.ele)::JSON); - END LOOP; - RETURN 1; - END IF; - RETURN 0; -END; -$remove_tribal_access_emails$ LANGUAGE plpgsql; - - - - ---The function below add_tribal_api_key_access adds read access to a tribal API for a specified email. ---It checks if the API user has read permissions. ---Otherwise, it adds the email with 'read-tribal' permission, logs the event, and returns true. - -CREATE OR REPLACE FUNCTION admin_api_v1_1_0.add_tribal_api_key_access(params JSON) -RETURNS JSON -AS $add_tribal_api_key_access$ -DECLARE - user_exists BOOLEAN; -BEGIN - -- If the API user has create permissions, we can proceed - IF admin_api_v1_1_0_functions.has_admin_data_access('CREATE') THEN - -- Check if the user with the given email - SELECT EXISTS ( - SELECT 1 - FROM public.dissemination_TribalApiAccessKeyIds - WHERE LOWER(email) = LOWER(params->>'email') - ) - INTO user_exists; - - -- If the user already exists, it means they have access. - -- For purposes of this function, lets call that "succses", and return true. - IF user_exists THEN - RAISE INFO 'ADMIN_API add_tribal_api_key_access ALREADY_EXISTS %', LOWER(params->>'email'); - RETURN json_build_object( - 'result', 'success', - 'message', 'User with this key already exists')::JSON; - - END IF; - - -- If the user does not exist, add a new record - INSERT INTO public.dissemination_TribalApiAccessKeyIds (email, key_id, date_added) - VALUES (LOWER(params->>'email'), params->>'key_id', CURRENT_TIMESTAMP); - RAISE INFO 'ADMIN_API add_tribal_api_key_access ACCESS_GRANTED % %', LOWER(params->>'email'), params->>'key_id'; - RETURN json_build_object( - 'result', 'success', - 'message', 'User access granted')::JSON; - ELSE - -- If the user does not have CREATE permissions, then we should return a message to that effect. - -- It is a permissions error, but still, we need to know this failed. - RAISE INFO 'ADMIN_API add_tribal_api_key_access ADMIN_LACKS_CREATE'; - RETURN json_build_object( - 'result', 'failure', - 'message', 'Admin user lacks CREATE permissions')::JSON; - END IF; - - -- Return false by default. - RAISE INFO 'ADMIN_API add_tribal_api_key_access WAT %', LOWER(params->>'email'); - RETURN json_build_object( - 'result', 'failure', - 'message', 'Unknown error in access addition')::JSON; -END; -$add_tribal_api_key_access$ LANGUAGE plpgsql; - --- The function below removes tribal API key access for a specified email. --- It checks if the API user has read permissions. --- If the email exists in the database with 'read-tribal' permission, it removes the entry, logs the removal event, and returns true. --- If the email doesn't exist or the user lacks proper permissions, the function returns false. - -CREATE OR REPLACE FUNCTION admin_api_v1_1_0.remove_tribal_api_key_access(params JSON) -RETURNS JSON -AS $remove_tribal_api_key_access$ -DECLARE - user_exists BOOLEAN; -BEGIN - -- If the API user has read permissions, give it a go - IF admin_api_v1_1_0_functions.has_admin_data_access('DELETE') THEN - -- Check if the user with the given email exists - SELECT EXISTS ( - SELECT 1 - FROM public.dissemination_TribalApiAccessKeyIds - WHERE LOWER(email) = LOWER(params->>'email') - ) - INTO user_exists; - - -- If the user exists, remove the record - IF user_exists THEN - DELETE FROM public.dissemination_TribalApiAccessKeyIds - WHERE LOWER(email) = LOWER(params->>'email'); - RAISE INFO 'ADMIN_API remove_tribal_api_key_access ACCESS_REMOVED %', LOWER(params->>'email'); - RETURN json_build_object( - 'result', 'success', - 'message', 'Removed record')::JSON; - ELSE - RAISE INFO 'ADMIN_API remove_tribal_api_key_access DID_NOT_EXIST %', LOWER(params->>'email'); - RETURN json_build_object( - 'result', 'failure', - 'message', 'User did not exist in table')::JSON; - END IF; - ELSE - RAISE INFO 'ADMIN_API remove_tribal_api_key_access ADMIN_LACKS_DELETE'; - RETURN json_build_object( - 'result', 'failure', - 'message', 'Admin user lacks DELETE permissions')::JSON; -- Return false if the API user doesn't have read permissions - END IF; - RAISE INFO 'ADMIN_API add_tribal_api_key_access WAT %', LOWER(params->>'email'); - RETURN json_build_object( - 'result', 'failure', - 'message', 'Uknown error in access removal')::JSON; -END; -$remove_tribal_api_key_access$ LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION admin_api_v1_1_0.request_file_access( - report_id TEXT -) RETURNS JSON LANGUAGE plpgsql AS -$$ -DECLARE - v_uuid_header TEXT; - v_access_uuid VARCHAR(200); - v_key_exists BOOLEAN; - v_key_added_date DATE; -BEGIN - - SELECT admin_api_v1_1_0_functions.get_api_key_uuid() INTO v_uuid_header; - - -- Check if the provided API key exists in public.dissemination_TribalApiAccessKeyIds - SELECT - EXISTS( - SELECT 1 - FROM public.dissemination_TribalApiAccessKeyIds - WHERE key_id = v_uuid_header - ) INTO v_key_exists; - - - -- Get the added date of the key from public.dissemination_TribalApiAccessKeyIds - SELECT date_added - INTO v_key_added_date - FROM public.dissemination_TribalApiAccessKeyIds - WHERE key_id = v_uuid_header; - - - -- Check if the key is less than 6 months old - IF v_uuid_header IS NOT NULL AND v_key_exists AND v_key_added_date >= CURRENT_DATE - INTERVAL '6 months' THEN - -- Generate UUID (using PostgreSQL's gen_random_uuid function) - SELECT gen_random_uuid() INTO v_access_uuid; - - -- Inserting data into the one_time_access table - INSERT INTO public.dissemination_onetimeaccess (uuid, api_key_id, timestamp, report_id) - VALUES (v_access_uuid::UUID, v_uuid_header, CURRENT_TIMESTAMP, report_id); - - -- Return the UUID to the user - RETURN json_build_object('access_uuid', v_access_uuid); - ELSE - -- Return an error for unauthorized access - RETURN json_build_object('error', 'Unauthorized access or key older than 6 months')::JSON; - END IF; -END; -$$; - - - -commit; - -NOTIFY pgrst, 'reload schema'; - -begin; - - ---------------------------------------- --- accesses ---------------------------------------- --- public.audit_access definition - --- Drop table - --- DROP TABLE public.audit_access; - -CREATE OR REPLACE VIEW admin_api_v1_1_0.audit_access AS - SELECT - aa.role, - aa.fullname, - aa.email, - aa.sac_id, - aa.user_id - FROM - public.audit_access aa - WHERE - admin_api_v1_1_0_functions.has_admin_data_access('READ') - ORDER BY aa.id -; - -CREATE OR REPLACE VIEW admin_api_v1_1_0.singleauditchecklist AS - SELECT - sac.id, - sac.date_created, - sac.submission_status, - sac.data_source, - sac.transition_name, - sac.transition_date, - sac.report_id, - sac.audit_type, - sac.general_information, - sac.audit_information, - sac.federal_awards, - sac.corrective_action_plan, - sac.findings_text, - sac.findings_uniform_guidance, - sac.additional_ueis, - sac.additional_eins, - sac.secondary_auditors, - sac.notes_to_sefa, - sac.auditor_certification, - sac.auditee_certification, - sac.tribal_data_consent, - sac.cognizant_agency, - sac.oversight_agency, - sac.submitted_by_id - from - public.audit_singleauditchecklist sac - where - admin_api_v1_1_0_functions.has_admin_data_access('READ') - order by sac.id -; - -CREATE OR REPLACE VIEW admin_api_v1_1_0.tribal_access AS - SELECT - uup.email, - up.slug as permission - FROM - users_userpermission uup, - users_permission up - WHERE - (uup.permission_id = up.id) - AND (up.slug = 'read-tribal') - AND admin_api_v1_1_0_functions.has_admin_data_access('READ') - ORDER BY uup.id -; - -CREATE OR REPLACE VIEW admin_api_v1_1_0.admin_api_events AS - SELECT - ae.timestamp, - ae.api_key_uuid, - ae.event, - ae.event_data - FROM - public.support_adminapievent ae - WHERE - admin_api_v1_1_0_functions.has_admin_data_access('READ') - ORDER BY ae.id -; - - -commit; - -notify pgrst, - 'reload schema'; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/050_admin_api_v1_1_1.sql b/backend/dissemination/sql/fac-snapshot-db/post/050_admin_api_v1_1_1.sql deleted file mode 100644 index c502d76c12..0000000000 --- a/backend/dissemination/sql/fac-snapshot-db/post/050_admin_api_v1_1_1.sql +++ /dev/null @@ -1,572 +0,0 @@ -DO -$do$ -BEGIN - IF EXISTS ( - SELECT FROM pg_catalog.pg_roles - WHERE rolname = 'authenticator') THEN - RAISE NOTICE 'Role "authenticator" already exists. Skipping.'; - ELSE - CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; - END IF; -END -$do$; - -DO -$do$ -BEGIN - IF EXISTS ( - SELECT FROM pg_catalog.pg_roles - WHERE rolname = 'api_fac_gov') THEN - RAISE NOTICE 'Role "api_fac_gov" already exists. Skipping.'; - ELSE - CREATE ROLE api_fac_gov NOLOGIN; - END IF; -END -$do$; - -GRANT api_fac_gov TO authenticator; - -NOTIFY pgrst, 'reload schema'; -begin; - -do -$$ -begin - DROP SCHEMA IF EXISTS admin_api_v1_1_1 CASCADE; - DROP SCHEMA IF EXISTS admin_api_v1_1_1_functions CASCADE; - - if not exists (select schema_name from information_schema.schemata where schema_name = 'admin_api_v1_1_1') then - create schema admin_api_v1_1_1; - create schema admin_api_v1_1_1_functions; - - grant usage on schema admin_api_v1_1_1_functions to api_fac_gov; - - -- Grant access to tables and views - alter default privileges - in schema admin_api_v1_1_1 - grant select - -- this includes views - on tables - to api_fac_gov; - - -- Grant access to sequences, if we have them - grant usage on schema admin_api_v1_1_1 to api_fac_gov; - grant select, usage on all sequences in schema admin_api_v1_1_1 to api_fac_gov; - alter default privileges - in schema admin_api_v1_1_1 - grant select, usage - on sequences - to api_fac_gov; - - -- The admin API needs to be able to write user permissions. - -- This is so we can add and remove people who will have tribal data access - -- via the administrative API. - GRANT INSERT, SELECT, DELETE on public.users_userpermission to api_fac_gov; - -- We need to be able to look up slugs and turn them into permission IDs. - GRANT SELECT on public.users_permission to api_fac_gov; - -- It also needs to be able to log events. - GRANT INSERT on public.support_adminapievent to api_fac_gov; - -- And, it wants to read the UUIDs of administrative keys - GRANT SELECT ON public.support_administrative_key_uuids TO api_fac_gov; - -- We want to see data in flight as admins. - GRANT SELECT ON public.audit_singleauditchecklist TO api_fac_gov; - - GRANT INSERT, SELECT, DELETE on public.dissemination_tribalapiaccesskeyids to api_fac_gov; - GRANT INSERT on public.dissemination_onetimeaccess to api_fac_gov; - end if; -end -$$ -; - -commit; - -notify pgrst, - 'reload schema'; - -begin; - - -CREATE OR REPLACE FUNCTION admin_api_v1_1_1_functions.get_header(item text) RETURNS text - AS $get_header$ - declare res text; - begin - SELECT (current_setting('request.headers', true)::json)->>item into res; - return res; - end; -$get_header$ LANGUAGE plpgsql; - -create or replace function admin_api_v1_1_1_functions.get_api_key_uuid() returns TEXT -as $gaku$ -declare uuid text; -begin - select admin_api_v1_1_1_functions.get_header('x-api-user-id') into uuid; - return uuid; -end; -$gaku$ LANGUAGE plpgsql; - --- log_api_event --- Maintain an internal table of administrative API events. --- Also RAISE INFO so that NR gets a copy. -create or replace function admin_api_v1_1_1_functions.log_admin_api_event(event TEXT, meta JSON) -returns boolean -as $log_admin_api_event$ -DECLARE - uuid_header text; -BEGIN - SELECT admin_api_v1_1_1_functions.get_api_key_uuid() INTO uuid_header; - - INSERT INTO public.support_adminapievent - (api_key_uuid, event, event_data, "timestamp") - VALUES (uuid_header, event, meta, NOW()); - - RAISE INFO 'ADMIN_API % % %', uuid_header, event, meta; - RETURN 1; -END; -$log_admin_api_event$ LANGUAGE plpgsql; - - --- has_admin_data_access :: permission -> bool --- The permissions (insert, select, delete) allow us to have users who can --- read administrative data in addition to users who can (say) update --- select tables like the tribal access lists. -create or replace function admin_api_v1_1_1_functions.has_admin_data_access(perm TEXT) returns boolean -as $has_admin_data_access$ -DECLARE - uuid_header text; - key_exists boolean; - has_permission boolean; -BEGIN - SELECT admin_api_v1_1_1_functions.get_api_key_uuid() INTO uuid_header; - - SELECT - CASE WHEN EXISTS ( - SELECT uuid - FROM public.support_administrative_key_uuids aku - WHERE aku.uuid = uuid_header) - THEN 1::BOOLEAN - ELSE 0::BOOLEAN - END - INTO key_exists; - - SELECT - CASE WHEN EXISTS ( - SELECT permissions - FROM public.support_administrative_key_uuids aku - WHERE aku.uuid = uuid_header - AND aku.permissions like '%' || perm || '%') - THEN 1::BOOLEAN - ELSE 0::BOOLEAN - END - INTO has_permission; - - -- This log event is an INSERT. When called from a VIEW (a SELECT-only context), - -- a call to log_admin_api_event() fails. So, we'll RAISE INFO right here, so we can - -- see the resultse of access checks in the log. We might later comment this out if - -- it becomes too noisy. - RAISE INFO 'ADMIN_API has_access_check % % %', uuid_header, key_exists, has_permission; - - RETURN key_exists AND has_permission; -END; -$has_admin_data_access$ LANGUAGE plpgsql; - --- Takes an email address and, if that address is not in the access table, --- inserts it. If the address already exists, the insert is skipped. --- --- ### Example from REST client --- POST http://localhost:3000/rpc/add_tribal_access_email --- authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} --- content-profile: admin_api_v1_1_1 --- content-type: application/json --- Prefer: params=single-object --- // Not actually a key UUID. --- X-Api-User-Id: 18ef0e72-8976-11ee-ad35-3f80b454d3cc --- { --- "email": "darth.vader@deathstar.org" --- } -create or replace function admin_api_v1_1_1.add_tribal_access_email(params JSON) -returns BOOLEAN -as $add_tribal_access_email$ -DECLARE - already_exists INTEGER; - read_tribal_id INTEGER; -BEGIN - -- If the API user has insert permissions, give it a go - IF admin_api_v1_1_1_functions.has_admin_data_access('CREATE') - THEN - -- Are they already in the table? - SELECT count(up.email) - FROM public.users_userpermission as up - WHERE LOWER(email) = LOWER(params->>'email') INTO already_exists; - - -- If they are, we're going to exit. - IF already_exists <> 0 - THEN - RETURN 0; - END IF; - - -- Grab the permission ID that we need for the insert below. - -- We want the 'read-tribal' permission, which has a human-readable - -- slug. But, we need it's ID, because that is the PK. - SELECT up.id INTO read_tribal_id - FROM public.users_permission AS up - WHERE up.slug = 'read-tribal'; - - IF already_exists = 0 - THEN - -- Can we make the 1 not magic... do a select into. - INSERT INTO public.users_userpermission - (email, permission_id, user_id) - VALUES (LOWER(params->>'email'), read_tribal_id, null); - - RAISE INFO 'ADMIN_API add_tribal_access_email OK %', LOWER(params->>'email'); - RETURN admin_api_v1_1_1_functions.log_admin_api_event('tribal-access-email-added', - json_build_object('email', LOWER(params->>'email'))); - END IF; - ELSE - RETURN 0; - END IF; -end; -$add_tribal_access_email$ LANGUAGE plpgsql; - --- Adds many email addresses. Calls `add_tribal_access_email` for each address. --- --- ### Example from REST client --- POST http://localhost:3000/rpc/add_tribal_access_emails --- authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} --- content-profile: admin_api_v1_1_1 --- content-type: application/json --- Prefer: params=single-object --- // Not actually a key UUID. --- X-Api-User-Id: 18ef0e72-8976-11ee-ad35-3f80b454d3cc --- { --- "emails": [ --- "darth.vader@deathstar.org", --- "bob.darth.vader@deathstar.org", --- "darthy.vader@deathstar.org", --- "bob@deathstar.org" --- ] --- } -CREATE OR REPLACE FUNCTION admin_api_v1_1_1.add_tribal_access_emails(params JSON) -returns BOOLEAN -as $add_tribal_access_emails$ -DECLARE - ele TEXT; - em record; -BEGIN - IF admin_api_v1_1_1_functions.has_admin_data_access('CREATE') - THEN - -- This is a FOR loop over a JSON array in plPgSQL - FOR em IN (SELECT json_array_elements_text((params->>'emails')::JSON) ele) - LOOP - -- PERFORM is how to execute code that does not return anything. - -- If a SELECT was used here, the SQL compiler would complain. - PERFORM admin_api_v1_1_1.add_tribal_access_email(json_build_object('email', em.ele)::JSON); - END LOOP; - RETURN 1; - END IF; - RETURN 0; -END; -$add_tribal_access_emails$ LANGUAGE plpgsql; - --- Removes the email. Will remove multiple rows. That shouldn't happen, but still. --- --- ### Example from REST client --- POST http://localhost:3000/rpc/remove_tribal_access_email --- authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} --- content-profile: admin_api_v1_1_1 --- content-type: application/json --- Prefer: params=single-object --- // Not actually a key UUID. --- X-Api-User-Id: 18ef0e72-8976-11ee-ad35-3f80b454d3cc --- { --- "email": "darth.vader@deathstar.org" --- } -CREATE OR REPLACE FUNCTION admin_api_v1_1_1.remove_tribal_access_email(params JSON) -returns BOOLEAN -as $remove_tribal_access_email$ -DECLARE - affected_rows INTEGER; -BEGIN - - IF admin_api_v1_1_1_functions.has_admin_data_access('DELETE') - THEN - -- Delete rows where the email address matches - DELETE FROM public.users_userpermission as up - WHERE LOWER(up.email) = LOWER(params->>'email'); - -- This is the Postgres way to find out how many rows - -- were affected by a DELETE. - GET DIAGNOSTICS affected_rows = ROW_COUNT; - -- If that is greater than zero, we were successful. - IF affected_rows > 0 - THEN - RETURN admin_api_v1_1_1_functions.log_admin_api_event('tribal-access-email-removed', - json_build_object('email', LOWER(params->>'email'))); - ELSE - RETURN 0; - END IF; - ELSE - -- If we did not have permission, consider it a failure. - RETURN 0; - END IF; -end; -$remove_tribal_access_email$ LANGUAGE plpgsql; - --- Removes many email addresses. Calls `remove_tribal_access_email` for each address. --- --- ### Example from REST client --- POST http://localhost:3000/rpc/remove_tribal_access_emails --- authorization: Bearer {{$processEnv CYPRESS_API_GOV_JWT}} --- content-profile: admin_api_v1_1_1 --- content-type: application/json --- Prefer: params=single-object --- // Not actually a key UUID. --- X-Api-User-Id: 18ef0e72-8976-11ee-ad35-3f80b454d3cc --- { --- "emails": [ --- "darth.vader@deathstar.org", --- "bob.darth.vader@deathstar.org", --- "darthy.vader@deathstar.org", --- "bob@deathstar.org" --- ] --- } -CREATE OR REPLACE FUNCTION admin_api_v1_1_1.remove_tribal_access_emails(params JSON) -returns BOOLEAN -as $remove_tribal_access_emails$ -DECLARE - ele TEXT; - em record; -BEGIN - IF admin_api_v1_1_1_functions.has_admin_data_access('DELETE') - THEN - FOR em IN (SELECT json_array_elements_text((params->>'emails')::JSON) ele) - LOOP - PERFORM admin_api_v1_1_1.remove_tribal_access_email(json_build_object('email', em.ele)::JSON); - END LOOP; - RETURN 1; - END IF; - RETURN 0; -END; -$remove_tribal_access_emails$ LANGUAGE plpgsql; - - - - ---The function below add_tribal_api_key_access adds read access to a tribal API for a specified email. ---It checks if the API user has read permissions. ---Otherwise, it adds the email with 'read-tribal' permission, logs the event, and returns true. - -CREATE OR REPLACE FUNCTION admin_api_v1_1_1.add_tribal_api_key_access(params JSON) -RETURNS JSON -AS $add_tribal_api_key_access$ -DECLARE - user_exists BOOLEAN; -BEGIN - -- If the API user has create permissions, we can proceed - IF admin_api_v1_1_1_functions.has_admin_data_access('CREATE') THEN - -- Check if the user with the given email - SELECT EXISTS ( - SELECT 1 - FROM public.dissemination_TribalApiAccessKeyIds - WHERE LOWER(email) = LOWER(params->>'email') - ) - INTO user_exists; - - -- If the user already exists, it means they have access. - -- For purposes of this function, lets call that "succses", and return true. - IF user_exists THEN - RAISE INFO 'ADMIN_API add_tribal_api_key_access ALREADY_EXISTS %', LOWER(params->>'email'); - RETURN json_build_object( - 'result', 'success', - 'message', 'User with this key already exists')::JSON; - - END IF; - - -- If the user does not exist, add a new record - INSERT INTO public.dissemination_TribalApiAccessKeyIds (email, key_id, date_added) - VALUES (LOWER(params->>'email'), params->>'key_id', CURRENT_TIMESTAMP); - RAISE INFO 'ADMIN_API add_tribal_api_key_access ACCESS_GRANTED % %', LOWER(params->>'email'), params->>'key_id'; - RETURN json_build_object( - 'result', 'success', - 'message', 'User access granted')::JSON; - ELSE - -- If the user does not have CREATE permissions, then we should return a message to that effect. - -- It is a permissions error, but still, we need to know this failed. - RAISE INFO 'ADMIN_API add_tribal_api_key_access ADMIN_LACKS_CREATE'; - RETURN json_build_object( - 'result', 'failure', - 'message', 'Admin user lacks CREATE permissions')::JSON; - END IF; - - -- Return false by default. - RAISE INFO 'ADMIN_API add_tribal_api_key_access WAT %', LOWER(params->>'email'); - RETURN json_build_object( - 'result', 'failure', - 'message', 'Unknown error in access addition')::JSON; -END; -$add_tribal_api_key_access$ LANGUAGE plpgsql; - --- The function below removes tribal API key access for a specified email. --- It checks if the API user has read permissions. --- If the email exists in the database with 'read-tribal' permission, it removes the entry, logs the removal event, and returns true. --- If the email doesn't exist or the user lacks proper permissions, the function returns false. - -CREATE OR REPLACE FUNCTION admin_api_v1_1_1.remove_tribal_api_key_access(params JSON) -RETURNS JSON -AS $remove_tribal_api_key_access$ -DECLARE - user_exists BOOLEAN; -BEGIN - -- If the API user has read permissions, give it a go - IF admin_api_v1_1_1_functions.has_admin_data_access('DELETE') THEN - -- Check if the user with the given email exists - SELECT EXISTS ( - SELECT 1 - FROM public.dissemination_TribalApiAccessKeyIds - WHERE LOWER(email) = LOWER(params->>'email') - ) - INTO user_exists; - - -- If the user exists, remove the record - IF user_exists THEN - DELETE FROM public.dissemination_TribalApiAccessKeyIds - WHERE LOWER(email) = LOWER(params->>'email'); - RAISE INFO 'ADMIN_API remove_tribal_api_key_access ACCESS_REMOVED %', LOWER(params->>'email'); - RETURN json_build_object( - 'result', 'success', - 'message', 'Removed record')::JSON; - ELSE - RAISE INFO 'ADMIN_API remove_tribal_api_key_access DID_NOT_EXIST %', LOWER(params->>'email'); - RETURN json_build_object( - 'result', 'failure', - 'message', 'User did not exist in table')::JSON; - END IF; - ELSE - RAISE INFO 'ADMIN_API remove_tribal_api_key_access ADMIN_LACKS_DELETE'; - RETURN json_build_object( - 'result', 'failure', - 'message', 'Admin user lacks DELETE permissions')::JSON; -- Return false if the API user doesn't have read permissions - END IF; - RAISE INFO 'ADMIN_API add_tribal_api_key_access WAT %', LOWER(params->>'email'); - RETURN json_build_object( - 'result', 'failure', - 'message', 'Uknown error in access removal')::JSON; -END; -$remove_tribal_api_key_access$ LANGUAGE plpgsql; - - -commit; - -NOTIFY pgrst, 'reload schema'; - -begin; - - ---------------------------------------- --- accesses ---------------------------------------- --- public.audit_access definition - --- Drop table - --- DROP TABLE public.audit_access; - -CREATE OR REPLACE VIEW admin_api_v1_1_1.tribal_access AS - SELECT - uup.email, - up.slug as permission - FROM - users_userpermission uup, - users_permission up - WHERE - (uup.permission_id = up.id) - AND (up.slug = 'read-tribal') - AND admin_api_v1_1_1_functions.has_admin_data_access('READ') - ORDER BY uup.id -; - -CREATE OR REPLACE VIEW admin_api_v1_1_1.admin_api_events AS - SELECT - ae.timestamp, - ae.api_key_uuid, - ae.event, - ae.event_data - FROM - public.support_adminapievent ae - WHERE - admin_api_v1_1_1_functions.has_admin_data_access('READ') - ORDER BY ae.id -; - ------------------ --- Expose more of the internal tables for analysis/trouble-shooting. ------------------ -CREATE OR REPLACE VIEW admin_api_v1_1_1.audit_access AS - SELECT * FROM public.audit_access - WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') -; - -CREATE OR REPLACE VIEW admin_api_v1_1_1.audit_deletedaccess AS - SELECT * FROM public.audit_deletedaccess - WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') -; - -CREATE OR REPLACE VIEW admin_api_v1_1_1.audit_excelfile AS - SELECT * FROM public.audit_excelfile - WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') -; - -CREATE OR REPLACE VIEW admin_api_v1_1_1.singleauditchecklist AS - SELECT * - FROM public.audit_singleauditchecklist sac - WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') -; - -CREATE OR REPLACE VIEW admin_api_v1_1_1.audit_sacvalidationwaiver AS - SELECT * FROM public.audit_sacvalidationwaiver - WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') -; - -CREATE OR REPLACE VIEW admin_api_v1_1_1.audit_singleauditreportfile AS - SELECT * FROM public.audit_singleauditreportfile - WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') -; - -CREATE OR REPLACE VIEW admin_api_v1_1_1.audit_submissionevent AS - SELECT * FROM public.audit_submissionevent - WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') -; - -CREATE OR REPLACE VIEW admin_api_v1_1_1.auth_user AS - SELECT * FROM public.auth_user - WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') -; - -CREATE OR REPLACE VIEW admin_api_v1_1_1.django_migrations AS - SELECT * FROM public.django_migrations - WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') -; - -CREATE OR REPLACE VIEW admin_api_v1_1_1.support_adminapievent AS - SELECT * FROM public.support_adminapievent - WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') -; - -CREATE OR REPLACE VIEW admin_api_v1_1_1.support_cognizantassignment AS - SELECT * FROM public.support_cognizantassignment - WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') -; - -CREATE OR REPLACE VIEW admin_api_v1_1_1.support_cognizantbaseline AS - SELECT * FROM public.support_cognizantbaseline - WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') -; - -CREATE OR REPLACE VIEW admin_api_v1_1_1.users_userpermission AS - SELECT * FROM public.users_userpermission - WHERE admin_api_v1_1_1_functions.has_admin_data_access('READ') -; - -commit; - -notify pgrst, - 'reload schema'; diff --git a/backend/dissemination/sql/fac-snapshot-db/pre/020_admin_api_v1_1_0.sql b/backend/dissemination/sql/fac-snapshot-db/pre/020_admin_api_v1_1_0.sql deleted file mode 100644 index e9bbae297e..0000000000 --- a/backend/dissemination/sql/fac-snapshot-db/pre/020_admin_api_v1_1_0.sql +++ /dev/null @@ -1,17 +0,0 @@ - -begin; - -DROP SCHEMA IF EXISTS admin_api_v1_1_0 CASCADE; - -commit; - -notify pgrst, - 'reload schema'; -begin; - - drop view if exists admin_api_v1_1_0.audit_access; - -commit; - -notify pgrst, - 'reload schema'; diff --git a/backend/dissemination/sql/fac-snapshot-db/pre/025_admin_api_access_tables.sql b/backend/dissemination/sql/fac-snapshot-db/pre/025_admin_api_access_tables.sql deleted file mode 100644 index e911176bd9..0000000000 --- a/backend/dissemination/sql/fac-snapshot-db/pre/025_admin_api_access_tables.sql +++ /dev/null @@ -1,2 +0,0 @@ - -DROP TABLE IF EXISTS support_administrative_key_uuids; diff --git a/backend/dissemination/test_views.py b/backend/dissemination/test_views.py index f2f3190bb1..1648cae2ea 100644 --- a/backend/dissemination/test_views.py +++ b/backend/dissemination/test_views.py @@ -671,7 +671,7 @@ def _mock_filename(self): return "some-report-name.xlsx", None def _mock_download_url(self): - return "http://example.com/gsa-fac-private-s3/temp/some-report-name.xlsx" + return "http://example.com/fac-private-s3/temp/some-report-name.xlsx" @patch("dissemination.summary_reports.prepare_workbook_for_download") def test_bad_search_returns_400(self, mock_prepare_workbook_for_download): diff --git a/backend/docker-compose.yml b/backend/docker-compose.yml index 0bb665ae60..29d260faf7 100644 --- a/backend/docker-compose.yml +++ b/backend/docker-compose.yml @@ -62,8 +62,9 @@ services: api: condition: service_started environment: - DATABASE_URL: postgres://postgres@db/postgres - SNAPSHOT_URL: postgres://postgres@db2/postgres + # Should these be mode disable locally? + DATABASE_URL: postgres://postgres@db/postgres?sslmode=disable + SNAPSHOT_URL: postgres://postgres@db2/postgres?sslmode=disable POSTGREST_URL: http://api:3000 DJANGO_DEBUG: true SAM_API_KEY: ${SAM_API_KEY} diff --git a/backend/run.sh b/backend/run.sh index 7e33169f9b..a9206ab8d4 100755 --- a/backend/run.sh +++ b/backend/run.sh @@ -9,6 +9,7 @@ source tools/migrate_app_tables.sh source tools/seed_cog_baseline.sh source tools/setup_env.sh source tools/sling_first_run.sh +source tools/sling_to_sqlite_in_s3.sh source tools/sql_pre_post.sh source tools/util_startup.sh @@ -30,7 +31,7 @@ gonogo "sql_pre" gonogo "migrate_app_tables" ##### -# PREP API TABLES +# MOVE DATA TO SECOND DB # This runs sling and preps tables in the snapshot DB. # Only runs if the tables are not present (e.g. first deploy) sling_first_run diff --git a/backend/tools/example_vcap.json b/backend/tools/example_vcap.json new file mode 100644 index 0000000000..ba6f776b66 --- /dev/null +++ b/backend/tools/example_vcap.json @@ -0,0 +1,201 @@ +{ + "s3": [ + { + "label": "s3", + "provider": "alpha-provider", + "plan": "basic", + "name": "backups", + "tags": [ + "AWS", + "S3", + "object-storage" + ], + "instance_guid": "UUIDALPHA1", + "instance_name": "backups", + "binding_guid": "UUIDALPHA2", + "binding_name": null, + "credentials": { + "uri": "s3://ACCESSKEYIDALPHA:SECRETACCESSKEYALPHA@s3-us-gov-alpha-1.amazonaws.com/BROKEREDBUCKETALPHA", + "insecure_skip_verify": false, + "access_key_id": "ACCESSKEYIDALPHA", + "secret_access_key": "SECRETACCESSKEY+ALPHA", + "region": "us-gov-west-1", + "bucket": "BROKEREDBUCKETALPHA", + "endpoint": "s3-us-gov-alpha-1.amazonaws.com", + "fips_endpoint": "s3-fips.us-gov-alpha-1.amazonaws.com", + "additional_buckets": [] + }, + "syslog_drain_url": "https://ALPHA.drain.url", + "volume_mounts": [ + "no_mounts" + ] + }, + { + "label": "s3", + "provider": null, + "plan": "basic", + "name": "fac-private-s3", + "tags": [ + "AWS", + "S3", + "object-storage" + ], + "instance_guid": "UUIDBRAVO1", + "instance_name": "fac-private-s3", + "binding_guid": "UUIDBRAVO2", + "binding_name": null, + "credentials": { + "uri": "s3://ACCESSKEYIDBRAVO:SECRETACCESSKEYBRAVO@s3-us-gov-bravo-1.amazonaws.com/BROKEREDBUCKETBRAVO1", + "insecure_skip_verify": false, + "access_key_id": "ACCESSKEYIDBRAVO", + "secret_access_key": "SECRETACCESSKEYBRAVO", + "region": "us-gov-bravo-1", + "bucket": "BROKEREDBUCKETBRAVO1", + "endpoint": "s3-us-gov-bravo-1.amazonaws.com", + "fips_endpoint": "s3-fips.us-gov-bravo-1.amazonaws.com", + "additional_buckets": [ + "BROKEREDBUCKETBRAVO2" + ] + }, + "syslog_drain_url": null, + "volume_mounts": [] + }, + { + "label": "s3", + "provider": null, + "plan": "basic", + "name": "fac-public-s3", + "tags": [ + "AWS", + "S3", + "object-storage" + ], + "instance_guid": "UUIDCHARLIE1", + "instance_name": "fac-public-s3", + "binding_guid": "UUIDCHARLIE2", + "binding_name": null, + "credentials": { + "uri": "s3://ACCESSKEYIDCHARLIE:SECRETACCESSKEYCHARLIE@s3-us-gov-charlie-1.amazonaws.com/BROKEREDBUCKETCHARLIE", + "insecure_skip_verify": false, + "access_key_id": "ACCESSKEYIDCHARLIE", + "secret_access_key": "SECRETACCESSKEYCHARLIE", + "region": "us-gov-west-1", + "bucket": "BROKEREDBUCKETCHARLIE", + "endpoint": "s3-us-gov-charlie-1.amazonaws.com", + "fips_endpoint": "s3-fips.us-gov-charlie-1.amazonaws.com", + "additional_buckets": [] + }, + "syslog_drain_url": null, + "volume_mounts": [] + } + ], + "user-provided": [ + { + "label": "user-provided", + "name": "key-service", + "tags": [], + "instance_guid": "UUIDDELTA1", + "instance_name": "key-service", + "binding_guid": "UUIDDELTA2", + "binding_name": null, + "credentials": { + "DJANGO_SECRET_LOGIN_KEY": "DJANGOSECRETKEYDELTA", + "LOGIN_CLIENT_ID": "urn:gov:gsa:openidconnect.profiles:sp:sso:delta:jwt", + "SAM_API_KEY": "APIKEYDELTA", + "SECRET_KEY": "DJANGOSECRETKEYDELTA" + }, + "syslog_drain_url": null, + "volume_mounts": [] + }, + { + "label": "user-provided", + "name": "https-proxy-creds", + "tags": [], + "instance_guid": "UUIDECHO1", + "instance_name": "https-proxy-creds", + "binding_guid": "UUIDECHO2", + "binding_name": null, + "credentials": { + "domain": "echo.egress-https-proxy.apps.internal", + "password": "PASSWORDECHO", + "port": "61443", + "protocol": "https", + "uri": "https://USERNAMEECHO:PASSWORDECHO@echo.egress-https-proxy.apps.internal:61443", + "username": "USERNAMEECHO" + }, + "syslog_drain_url": null, + "volume_mounts": [] + }, + { + "label": "user-provided", + "name": "smtp-proxy-creds", + "tags": [], + "instance_guid": "UUIDFOXTROT1", + "instance_name": "smtp-proxy-creds", + "binding_guid": "UUIDFOXTROT2", + "binding_name": null, + "credentials": { + "domain": "echo.egress-smtp-proxy.apps.internal", + "port": "8080" + }, + "syslog_drain_url": null, + "volume_mounts": [] + }, + { + "label": "user-provided", + "name": "newrelic-creds", + "tags": [ + "newrelic-creds" + ], + "instance_guid": "UUIDGOLF1", + "instance_name": "newrelic-creds", + "binding_guid": "UUIDGOLF2", + "binding_name": null, + "credentials": { + "NEW_RELIC_LICENSE_KEY": "LICENCEKEYGOLF", + "NEW_RELIC_LOGS_ENDPOINT": "https://golf.newrelic.com/log/v1" + }, + "syslog_drain_url": null, + "volume_mounts": [] + }, + { + "label": "user-provided", + "name": "logdrain", + "tags": [], + "instance_guid": "UUIDHOTEL1", + "instance_name": "logdrain", + "binding_guid": "UUIDHOTEL2", + "binding_name": null, + "credentials": null, + "syslog_drain_url": "https://HOTELUSERNAME:HOTELPASSWORD@hotel.app.cloud.gov/?drain-type=all", + "volume_mounts": [] + } + ], + "aws-rds": [ + { + "label": "aws-rds", + "provider": null, + "plan": "medium-gp-psql", + "name": "db", + "tags": [ + "database", + "RDS" + ], + "instance_guid": "UUIDINDIA1", + "instance_name": "db", + "binding_guid": "UUIDINDIA2", + "binding_name": null, + "credentials": { + "db_name": "DBNAMEINDIA", + "host": "host.us-gov-india-1.rds.amazonaws.com", + "name": "DBNAMEINDIA", + "password": "PASSWORDINDIA", + "port": "5432", + "uri": "postgres://USERNAMEINDIA:PASSWORDINDIA@host.us-gov-india-1.rds.amazonaws.com:5432/DBNAMEINDIA", + "username": "USERNAMEINDIA" + }, + "syslog_drain_url": null, + "volume_mounts": [] + } + ] +} diff --git a/backend/tools/setup_cgov_env.sh b/backend/tools/setup_cgov_env.sh index 266d8349f8..9dc60bfa0f 100644 --- a/backend/tools/setup_cgov_env.sh +++ b/backend/tools/setup_cgov_env.sh @@ -2,8 +2,49 @@ source tools/util_startup.sh # Aliases need to be outside of function scope +export _GET_AWS_RESULT="NONE" + +function get_aws_s3() { + local bucket="$1" + local key="$2" + _GET_AWS_RESULT=$(echo $VCAP_SERVICES | \ + jq --arg bucket "$bucket" '.s3 | map(select(.instance_name==$bucket))' | \ + jq .[] | \ + jq --arg key "$key" '.credentials | .[$key]') + return 0 +} + function setup_cgov_env { - set -e + startup_log "CGOV_ENV" "We are in a cloud.gov envirnoment." + + # https://stackoverflow.com/questions/48712545/break-jq-query-string-into-lines + # jq is fine with line breaks in strings. Just don't escape them. + # Makes long queries more readable. Maybe. + export AWS_PRIVATE_BUCKET_NAME=$(echo $VCAP_SERVICES \ + | jq '.s3 + | map(select(.instance_name + | contains("fac-private-s3"))) + | .[] .credentials.bucket') + export AWS_PUBLIC_BUCKET_NAME=$(echo $VCAP_SERVICES \ + | jq '.s3 + | map(select(.instance_name + | contains("fac-public-s3"))) + | .[] .credentials.bucket') + + get_aws_s3 "fac-private-s3" "access_key_id" + export AWS_PRIVATE_ACCESS_KEY_ID=$_GET_AWS_RESULT + get_aws_s3 "fac-private-s3" "secret_access_key" + export AWS_PRIVATE_SECRET_ACCESS_KEY=$_GET_AWS_RESULT + get_aws_s3 "fac-private-s3" "endpoint" + export AWS_S3_PRIVATE_ENDPOINT=$_GET_AWS_RESULT + + get_aws_s3 "fac-public-s3" "access_key_id" + export AWS_PUBLIC_ACCESS_KEY_ID=$_GET_AWS_RESULT + get_aws_s3 "fac-public-s3" "secret_access_key" + export AWS_PUBLIC_SECRET_ACCESS_KEY=$_GET_AWS_RESULT + get_aws_s3 "fac-public-s3" "endpoint" + export AWS_S3_PUBLIC_ENDPOINT=$_GET_AWS_RESULT + export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt diff --git a/backend/tools/setup_local_env.sh b/backend/tools/setup_local_env.sh index 57e332acfc..64c6a685db 100644 --- a/backend/tools/setup_local_env.sh +++ b/backend/tools/setup_local_env.sh @@ -4,18 +4,33 @@ function setup_local_env { if [[ "${ENV}" == "LOCAL" || "${ENV}" == "TESTING" ]]; then startup_log "LOCAL_ENV" "We are in a local envirnoment." - export AWS_PRIVATE_ACCESS_KEY_ID=longtest - export AWS_PRIVATE_SECRET_ACCESS_KEY=longtest - export AWS_S3_PRIVATE_ENDPOINT="http://minio:9000" + + export AWS_PUBLIC_BUCKET_NAME="fac-public-s3" + export AWS_PRIVATE_BUCKET_NAME="fac-public-s3" + + export AWS_PRIVATE_ACCESS_KEY_ID="longtest" + export AWS_PRIVATE_SECRET_ACCESS_KEY="longtest" + export AWS_PRIVATE_ENDPOINT="http://minio:9001" + + export AWS_PUBLIC_ACCESS_KEY_ID="longtest" + export AWS_PUBLIC_SECRET_ACCESS_KEY="longtest" + export AWS_PUBLIC_ENDPOINT="http://minio:9001" + mc alias set myminio "${AWS_S3_PRIVATE_ENDPOINT}" minioadmin minioadmin # Do nothing if the bucket already exists. # https://min.io/docs/minio/linux/reference/minio-mc/mc-mb.html - mc mb --ignore-existing myminio/gsa-fac-private-s3 - mc admin user svcacct add --access-key="${AWS_PRIVATE_ACCESS_KEY_ID}" --secret-key="${AWS_PRIVATE_SECRET_ACCESS_KEY}" myminio minioadmin + mc mb --ignore-existing myminio/fac-private-s3 + mc mb --ignore-existing myminio/fac-public-s3 + + # MCJ 20241016 FIXME: Is this even needed locally? I don't think so. + # mc admin user svcacct add \ + # --access-key="${AWS_PRIVATE_ACCESS_KEY_ID}" \ + # --secret-key="${AWS_PRIVATE_SECRET_ACCESS_KEY}" \ + # myminio minioadmin # For database work - export FAC_DB_URI=${DATABASE_URL}?sslmode=disable - export FAC_SNAPSHOT_URI=${SNAPSHOT_URL}?sslmode=disable + export FAC_DB_URI=${DATABASE_URL} #?sslmode=disable + export FAC_SNAPSHOT_URI=${SNAPSHOT_URL} export PSQL_EXE='psql --single-transaction -v ON_ERROR_STOP=on' export PSQL_EXE_NO_TXN='psql -v ON_ERROR_STOP=on' @@ -29,11 +44,13 @@ function setup_local_env { rm -f sling_linux_amd64.tar.gz chmod +x sling mv sling /bin/sling + # And we need cgov-util curl -L -O https://github.com/GSA-TTS/fac-backup-utility/releases/download/v0.1.8/gov.gsa.fac.cgov-util-v0.1.8-linux-amd64.tar.gz tar xvzf gov.gsa.fac.cgov-util-v0.1.8-linux-amd64.tar.gz gov.gsa.fac.cgov-util chmod 755 gov.gsa.fac.cgov-util mv gov.gsa.fac.cgov-util /bin/cgov-util + # We need a config.json in the directory we are running # things from (or PWD). cp util/load_public_dissem_data/data/config.json . diff --git a/backend/util/api_perf_test/perf.py b/backend/util/api_perf_test/perf.py index db1e8954ad..9c7043082c 100644 --- a/backend/util/api_perf_test/perf.py +++ b/backend/util/api_perf_test/perf.py @@ -5,7 +5,7 @@ import math import json -URI = "http://localhost:3000" +URI = "http://localhost:8080" # GET {{scheme}}://{{apiUrl}}/general?report_id=eq.2021-12-CENSUS-0000250449 # authorization: {{authorization}} @@ -13,17 +13,22 @@ # accept-profile: public_api_v1_0_0 # Accept: application/vnd.pgrst.plan +NOT_ACTUALLY_SECRET = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.zJrV44Lhr1Ck4vg1dMnldql0adLgut241jo0FbFXMlI" +MORE_SECRET = os.getenv("CYPRESS_API_GOV_JWT") +SEKRET = NOT_ACTUALLY_SECRET + def fetch_fa_exp(api_version): total_cost = 0 for offset in range(0, 4000000, 20000): print(f"fetch_fa_exp api {api_version} offset {offset}") - query = f"{URI}/federal_awards?limit=20000&offset={offset}" + query = f"{URI}/federal_awards" # ?limit=20000&offset={offset} headers = { "accept-profile": api_version, "accept": "application/vnd.pgrst.plan+json", "x-api-user-id": os.getenv("API_KEY_ID"), - "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", + "authorization": f"bearer {SEKRET}", + "range": f"{offset}-{offset+19999}", } resp = requests.get(query, headers=headers) @@ -40,15 +45,17 @@ def fetch_fa_by_year_exp(api_version): print( f"fetch_fa_by_year_exp api {api_version} ay {audit_year} offset {offset}" ) - query = f"{URI}/federal_awards?audit_year=eq.{audit_year}&limit=20000&offset={offset}" + query = f"{URI}/federal_awards?audit_year=eq.{audit_year}" # &limit=20000&offset={offset}" headers = { "accept-profile": api_version, "accept": "application/vnd.pgrst.plan+json", "x-api-user-id": os.getenv("API_KEY_ID"), - "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", + "authorization": f"bearer {SEKRET}", + "range": f"{offset}-{offset+19999}", } resp = requests.get(query, headers=headers) + pprint(resp) # We get back a list of one plan, and we want the total cost. total_cost += resp.json()[0]["Plan"]["Total Cost"] return math.floor(total_cost) @@ -61,7 +68,7 @@ def fetch_fa_time(api_version): headers = { "accept-profile": api_version, "x-api-user-id": os.getenv("API_KEY_ID"), - "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", + "authorization": f"bearer {SEKRET}", } t0 = time.time() resp = requests.get(query, headers=headers) @@ -77,11 +84,12 @@ def fetch_fa_time_by_year(api_version): for year in range(16, 24): for offset in range(0, 1000000, 20000): audit_year = f"20{year:02}" - query = f"{URI}/federal_awards?audit_year=eq.{audit_year}&limit=20000&offset={offset}" + query = f"{URI}/federal_awards?audit_year=eq.{audit_year}" # &limit=20000&offset={offset}" headers = { "accept-profile": api_version, "x-api-user-id": os.getenv("API_KEY_ID"), - "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", + "authorization": f"bearer {SEKRET}", + "range": f"{offset}-{offset+19999}", } t0 = time.time() resp = requests.get(query, headers=headers) @@ -102,7 +110,7 @@ def fetch_fa_batches_exp(): "accept-profile": "public_api_v1_0_0", "accept": "application/vnd.pgrst.plan+json", "x-api-user-id": os.getenv("API_KEY_ID"), - "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", + "authorization": f"bearer {SEKRET}", } resp = requests.get(query, headers=headers) @@ -119,7 +127,7 @@ def fetch_fa_batches_time(): headers = { "accept-profile": "public_api_v1_0_0", "x-api-user-id": os.getenv("API_KEY_ID"), - "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", + "authorization": f"bearer {SEKRET}", } t0 = time.time() resp = requests.get(query, headers=headers) @@ -144,11 +152,11 @@ def make_ratios(d1, d2): results3 = {} results4 = {} - results1["api110_by_year"] = fetch_fa_by_year_exp("api_v1_1_0") - results1["ap110"] = fetch_fa_exp("api_v1_1_0") - results1["public100"] = fetch_fa_exp("public_api_v1_0_0") - results1["public100_batches"] = fetch_fa_batches_exp() - results1["public100_by_year"] = fetch_fa_by_year_exp("public_api_v1_0_0") + # results1["api110_by_year"] = fetch_fa_by_year_exp("api_v1_1_0") + # results1["ap110"] = fetch_fa_exp("api_v1_1_0") + # results1["public100"] = fetch_fa_exp("public_api_v1_0_0") + # results1["public100_batches"] = fetch_fa_batches_exp() + # results1["public100_by_year"] = fetch_fa_by_year_exp("public_api_v1_0_0") print("Running timing tests... ~5m") @@ -221,3 +229,40 @@ def make_ratios(d1, d2): # 'public100': 2.17, # 'public100_batches': 1.0, # 'public100_by_year': 1.55} + +# For local testing. +# sqlite-rest serve --auth-token-file test.token --security-allow-table general,federal_awards --db-dsn public-data.sqlite +# export AUTH_TOKEN="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.zJrV44Lhr1Ck4vg1dMnldql0adLgut241jo0FbFXMlI" +# echo -n "topsecret" > test.token + +# SQLite, no indexes +# {'desc': 'EXPLAIN raw'} +# {'desc': 'EXPLAIN ratio'} +# {'ap110': 70, +# 'ap110_by_year': 133, +# 'desc': 'TIME raw', +# 'public100': 69, +# 'public100_batches': 123, +# 'public100_by_year': 130} +# {'ap110': 1.01, +# 'ap110_by_year': 1.93, +# 'desc': 'TIME ratio', +# 'public100': 1.0, +# 'public100_batches': 1.78, +# 'public100_by_year': 1.88} + +# After applying same indexes as PG has. +# {'desc': 'EXPLAIN raw'} +# {'desc': 'EXPLAIN ratio'} +# {'ap110': 70, +# 'ap110_by_year': 72, +# 'desc': 'TIME raw', +# 'public100': 71, +# 'public100_batches': 69, +# 'public100_by_year': 71} +# {'ap110': 1.01, +# 'ap110_by_year': 1.04, +# 'desc': 'TIME ratio', +# 'public100': 1.03, +# 'public100_batches': 1.0, +# 'public100_by_year': 1.03} From a58ebb2bbc40c10e02863a77829a00069f346809 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Thu, 24 Oct 2024 16:25:53 -0400 Subject: [PATCH 36/89] Big update... Probably wanted more incremental commits. This shuffled the API around, and has a pytest for the API that can be run locally against the local stack or the remote/live environments. --- backend/config/settings.py | 2 +- backend/cypress/support/full-submission.js | 5 +- backend/dissemination/sql/SQL_README.md | 7 + backend/dissemination/sql/conftest.py | 10 + .../sql/fac-db/post/999_finalize.sql | 3 + .../sql/fac-db/pre/010_drop_schemas.sql | 9 + .../post/002_define_batch_size.sql | 16 + ...010_api_v1_0_3.sql => 010_api_v1_0_3.SKIP} | 6 + .../post/0144_api_v2_0_0_rpc.sql | 46 ++ .../post/020_public_data_v1_0_0_tables.sql | 655 ++++++++++++++++++ .../021_suppressed_data_v1_0_0_tables.sql | 130 ++++ .../post/030_public_data_v1_0_0_metadata.sql | 91 +++ .../post/060_public_api_v1_0_0.sql | 295 -------- .../post/070_public_data_v1_0_0.sql | 6 - .../fac-snapshot-db/post/080_permissions.sql | 6 +- .../post/120_api_v1_1_0_gate.sql | 28 + .../post/121_api_v1_1_0_create_schema.sql | 3 + .../post/122_api_v1_1_0_functions.sql | 53 ++ ...pi_v1_1_0.sql => 123_api_v1_1_0_views.sql} | 374 +++------- .../post/124_api_v1_1_0_rpc.sql | 46 ++ .../post/129_api_v1_1_0_perms.sql | 5 + .../post/140_api_v2_0_0_gate.sql | 28 + .../post/141_api_v2_0_0_create_schema.sql | 2 + .../post/142_api_v2_0_0_functions.sql | 53 ++ .../post/143_api_v2_0_0_views.sql | 112 +++ .../post/149_api_v2_0_0_perms.sql | 7 + .../fac-snapshot-db/pre/000_api_v1_0_3.sql | 2 - .../fac-snapshot-db/pre/010_api_v1_1_0.sql | 27 - .../pre/{070_schemas.sql => 010_roles.sql} | 7 +- .../pre/030_admin_api_v1_1_1.sql | 3 - .../pre/040_public_api_v1_0_0.sql | 8 - .../pre/050_public_data_v1_0_0.sql | 0 .../sql/fac-snapshot-db/pre/060_schemas.sql | 65 +- backend/dissemination/sql/public.rest | 12 +- .../public_data_v1_0_0.yaml | 150 ++-- .../public_metadata_v1_0_0.yaml | 26 +- backend/dissemination/sql/test_api.py | 172 +++++ backend/docker-compose-db-only.yml | 2 +- backend/docker-compose.yml | 35 +- backend/fac-backup-util.sh | 6 + backend/run.sh | 7 +- backend/tools/remove_api_from_fac_db.sh | 3 - backend/tools/setup_local_env.sh | 21 +- backend/tools/sling_first_run.sh | 2 +- backend/util/api_perf_test/perf.py | 48 +- backend/util/fac-backup-and-sling.sh | 106 --- terraform/shared/modules/env/postgrest.tf | 2 +- 47 files changed, 1810 insertions(+), 892 deletions(-) create mode 100644 backend/dissemination/sql/conftest.py create mode 100644 backend/dissemination/sql/fac-db/post/999_finalize.sql create mode 100644 backend/dissemination/sql/fac-db/pre/010_drop_schemas.sql create mode 100644 backend/dissemination/sql/fac-snapshot-db/post/002_define_batch_size.sql rename backend/dissemination/sql/fac-snapshot-db/post/{010_api_v1_0_3.sql => 010_api_v1_0_3.SKIP} (97%) create mode 100644 backend/dissemination/sql/fac-snapshot-db/post/0144_api_v2_0_0_rpc.sql create mode 100644 backend/dissemination/sql/fac-snapshot-db/post/020_public_data_v1_0_0_tables.sql create mode 100644 backend/dissemination/sql/fac-snapshot-db/post/021_suppressed_data_v1_0_0_tables.sql create mode 100644 backend/dissemination/sql/fac-snapshot-db/post/030_public_data_v1_0_0_metadata.sql delete mode 100644 backend/dissemination/sql/fac-snapshot-db/post/060_public_api_v1_0_0.sql delete mode 100644 backend/dissemination/sql/fac-snapshot-db/post/070_public_data_v1_0_0.sql create mode 100644 backend/dissemination/sql/fac-snapshot-db/post/120_api_v1_1_0_gate.sql create mode 100644 backend/dissemination/sql/fac-snapshot-db/post/121_api_v1_1_0_create_schema.sql create mode 100644 backend/dissemination/sql/fac-snapshot-db/post/122_api_v1_1_0_functions.sql rename backend/dissemination/sql/fac-snapshot-db/post/{020_api_v1_1_0.sql => 123_api_v1_1_0_views.sql} (57%) create mode 100644 backend/dissemination/sql/fac-snapshot-db/post/124_api_v1_1_0_rpc.sql create mode 100644 backend/dissemination/sql/fac-snapshot-db/post/129_api_v1_1_0_perms.sql create mode 100644 backend/dissemination/sql/fac-snapshot-db/post/140_api_v2_0_0_gate.sql create mode 100644 backend/dissemination/sql/fac-snapshot-db/post/141_api_v2_0_0_create_schema.sql create mode 100644 backend/dissemination/sql/fac-snapshot-db/post/142_api_v2_0_0_functions.sql create mode 100644 backend/dissemination/sql/fac-snapshot-db/post/143_api_v2_0_0_views.sql create mode 100644 backend/dissemination/sql/fac-snapshot-db/post/149_api_v2_0_0_perms.sql delete mode 100644 backend/dissemination/sql/fac-snapshot-db/pre/000_api_v1_0_3.sql delete mode 100644 backend/dissemination/sql/fac-snapshot-db/pre/010_api_v1_1_0.sql rename backend/dissemination/sql/fac-snapshot-db/pre/{070_schemas.sql => 010_roles.sql} (87%) delete mode 100644 backend/dissemination/sql/fac-snapshot-db/pre/030_admin_api_v1_1_1.sql delete mode 100644 backend/dissemination/sql/fac-snapshot-db/pre/040_public_api_v1_0_0.sql delete mode 100644 backend/dissemination/sql/fac-snapshot-db/pre/050_public_data_v1_0_0.sql create mode 100644 backend/dissemination/sql/test_api.py delete mode 100644 backend/tools/remove_api_from_fac_db.sh delete mode 100755 backend/util/fac-backup-and-sling.sh diff --git a/backend/config/settings.py b/backend/config/settings.py index cb37c79640..52465e5509 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -263,7 +263,7 @@ # MinIO only matters for local development and GitHub action environments. # These should match what we're setting in backend/run.sh - AWS_PRIVATE_ACCESS_KEY_ID = os.environ.get("AWS_PRIVATE_ACCESS_KEY_ID", "longtest") + AWS_PRIVATE_ACCESS_KEY_ID = os.environ.get("AWS_PRIVATE_ACCESS_KEY_ID", "nutnutnut") AWS_PRIVATE_SECRET_ACCESS_KEY = os.environ.get( "AWS_PRIVATE_SECRET_ACCESS_KEY", "longtest" ) diff --git a/backend/cypress/support/full-submission.js b/backend/cypress/support/full-submission.js index 4fb304e486..38aded83c0 100644 --- a/backend/cypress/support/full-submission.js +++ b/backend/cypress/support/full-submission.js @@ -5,7 +5,7 @@ import { testAuditInformationForm } from './audit-info-form.js'; import { testPdfAuditReport } from './report-pdf.js'; import { testAuditorCertification } from './auditor-certification.js'; import { testAuditeeCertification } from './auditee-certification.js'; -import { testSubmissionAccess } from './dissemination-table.js'; +import { testSubmissionAccessViaPDF } from './dissemination-table-via-pdf.js'; import { testTribalAuditPublic, testTribalAuditPrivate } from './tribal-audit-form.js'; import { testInitializeAudit } from './initialize-audit.js'; import { testUnlock } from './unlock-cert.js'; @@ -133,7 +133,8 @@ export function testFullSubmission(isTribal, isPublic) { /audits are complete/ ).siblings().contains('td', reportId); - testSubmissionAccess(reportId, isTribal, isPublic); + // FIXME Ticketed as + testSubmissionAccessViaPDF(reportId, isTribal, isPublic); }); testLogoutGov(); diff --git a/backend/dissemination/sql/SQL_README.md b/backend/dissemination/sql/SQL_README.md index 97f082bff6..816da9f82c 100644 --- a/backend/dissemination/sql/SQL_README.md +++ b/backend/dissemination/sql/SQL_README.md @@ -45,5 +45,12 @@ The API is torn down every time we deploy (`pre`). This is because the API has ` * The *read* portion of tribal access (reading files) talks to fac-snapshot-db. * The admin API only talks to fac-db. +# running tests +``` +pytest -s --env local test_api.py +``` +where the env can be `local`, `preview`, `dev`, `staging`, or `production` to run queries against the API in any of those environments. + +`FAC_API_KEY` and `CYPRESS_API_GOV_JWT` need to be set in the environment for this script to work. diff --git a/backend/dissemination/sql/conftest.py b/backend/dissemination/sql/conftest.py new file mode 100644 index 0000000000..baf4dcd4e0 --- /dev/null +++ b/backend/dissemination/sql/conftest.py @@ -0,0 +1,10 @@ +def pytest_addoption(parser): + parser.addoption("--env", action="store", default="local") + + +def pytest_generate_tests(metafunc): + # This is called for every test. Only get/set command line arguments + # if the argument is specified in the list of test "fixturenames". + option_value = metafunc.config.option.env + if "env" in metafunc.fixturenames and option_value is not None: + metafunc.parametrize("env", [option_value]) diff --git a/backend/dissemination/sql/fac-db/post/999_finalize.sql b/backend/dissemination/sql/fac-db/post/999_finalize.sql new file mode 100644 index 0000000000..d1168e7a39 --- /dev/null +++ b/backend/dissemination/sql/fac-db/post/999_finalize.sql @@ -0,0 +1,3 @@ +-- PostgREST likes to know when the schemas and things +-- attached to them change. +NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/fac-db/pre/010_drop_schemas.sql b/backend/dissemination/sql/fac-db/pre/010_drop_schemas.sql new file mode 100644 index 0000000000..ac2a7d582e --- /dev/null +++ b/backend/dissemination/sql/fac-db/pre/010_drop_schemas.sql @@ -0,0 +1,9 @@ +-- On fac-db, we can drop all the schemas that we're no longer +-- serving off of this database. +-- It is a kind of "spring cleaning," and it is OK if it continues to +-- run for the forseeable future. +DROP SCHEMA IF EXISTS api_v1_0_3 CASCADE; +DROP SCHEMA IF EXISTS api_v1_1_0 CASCADE; +DROP SCHEMA IF EXISTS api_v1_1_1 CASCADE; +DROP SCHEMA IF EXISTS admin_api_v1_1_0 CASCADE; +DROP SCHEMA IF EXISTS admin_api_v1_1_1 CASCADE; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/002_define_batch_size.sql b/backend/dissemination/sql/fac-snapshot-db/post/002_define_batch_size.sql new file mode 100644 index 0000000000..d6d5aee111 --- /dev/null +++ b/backend/dissemination/sql/fac-snapshot-db/post/002_define_batch_size.sql @@ -0,0 +1,16 @@ +-- The batch_size is used to determine how many rows of data +-- are available in a single download. The PostgREST limit is +-- (as of 20241024) set to 20K rows. Howwever, we could make our +-- batches smaller. (This doesn't make things better, BTW.) So, +-- we set our batch size to the same size as the PgREST limit. +-- As a result, public tables are created with a batch number column, +-- and that batch number is incremented as DIV(n, batch_size()). +-- +-- This is defined as a function because there is no good way to define +-- a constant in Postgres/SQL. +-- https://stackoverflow.com/questions/13316773/is-there-a-way-to-define-a-named-constant-in-a-postgresql-query + +CREATE OR REPLACE FUNCTION public.batch_size() + RETURNS INT + LANGUAGE sql IMMUTABLE PARALLEL SAFE AS +'SELECT 20000'; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/010_api_v1_0_3.sql b/backend/dissemination/sql/fac-snapshot-db/post/010_api_v1_0_3.SKIP similarity index 97% rename from backend/dissemination/sql/fac-snapshot-db/post/010_api_v1_0_3.sql rename to backend/dissemination/sql/fac-snapshot-db/post/010_api_v1_0_3.SKIP index 4a7f33c993..da5175afd6 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/010_api_v1_0_3.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/010_api_v1_0_3.SKIP @@ -1,3 +1,9 @@ +-- 20241024 +-- The 1.0.3 API just plain returns wrong data at this point. +-- Although it may break some clients, we are going to retire it rapidly. +-- Version 1.1.0 has all the same fields (plus 1-2 more), and it correctly +-- publishes all data, as opposed to suppressing all Tribal/Tribal org data. + ------------------------------------------------------------------ -- GATE ------------------------------------------------------------------ diff --git a/backend/dissemination/sql/fac-snapshot-db/post/0144_api_v2_0_0_rpc.sql b/backend/dissemination/sql/fac-snapshot-db/post/0144_api_v2_0_0_rpc.sql new file mode 100644 index 0000000000..5b7c925678 --- /dev/null +++ b/backend/dissemination/sql/fac-snapshot-db/post/0144_api_v2_0_0_rpc.sql @@ -0,0 +1,46 @@ +CREATE OR REPLACE FUNCTION api_v2_0_0.request_file_access( + report_id TEXT +) RETURNS JSON LANGUAGE plpgsql AS +$$ +DECLARE + v_uuid_header TEXT; + v_access_uuid VARCHAR(200); + v_key_exists BOOLEAN; + v_key_added_date DATE; +BEGIN + + SELECT api_v2_0_0_functions.get_api_key_uuid() INTO v_uuid_header; + + -- Check if the provided API key exists in public.dissemination_TribalApiAccessKeyIds + SELECT + EXISTS( + SELECT 1 + FROM public.dissemination_tribalapiaccesskeyids + WHERE key_id = v_uuid_header + ) INTO v_key_exists; + + + -- Get the added date of the key from public.dissemination_TribalApiAccessKeyIds + SELECT date_added + INTO v_key_added_date + FROM public.dissemination_tribalapiaccesskeyids + WHERE key_id = v_uuid_header; + + + -- Check if the key is less than 6 months old + IF v_uuid_header IS NOT NULL AND v_key_exists AND v_key_added_date >= CURRENT_DATE - INTERVAL '6 months' THEN + -- Generate UUID (using PostgreSQL's gen_random_uuid function) + SELECT gen_random_uuid() INTO v_access_uuid; + + -- Inserting data into the one_time_access table + INSERT INTO public.dissemination_onetimeaccess (uuid, api_key_id, timestamp, report_id) + VALUES (v_access_uuid::UUID, v_uuid_header, CURRENT_TIMESTAMP, report_id); + + -- Return the UUID to the user + RETURN json_build_object('access_uuid', v_access_uuid); + ELSE + -- Return an error for unauthorized access + RETURN json_build_object('error', 'Unauthorized access or key older than 6 months')::JSON; + END IF; +END; +$$; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/020_public_data_v1_0_0_tables.sql b/backend/dissemination/sql/fac-snapshot-db/post/020_public_data_v1_0_0_tables.sql new file mode 100644 index 0000000000..1ab1ae2c4c --- /dev/null +++ b/backend/dissemination/sql/fac-snapshot-db/post/020_public_data_v1_0_0_tables.sql @@ -0,0 +1,655 @@ +----------------------------------------------------------- +-- general +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_general() + RETURNS VOID +AS +$ct$ +BEGIN + CREATE TABLE public_data_v1_0_0.general AS + SELECT + gen.id as id, + NEXTVAL('public_data_v1_0_0.seq_v1_0_0_general') AS seq, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.agencies_with_prior_findings, + gen.audit_period_covered, + gen.audit_type, + gen.auditee_address_line_1, + gen.auditee_certified_date, + gen.auditee_certify_name, + gen.auditee_certify_title, + gen.auditee_city, + gen.auditee_contact_name, + gen.auditee_contact_title, + gen.auditee_ein, + gen.auditee_email, + gen.auditee_name, + gen.auditee_phone, + gen.auditee_state, + gen.auditee_zip, + gen.auditor_address_line_1, + gen.auditor_certified_date, + gen.auditor_certify_name, + gen.auditor_certify_title, + gen.auditor_city, + gen.auditor_contact_name, + gen.auditor_contact_title, + gen.auditor_country, + gen.auditor_ein, + gen.auditor_email, + gen.auditor_firm_name, + gen.auditor_foreign_address, + gen.auditor_phone, + gen.auditor_state, + gen.auditor_zip, + gen.cognizant_agency, + gen.data_source, + gen.date_created, + gen.dollar_threshold, + gen.entity_type, + gen.fac_accepted_date, + gen.fy_end_date, + gen.fy_start_date, + gen.gaap_results, + gen.is_additional_ueis, + gen.is_aicpa_audit_guide_included, + gen.is_going_concern_included, + gen.is_internal_control_deficiency_disclosed, + gen.is_internal_control_material_weakness_disclosed, + gen.is_low_risk_auditee, + gen.is_material_noncompliance_disclosed, + CASE EXISTS + ( + SELECT + ein.report_id + FROM + dissemination_additionalein ein + WHERE + ein.report_id = gen.report_id + ) + WHEN FALSE THEN 'No' + ELSE 'Yes' + END AS is_multiple_eins, + gen.is_public, + CASE EXISTS + ( + SELECT + aud.report_id + FROM + dissemination_secondaryauditor aud + WHERE + aud.report_id = gen.report_id + ) + WHEN FALSE THEN 'No' + ELSE 'Yes' + END AS is_secondary_auditors, + gen.is_sp_framework_required, + gen.number_months, + gen.oversight_agency, + gen.ready_for_certification_date, + gen.sp_framework_basis, + gen.sp_framework_opinions, + gen.submitted_date, + gen.total_amount_expended, + gen.type_audit_code + FROM + public.dissemination_general gen + ORDER BY gen.id; + + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.general + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.general SET batch_number=DIV(seq, public.batch_size()); +END +$ct$ +LANGUAGE plpgsql; + +----------------------------------------------------------- +-- addition_eins +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_additional_eins() + RETURNS VOID +AS +$ct$ +BEGIN + CREATE TABLE public_data_v1_0_0.additional_eins AS + SELECT + ein.id as id, + NEXTVAL('public_data_v1_0_0.seq_v1_0_0_additional_eins') AS seq, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + ein.additional_ein + FROM + public_data_v1_0_0.general gen, + public.dissemination_additionalein ein + WHERE + gen.report_id = ein.report_id + ORDER BY ein.id; + + ALTER TABLE public_data_v1_0_0.additional_eins + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.additional_eins SET batch_number=DIV(seq, public.batch_size()); +END +$ct$ +LANGUAGE plpgsql; + + +----------------------------------------------------------- +-- additional_ueis +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_additional_ueis() + RETURNS VOID +AS +$ct$ +BEGIN + CREATE TABLE public_data_v1_0_0.additional_ueis AS + SELECT + uei.id as id, + NEXTVAL('public_data_v1_0_0.seq_v1_0_0_additional_ueis') AS seq, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + uei.additional_uei + FROM + public_data_v1_0_0.general gen, + public.dissemination_additionaluei uei + WHERE + gen.report_id = uei.report_id + ORDER BY uei.id; + + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.additional_ueis + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.additional_ueis SET batch_number=DIV(seq, public.batch_size()); +END +$ct$ +LANGUAGE plpgsql; + +----------------------------------------------------------- +-- corrective_action_plans +----------------------------------------------------------- + +CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_corrective_action_plans() + RETURNS VOID +AS +$ct$ +BEGIN + CREATE TABLE public_data_v1_0_0.corrective_action_plans AS + SELECT + cap.id as id, + NEXTVAL('public_data_v1_0_0.seq_v1_0_0_corrective_action_plans') as seq, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + cap.contains_chart_or_table, + cap.finding_ref_number, + cap.planned_action + FROM + public_data_v1_0_0.general gen, + public.dissemination_captext cap + WHERE + cap.report_id = gen.report_id + AND + -- Only include the public corrective action plans. + gen.is_public = true + ORDER BY cap.id; + + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.corrective_action_plans + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.corrective_action_plans SET batch_number=DIV(seq, public.batch_size()); +END +$ct$ +LANGUAGE plpgsql; + +----------------------------------------------------------- +-- federal_awards +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_federal_awards() + RETURNS VOID +AS +$ct$ +BEGIN + CREATE TABLE public_data_v1_0_0.federal_awards AS + SELECT + award.id as id, + NEXTVAL('public_data_v1_0_0.seq_v1_0_0_federal_awards') as seq, + dg.report_id, + dg.auditee_uei, + dg.audit_year, + dg.fac_accepted_date, + concat(award.federal_agency_prefix,'.',award.federal_award_extension) as aln, + --- + award.additional_award_identification, + award.amount_expended, + award.audit_report_type, + award.award_reference, + award.cluster_name, + award.cluster_total, + award.federal_agency_prefix, + award.federal_award_extension, + award.federal_program_name, + award.federal_program_total, + award.findings_count, + award.is_direct, + award.is_loan, + award.is_major, + award.is_passthrough_award, + award.loan_balance, + award.other_cluster_name, + award.passthrough_amount, + award.state_cluster_name + FROM + public_data_v1_0_0.general dg, + public.dissemination_federalaward award + WHERE + award.report_id = dg.report_id + ORDER BY award.id; + + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.federal_awards + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.federal_awards SET batch_number=DIV(seq, public.batch_size()); +END +$ct$ +LANGUAGE plpgsql; + +----------------------------------------------------------- +-- findings +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_findings() + RETURNS VOID +AS +$ct$ +BEGIN + CREATE TABLE public_data_v1_0_0.findings AS + SELECT + finding.id as id, + NEXTVAL('public_data_v1_0_0.seq_v1_0_0_findings') as seq, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + finding.award_reference, + finding.is_material_weakness, + finding.is_modified_opinion, + finding.is_other_findings, + finding.is_other_matters, + finding.is_questioned_costs, + finding.is_repeat_finding, + finding.is_significant_deficiency, + finding.prior_finding_ref_numbers, + finding.reference_number, + finding.type_requirement + FROM + public_data_v1_0_0.general gen, + public.dissemination_finding finding + WHERE + finding.report_id = gen.report_id + ORDER BY finding.id; + + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.findings + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.findings SET batch_number=DIV(seq, public.batch_size()); +END +$ct$ +LANGUAGE plpgsql; + +----------------------------------------------------------- +-- findings_text +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_findings_text() + RETURNS VOID +AS +$ct$ +BEGIN + CREATE TABLE public_data_v1_0_0.findings_text AS + SELECT + ft.id as id, + NEXTVAL('public_data_v1_0_0.seq_v1_0_0_findings_text') as seq, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + ft.finding_ref_number, + ft.contains_chart_or_table, + ft.finding_text + FROM + public_data_v1_0_0.general gen, + public.dissemination_findingtext ft + WHERE + ft.report_id = gen.report_id + AND + gen.is_public = true + ORDER BY ft.id; + + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.findings_text + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.findings_text SET batch_number=DIV(seq, public.batch_size()); +END +$ct$ +LANGUAGE plpgsql; + +----------------------------------------------------------- +-- notes_to_sefa +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_notes_to_sefa() + RETURNS VOID +AS +$ct$ +BEGIN + CREATE TABLE public_data_v1_0_0.notes_to_sefa AS + SELECT + note.id as id, + NEXTVAL('public_data_v1_0_0.seq_v1_0_0_notes_to_sefa') as seq, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + note.accounting_policies, + note.contains_chart_or_table, + note.content, + note.is_minimis_rate_used, + note.note_title as title, + note.rate_explained + FROM + public_data_v1_0_0.general gen, + public.dissemination_note note + WHERE + note.report_id = gen.report_id + AND + -- Some notes are not public. + gen.is_public = true + ORDER BY note.id; + + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.notes_to_sefa + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.notes_to_sefa SET batch_number=DIV(seq, public.batch_size()); +END +$ct$ +LANGUAGE plpgsql; + +----------------------------------------------------------- +-- passthrough +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_passthrough() + RETURNS VOID +AS +$ct$ +BEGIN + CREATE TABLE public_data_v1_0_0.passthrough AS + SELECT + pass.id as id, + NEXTVAL('public_data_v1_0_0.seq_v1_0_0_passthrough') as seq, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + pass.award_reference, + pass.passthrough_id, + pass.passthrough_name + FROM + public_data_v1_0_0.general gen, + public.dissemination_passthrough pass + WHERE + gen.report_id = pass.report_id + ORDER BY pass.id; + + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.passthrough + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.passthrough SET batch_number=DIV(seq, public.batch_size()); +END +$ct$ +LANGUAGE plpgsql; + +----------------------------------------------------------- +-- secondary_auditors +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_secondary_auditors() + RETURNS VOID +AS +$ct$ +BEGIN + CREATE TABLE public_data_v1_0_0.secondary_auditors AS + SELECT + sa.id as id, + NEXTVAL('public_data_v1_0_0.seq_v1_0_0_secondary_auditors') as seq, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + sa.address_city, + sa.address_state, + sa.address_street, + sa.address_zipcode, + sa.auditor_ein, + sa.auditor_name, + sa.contact_email, + sa.contact_name, + sa.contact_phone, + sa.contact_title + FROM + public_data_v1_0_0.general gen, + public.dissemination_secondaryauditor sa + WHERE + sa.report_id = gen.report_id + ORDER BY sa.id; + + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.secondary_auditors + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.secondary_auditors SET batch_number=DIV(seq, public.batch_size()); +END +$ct$ +LANGUAGE plpgsql; + +----------------------------------------------------------- +-- combined +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_combined() + RETURNS VOID +AS +$ct$ +BEGIN + CREATE TABLE public_data_v1_0_0.combined AS + SELECT + dg.report_id, + NEXTVAL('public_data_v1_0_0.seq_v1_0_0_combined') as seq, + dfa.award_reference, + df.reference_number, + concat(dfa.federal_agency_prefix,'.',dfa.federal_award_extension) as aln, + -- + -- general + -- + dg.id as general_row_id, + dg.auditee_uei, + dg.audit_year, + dg.agencies_with_prior_findings, + dg.audit_period_covered, + dg.audit_type, + dg.auditee_address_line_1, + dg.auditee_certified_date, + dg.auditee_certify_name, + dg.auditee_certify_title, + dg.auditee_city, + dg.auditee_contact_name, + dg.auditee_contact_title, + dg.auditee_ein, + dg.auditee_email, + dg.auditee_name, + dg.auditee_phone, + dg.auditee_state, + dg.auditee_zip, + dg.auditor_address_line_1, + dg.auditor_certified_date, + dg.auditor_certify_name, + dg.auditor_certify_title, + dg.auditor_city, + dg.auditor_contact_name, + dg.auditor_contact_title, + dg.auditor_country, + dg.auditor_ein, + dg.auditor_email, + dg.auditor_firm_name, + dg.auditor_foreign_address, + dg.auditor_phone, + dg.auditor_state, + dg.auditor_zip, + dg.cognizant_agency, + dg.data_source, + dg.date_created, + dg.dollar_threshold, + dg.entity_type, + dg.fac_accepted_date, + dg.fy_end_date, + dg.fy_start_date, + dg.gaap_results, + dg.is_additional_ueis, + dg.is_aicpa_audit_guide_included, + dg.is_going_concern_included, + dg.is_internal_control_deficiency_disclosed, + dg.is_internal_control_material_weakness_disclosed, + dg.is_low_risk_auditee, + dg.is_material_noncompliance_disclosed, + dg.is_multiple_eins, + dg.is_public, + dg.is_secondary_auditors, + dg.is_sp_framework_required, + dg.number_months, + dg.oversight_agency, + dg.ready_for_certification_date, + dg.sp_framework_basis, + dg.sp_framework_opinions, + dg.submitted_date, + dg.total_amount_expended, + dg.type_audit_code, + -- + -- federal_award + -- + dfa.id as federal_award_row_id, + dfa.additional_award_identification, + dfa.amount_expended, + dfa.audit_report_type, + dfa.cluster_name, + dfa.cluster_total, + dfa.federal_agency_prefix, + dfa.federal_award_extension, + dfa.federal_program_name, + dfa.federal_program_total, + dfa.findings_count, + dfa.is_direct, + dfa.is_loan, + dfa.is_major, + dfa.is_passthrough_award, + dfa.loan_balance, + dfa.other_cluster_name, + dfa.passthrough_amount, + dfa.state_cluster_name, + -- + -- finding + -- + df.id as finding_row_id, + df.is_material_weakness, + df.is_modified_opinion, + df.is_other_findings, + df.is_other_matters, + df.is_questioned_costs, + df.is_repeat_finding, + df.is_significant_deficiency, + df.prior_finding_ref_numbers, + df.type_requirement, + -- + -- passthrough + -- + dp.id as passthrough_row_id, + dp.passthrough_id, + dp.passthrough_name + FROM + public.dissemination_federalaward dfa + LEFT JOIN public_data_v1_0_0.general dg + ON dfa.report_id = dg.report_id + LEFT JOIN public.dissemination_finding df + ON dfa.report_id = df.report_id + AND dfa.award_reference = df.award_reference + LEFT JOIN public.dissemination_passthrough dp + ON dfa.report_id = dp.report_id + AND dfa.award_reference = dp.award_reference + ORDER BY seq + -- FIXME Do not leave this limit in place. + LIMIT 1; + + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.combined + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.combined SET batch_number=DIV(seq, public.batch_size()); +END +$ct$ +LANGUAGE plpgsql; + + +----------------------------------------------------------- +-- CONDITIONAL TABLE CREATION +-- We make this conditional at startup/on deploy. +-- The reason is that every time we deploy, this would tear down +-- the entire API, interrupting service. We only do that nightly, if we can. +-- However, on a clean deploy or a first deploy to a 2-DB config, we will +-- need this to run. +----------------------------------------------------------- +DO LANGUAGE plpgsql +$GATE$ + DECLARE + the_schema varchar := 'public_data_v1_0_0'; + the_table varchar := 'metadata'; + api_ver varchar := 'api_v2_0_0'; + BEGIN + IF EXISTS ( + SELECT FROM pg_tables + WHERE schemaname = the_schema + AND tablename = the_table + ) + THEN + RAISE info '% Gate condition met. Skipping table creation.', api_ver; + ELSE + RAISE info '% %.% not found. Creating tables', api_ver, the_schema, the_table; + RAISE info 'Creating general'; + PERFORM public_data_v1_0_0_functions.create_general(); + RAISE info 'Creating additional_eins'; + PERFORM public_data_v1_0_0_functions.create_additional_eins(); + RAISE info 'Creating additional_ueis'; + PERFORM public_data_v1_0_0_functions.create_additional_ueis(); + RAISE info 'Creating corrective_action_plans'; + PERFORM public_data_v1_0_0_functions.create_corrective_action_plans(); + RAISE info 'Creating federal_awards'; + PERFORM public_data_v1_0_0_functions.create_federal_awards(); + RAISE info 'Creating findings'; + PERFORM public_data_v1_0_0_functions.create_findings(); + RAISE info 'Creating findings_text'; + PERFORM public_data_v1_0_0_functions.create_findings_text(); + RAISE info 'Creating notes_to_sefa'; + PERFORM public_data_v1_0_0_functions.create_notes_to_sefa(); + RAISE info 'Creating passthrough'; + PERFORM public_data_v1_0_0_functions.create_passthrough(); + RAISE info 'Creating secondary_auditors'; + PERFORM public_data_v1_0_0_functions.create_secondary_auditors(); + RAISE info 'Creating combined'; + PERFORM public_data_v1_0_0_functions.create_combined(); + END IF; + END +$GATE$; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/021_suppressed_data_v1_0_0_tables.sql b/backend/dissemination/sql/fac-snapshot-db/post/021_suppressed_data_v1_0_0_tables.sql new file mode 100644 index 0000000000..389d13ef75 --- /dev/null +++ b/backend/dissemination/sql/fac-snapshot-db/post/021_suppressed_data_v1_0_0_tables.sql @@ -0,0 +1,130 @@ +----------------------------------------------------------- +-- general +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION suppressed_data_v1_0_0.create_corrective_action_plans() + RETURNS VOID +AS +$ct$ +BEGIN + CREATE TABLE suppressed_data_v1_0_0.corrective_action_plans AS + SELECT + cap.id as id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + cap.contains_chart_or_table, + cap.finding_ref_number, + cap.planned_action + FROM + public_data_v1_0_0.general gen, + public.dissemination_captext cap + WHERE + cap.report_id = gen.report_id + AND + -- Only include the suppressed corrective action plans. + gen.is_public = false + ORDER BY cap.id; +END +$ct$ +LANGUAGE plpgsql; + +----------------------------------------------------------- +-- general +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION suppressed_data_v1_0_0.create_findings_text() + RETURNS VOID +AS +$ct$ +BEGIN + CREATE TABLE suppressed_data_v1_0_0.findings_text AS + SELECT + ft.id as id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + ft.finding_ref_number, + ft.contains_chart_or_table, + ft.finding_text + FROM + public_data_v1_0_0.general gen, + public.dissemination_findingtext ft + WHERE + ft.report_id = gen.report_id + AND + -- Findings text is not always public + gen.is_public = false + ORDER BY ft.id; +END +$ct$ +LANGUAGE plpgsql; + +----------------------------------------------------------- +-- general +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION suppressed_data_v1_0_0.create_notes_to_sefa() + RETURNS VOID +AS +$ct$ +BEGIN + CREATE TABLE suppressed_data_v1_0_0.notes_to_sefa AS + SELECT + note.id as id, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + note.accounting_policies, + note.contains_chart_or_table, + note.content, + note.is_minimis_rate_used, + note.note_title as title, + note.rate_explained + FROM + public_data_v1_0_0.general gen, + public.dissemination_note note + WHERE + note.report_id = gen.report_id + AND + -- Some notes are not public. + gen.is_public = false + ORDER BY note.id; +END +$ct$ +LANGUAGE plpgsql; + +----------------------------------------------------------- +-- CONDITIONAL TABLE CREATION +-- We make this conditional at startup/on deploy. +-- The reason is that every time we deploy, this would tear down +-- the entire API, interrupting service. We only do that nightly, if we can. +-- However, on a clean deploy or a first deploy to a 2-DB config, we will +-- need this to run. +----------------------------------------------------------- +DO LANGUAGE plpgsql +$GATE$ + DECLARE + the_schema varchar := 'public_data_v1_0_0'; + the_table varchar := 'metadata'; + BEGIN + IF EXISTS ( + SELECT FROM pg_tables + WHERE schemaname = the_schema + AND tablename = the_table + ) + THEN + RAISE info 'Found the metadata table; skipping suppressed data table creation.'; + ELSE + RAISE info 'Metadata table not found. Creating suppressed data tables.'; + RAISE info 'Creating suppressed corrective_action_plans'; + PERFORM suppressed_data_v1_0_0.create_corrective_action_plans(); + RAISE info 'Creating suppressed findings_text'; + PERFORM suppressed_data_v1_0_0.create_findings_text(); + RAISE info 'Creating suppressed notes_to_sefa'; + PERFORM suppressed_data_v1_0_0.create_notes_to_sefa(); + END IF; + END +$GATE$; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/030_public_data_v1_0_0_metadata.sql b/backend/dissemination/sql/fac-snapshot-db/post/030_public_data_v1_0_0_metadata.sql new file mode 100644 index 0000000000..c0940c1d4f --- /dev/null +++ b/backend/dissemination/sql/fac-snapshot-db/post/030_public_data_v1_0_0_metadata.sql @@ -0,0 +1,91 @@ +CREATE OR REPLACE FUNCTION public_data_v1_0_0.create_metadata() + RETURNS VOID +AS +$ct$ +BEGIN + CREATE TABLE public_data_v1_0_0.metadata AS + SELECT 'additional_eins' + AS table, COUNT(*) + FROM public_data_v1_0_0.additional_eins + UNION + SELECT 'additional_ueis' + AS table, COUNT(*) + FROM public_data_v1_0_0.additional_ueis + UNION + SELECT 'combined' + AS table, COUNT(*) + FROM public_data_v1_0_0.combined + UNION + SELECT 'federal_awards' + AS table, COUNT(*) + FROM public_data_v1_0_0.federal_awards + UNION + SELECT 'findings_text' + AS table, COUNT(*) + FROM public_data_v1_0_0.findings_text + UNION + SELECT 'findings' + AS table, COUNT(*) + FROM public_data_v1_0_0.findings + UNION + SELECT 'general' + AS table, COUNT(*) + FROM public_data_v1_0_0.general + UNION + SELECT 'notes_to_sefa' + AS table, COUNT(*) + FROM public_data_v1_0_0.notes_to_sefa + UNION + SELECT 'passthrough' + AS table, COUNT(*) + FROM public_data_v1_0_0.passthrough + UNION + SELECT 'secondary_auditors' + AS table, COUNT(*) + FROM public_data_v1_0_0.secondary_auditors + UNION + SELECT 'suppressed_corrective_action_plans' + AS table, COUNT(*) + FROM suppressed_data_v1_0_0.corrective_action_plans + UNION + SELECT 'suppressed_findings_text' + AS table, COUNT(*) + FROM suppressed_data_v1_0_0.findings_text + UNION + SELECT 'suppressed_notes_to_sefa' + AS table, COUNT(*) + FROM suppressed_data_v1_0_0.notes_to_sefa + UNION + SELECT 'public_submission_count' + AS table, COUNT(*) + FROM public_data_v1_0_0.general gen + WHERE gen.is_public = true + UNION + SELECT 'suppressed_submission_count' + AS table, COUNT(*) + FROM public_data_v1_0_0.general gen + WHERE gen.is_public = false + ; +END +$ct$ +LANGUAGE plpgsql; + +DO LANGUAGE plpgsql +$GATE$ + DECLARE + the_schema varchar := 'public_data_v1_0_0'; + the_table varchar := 'metadata'; + BEGIN + IF EXISTS ( + SELECT FROM pg_tables + WHERE schemaname = the_schema + AND tablename = the_table + ) + THEN + RAISE info 'Gate condition met. Skipping metadata table creation.'; + ELSE + RAISE info 'Creating metadata table'; + PERFORM public_data_v1_0_0.create_metadata(); + END IF; + END +$GATE$; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/060_public_api_v1_0_0.sql b/backend/dissemination/sql/fac-snapshot-db/post/060_public_api_v1_0_0.sql deleted file mode 100644 index 454e1c0bf8..0000000000 --- a/backend/dissemination/sql/fac-snapshot-db/post/060_public_api_v1_0_0.sql +++ /dev/null @@ -1,295 +0,0 @@ ------------------------------------------------------------------- --- GATE ------------------------------------------------------------------- --- We only want the API to run if certain conditions are met. --- We could try and encode that in the `bash` portion of the code. --- Or, we could just gate things at the top of our SQL. --- If the conditions are not met, we should exit noisily. --- A cast to regclass will fail with an exception if the table --- does not exist. -DO LANGUAGE plpgsql -$GATE$ - DECLARE - the_schema varchar := 'public_data_v1_0_0'; - the_table varchar := 'metadata'; - api_ver varchar := 'PUBLIC_V1_0_0'; - BEGIN - IF EXISTS ( - SELECT FROM pg_tables - WHERE schemaname = the_schema - AND tablename = the_table - ) - THEN - RAISE info '% Gate condition met. Continuing.', api_ver; - ELSE - RAISE exception '% %.% not found.', api_ver, the_schema, the_table; - END IF; - END -$GATE$; ------------------------------------------------------ --- PERMISSIONS ------------------------------------------------------ -do -$$ -BEGIN - DROP SCHEMA IF EXISTS public_api_v1_0_0 CASCADE; - DROP SCHEMA IF EXISTS public_api_v1_0_0_functions CASCADE; - CREATE SCHEMA IF NOT EXISTS public_api_v1_0_0; - CREATE SCHEMA IF NOT EXISTS public_api_v1_0_0_functions; - -- Functions are loaded before sling comes up. - - GRANT USAGE ON SCHEMA public_api_v1_0_0_functions TO api_fac_gov; - GRANT USAGE ON SCHEMA public_api_v1_0_0 TO api_fac_gov; - GRANT USAGE ON SCHEMA public_data_v1_0_0 TO api_fac_gov; - -END -$$ -; - ---- --- FUNCTIONS --- - -CREATE OR REPLACE FUNCTION public_api_v1_0_0_functions.rows_per_batch() - RETURNS integer - LANGUAGE sql IMMUTABLE PARALLEL SAFE AS - 'SELECT 20000'; - -CREATE OR REPLACE FUNCTION public_api_v1_0_0_functions.batch (id bigint) - RETURNS bigint - AS $batch$ - DECLARE result bigint; - DECLARE RPB integer; - BEGIN - SELECT public_api_v1_0_0_functions.rows_per_batch() INTO RPB; - SELECT div(id, RPB) INTO result; - RETURN result; - END - $batch$ - LANGUAGE plpgsql IMMUTABLE; - -CREATE OR REPLACE FUNCTION public_api_v1_0_0_functions.batches (_table text) -returns integer -as $batches$ -declare - count integer; - RPB integer; -begin - select public_api_v1_0_0_functions.rows_per_batch() into RPB; - case - when _table = 'additional_eins' then - select div(count(*), RPB) into count - from public_data_v1_0_0.additional_eins; - when _table = 'additional_ueis' then - select div(count(*), RPB) into count - from public_data_v1_0_0.additional_ueis; - when _table = 'combined' then - select div(count(*), RPB) into count - from public_data_v1_0_0.combined; - when _table = 'corrective_action_plans' then - select div(count(*), RPB) into count - from public_data_v1_0_0.corrective_action_plans; - when _table = 'federal_awards' then - select div(count(*), RPB) into count - from public_data_v1_0_0.federal_awards; - when _table = 'findings_text' then - select div(count(*), RPB) into count - from public_data_v1_0_0.findings_text; - when _table = 'findings' then - select div(count(*), RPB) into count - from public_data_v1_0_0.findings; - when _table = 'general' then - select div(count(*), RPB) into count - from public_data_v1_0_0.general; - when _table = 'notes_to_sefa' then - select div(count(*), RPB) into count - from public_data_v1_0_0.notes_to_sefa; - when _table = 'passthrough' then - select div(count(*), RPB) into count - from public_data_v1_0_0.passthrough; - when _table = 'secondary_auditors' then - select div(count(*), RPB) into count - from public_data_v1_0_0.secondary_auditors; - else - count := 0; - end case; - RETURN count; -end -$batches$ -language plpgsql; - -CREATE OR REPLACE FUNCTION public_api_v1_0_0.compute_batch(row_id bigint) - RETURNS BIGINT - AS $$ - DECLARE result bigint; - BEGIN - SELECT public_api_v1_0_0_functions.batch(row_id) INTO result; - return result; - END - $$ LANGUAGE plpgsql; - - -CREATE OR REPLACE FUNCTION public_api_v1_0_0.get_batch_additional_eins (_batch bigint) -RETURNS SETOF record -AS $get_batch$ -BEGIN - RETURN QUERY SELECT * - FROM public_data_v1_0_0.additional_eins - WHERE batch_number = _batch; -END; -$get_batch$ -LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION public_api_v1_0_0.get_batch_federal_awards (_batch bigint) -RETURNS SETOF public_data_v1_0_0.federal_awards -AS $get_batch$ -BEGIN - RETURN QUERY SELECT * - FROM public_data_v1_0_0.federal_awards - WHERE batch_number = _batch; -END; -$get_batch$ -LANGUAGE plpgsql; - -NOTIFY pgrst, 'reload schema'; - - --- when _table = 'additional_ueis' then --- RETURN QUERY SELECT * --- FROM public_data_v1_0_0.additional_ueis --- WHERE batch_number = _batch; --- when _table = 'combined' then --- RETURN QUERY SELECT * --- FROM public_data_v1_0_0.combined --- WHERE batch_number = _batch; --- when _table = 'corrective_action_plans' then --- RETURN QUERY SELECT * --- FROM public_data_v1_0_0.corrective_action_plans --- WHERE batch_number = _batch; --- when _table = 'federal_awards' then --- RETURN QUERY SELECT * --- FROM public_data_v1_0_0.federal_awards --- WHERE batch_number = _batch; --- when _table = 'findings_text' then --- RETURN QUERY SELECT * --- FROM public_data_v1_0_0.findings_text --- WHERE batch_number = _batch; --- when _table = 'findings' then --- RETURN QUERY SELECT * --- FROM public_data_v1_0_0.findings --- WHERE batch_number = _batch; --- when _table = 'general' then --- RETURN QUERY SELECT * --- FROM public_data_v1_0_0.general --- WHERE batch_number = _batch; --- when _table = 'notes_to_sefa' then --- RETURN QUERY SELECT * --- FROM public_data_v1_0_0.notes_to_sefa --- WHERE batch_number = _batch; --- when _table = 'passthrough' then --- RETURN QUERY SELECT * --- FROM public_data_v1_0_0.passthrough --- WHERE batch_number = _batch; --- when _table = 'secondary_auditors' then --- RETURN QUERY SELECT * --- FROM public_data_v1_0_0.secondary_auditors --- WHERE batch_number = _batch; --- end case; --- END; --- $get_batch$ --- language plpgsql; - - -NOTIFY pgrst, 'reload schema'; - -BEGIN; - -CREATE VIEW public_api_v1_0_0.additional_eins AS - SELECT * FROM public_data_v1_0_0.additional_eins ae - ORDER BY ae.id -; - ---------------------------------------- --- additional_ueis ---------------------------------------- -create view public_api_v1_0_0.additional_ueis AS - SELECT * FROM public_data_v1_0_0.additional_ueis au - ORDER BY au.id -; - ---------------------------------------- --- corrective_action_plan ---------------------------------------- -CREATE VIEW public_api_v1_0_0.corrective_action_plans AS - SELECT * FROM public_data_v1_0_0.corrective_action_plans cap - ORDER BY cap.id -; - ---------------------------------------- --- finding ---------------------------------------- -CREATE VIEW public_api_v1_0_0.findings as - SELECT * FROM public_data_v1_0_0.findings f - ORDER BY f.id -; - ---------------------------------------- --- finding_text ---------------------------------------- -CREATE VIEW public_api_v1_0_0.findings_text AS - SELECT * FROM public_data_v1_0_0.findings_text ft - ORDER BY ft.id -; - ---------------------------------------- --- federal award ---------------------------------------- -CREATE VIEW public_api_v1_0_0.federal_awards AS - SELECT * FROM public_data_v1_0_0.federal_awards fa - ORDER BY fa.id -; - ---------------------------------------- --- general ---------------------------------------- -CREATE VIEW public_api_v1_0_0.general AS - SELECT * FROM public_data_v1_0_0.general -; - ---------------------------------------- --- notes_to_sefa ---------------------------------------- -create view public_api_v1_0_0.notes_to_sefa AS - SELECT * FROM public_data_v1_0_0.notes_to_sefa nts - ORDER BY nts.id -; - ---------------------------------------- --- passthrough ---------------------------------------- -CREATE VIEW public_api_v1_0_0.passthrough AS - SELECT * FROM public_data_v1_0_0.passthrough p - ORDER BY p.id -; - ---------------------------------------- --- auditor (secondary auditor) ---------------------------------------- -CREATE VIEW public_api_v1_0_0.secondary_auditors AS - SELECT * FROM public_data_v1_0_0.secondary_auditors sa - ORDER BY sa.id - ; - ---------------------------------------- --- combined ---------------------------------------- -CREATE VIEW public_api_v1_0_0.combined AS - SELECT * FROM public_data_v1_0_0.combined comb - ; - ---------------------------------------- --- metadata ---------------------------------------- -CREATE VIEW public_api_v1_0_0.metadata AS - SELECT * FROM public_data_v1_0_0.metadata - ; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/070_public_data_v1_0_0.sql b/backend/dissemination/sql/fac-snapshot-db/post/070_public_data_v1_0_0.sql deleted file mode 100644 index e02d3d1929..0000000000 --- a/backend/dissemination/sql/fac-snapshot-db/post/070_public_data_v1_0_0.sql +++ /dev/null @@ -1,6 +0,0 @@ -DO LANGUAGE plpgsql -$NOTE$ - BEGIN - RAISE info 'PUBLIC_DATA_V1_0_0 standup no-op.'; - END; -$NOTE$; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/080_permissions.sql b/backend/dissemination/sql/fac-snapshot-db/post/080_permissions.sql index 4b2efa9e56..1da5de0b8e 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/080_permissions.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/080_permissions.sql @@ -1,5 +1,7 @@ -- This grants access to the tables and views that were created -- to the API server. -GRANT SELECT ON ALL TABLES IN SCHEMA api_v1_0_3 TO api_fac_gov; + +-- Decommissioned 20241024 +-- GRANT SELECT ON ALL TABLES IN SCHEMA api_v1_0_3 TO api_fac_gov; GRANT SELECT ON ALL TABLES IN SCHEMA api_v1_1_0 TO api_fac_gov; -GRANT SELECT ON ALL TABLES IN SCHEMA public_api_v1_0_0 TO api_fac_gov; +GRANT SELECT ON ALL TABLES IN SCHEMA api_v2_0_0 TO api_fac_gov; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/120_api_v1_1_0_gate.sql b/backend/dissemination/sql/fac-snapshot-db/post/120_api_v1_1_0_gate.sql new file mode 100644 index 0000000000..8cc1878ce8 --- /dev/null +++ b/backend/dissemination/sql/fac-snapshot-db/post/120_api_v1_1_0_gate.sql @@ -0,0 +1,28 @@ +------------------------------------------------------------------ +-- GATE +------------------------------------------------------------------ +-- We only want the API to run if certain conditions are met. +-- We could try and encode that in the `bash` portion of the code. +-- Or, we could just gate things at the top of our SQL. +-- If the conditions are not met, we should exit noisily. +-- A cast to regclass will fail with an exception if the table +-- does not exist. +DO LANGUAGE plpgsql +$GATE$ + DECLARE + the_schema varchar := 'public'; + the_table varchar := 'dissemination_general'; + api_ver varchar := 'api_v1_1_0'; + BEGIN + IF EXISTS ( + SELECT FROM pg_tables + WHERE schemaname = the_schema + AND tablename = the_table + ) + THEN + RAISE info '% Gate condition met. Continuing.', api_ver; + ELSE + RAISE exception '% %.% not found.', api_ver, the_schema, the_table; + END IF; + END +$GATE$; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/121_api_v1_1_0_create_schema.sql b/backend/dissemination/sql/fac-snapshot-db/post/121_api_v1_1_0_create_schema.sql new file mode 100644 index 0000000000..c2d48c5db8 --- /dev/null +++ b/backend/dissemination/sql/fac-snapshot-db/post/121_api_v1_1_0_create_schema.sql @@ -0,0 +1,3 @@ +CREATE SCHEMA IF NOT EXISTS api_v1_1_0; +CREATE SCHEMA IF NOT EXISTS api_v1_1_0_functions; + \ No newline at end of file diff --git a/backend/dissemination/sql/fac-snapshot-db/post/122_api_v1_1_0_functions.sql b/backend/dissemination/sql/fac-snapshot-db/post/122_api_v1_1_0_functions.sql new file mode 100644 index 0000000000..3abebc0b9b --- /dev/null +++ b/backend/dissemination/sql/fac-snapshot-db/post/122_api_v1_1_0_functions.sql @@ -0,0 +1,53 @@ +----------------------------------------------------- +-- get_header +-- Reaches into the headers provided through PostgREST in order to +-- grab a particular value, keyed by the header key (e.g. "x-api-key") +----------------------------------------------------- +CREATE OR REPLACE FUNCTION api_v1_1_0_functions.get_header(item text) RETURNS TEXT + AS $get_header$ + DECLARE res TEXT; + BEGIN + SELECT (current_setting('request.headers', true)::json)->>item INTO res; + RETURN res; + END; +$get_header$ LANGUAGE plpgsql; + +----------------------------------------------------- +-- get_api_key_uuid +-- Uses the get_header function to grab the user id provided by api.data.gov +----------------------------------------------------- +CREATE OR REPLACE FUNCTION api_v1_1_0_functions.get_api_key_uuid() RETURNS TEXT +AS $gaku$ +DECLARE uuid TEXT; +BEGIN + SELECT api_v1_1_0_functions.get_header('x-api-user-id') INTO uuid; + RETURN uuid; +end; +$gaku$ LANGUAGE plpgsql; + +----------------------------------------------------- +-- has_tribal_data_access +-- Determines whether the key id in question has been granted +-- tribal data access. Required for accessing all of the suppressed tables. +----------------------------------------------------- +CREATE OR REPLACE FUNCTION api_v1_1_0_functions.has_tribal_data_access() +RETURNS BOOLEAN +AS $has_tribal_data_access$ +DECLARE + uuid_header UUID; + key_exists BOOLEAN; +BEGIN + SELECT api_v1_1_0_functions.get_api_key_uuid() INTO uuid_header; + SELECT + CASE WHEN EXISTS ( + SELECT key_id + FROM public.dissemination_tribalapiaccesskeyids taaki + WHERE taaki.key_id = uuid_header::TEXT) + THEN 1::BOOLEAN + ELSE 0::BOOLEAN + END + INTO key_exists; + -- RAISE INFO 'api_v1_1_0 has_tribal % %', uuid_header, key_exists; + RETURN key_exists; +END; +$has_tribal_data_access$ LANGUAGE plpgsql; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/020_api_v1_1_0.sql b/backend/dissemination/sql/fac-snapshot-db/post/123_api_v1_1_0_views.sql similarity index 57% rename from backend/dissemination/sql/fac-snapshot-db/post/020_api_v1_1_0.sql rename to backend/dissemination/sql/fac-snapshot-db/post/123_api_v1_1_0_views.sql index 2df37faf7e..f3d9f28dff 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/020_api_v1_1_0.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/123_api_v1_1_0_views.sql @@ -1,209 +1,19 @@ ------------------------------------------------------------------- --- GATE ------------------------------------------------------------------- --- We only want the API to run if certain conditions are met. --- We could try and encode that in the `bash` portion of the code. --- Or, we could just gate things at the top of our SQL. --- If the conditions are not met, we should exit noisily. --- A cast to regclass will fail with an exception if the table --- does not exist. -DO LANGUAGE plpgsql -$GATE$ - DECLARE - the_schema varchar := 'public'; - the_table varchar := 'dissemination_general'; - api_ver varchar := 'API_v1_1_0'; - BEGIN - IF EXISTS ( - SELECT FROM pg_tables - WHERE schemaname = the_schema - AND tablename = the_table - ) - THEN - RAISE info '% Gate condition met. Continuing.', api_ver; - ELSE - RAISE exception '% %.% not found.', api_ver, the_schema, the_table; - END IF; - END -$GATE$; - -do -$$ -begin - DROP SCHEMA IF EXISTS api_v1_1_0 CASCADE; - DROP SCHEMA IF EXISTS api_v1_1_0_functions CASCADE; - - if not exists (select schema_name from information_schema.schemata where schema_name = 'api_v1_1_0') then - create schema api_v1_1_0; - create schema api_v1_1_0_functions; - - grant usage on schema api_v1_1_0_functions to api_fac_gov; - grant usage on schema api_v1_1_0 to api_fac_gov; - grant select, usage on all sequences in schema api_v1_1_0 to api_fac_gov; - - -- Grant access to tables and views - -- NOTE 20240920: This grants SELECT to all tables - -- created in this schema *in the future*. Nothing - -- that already exists is affected. - -- https://www.postgresql.org/docs/current/sql-alterdefaultprivileges.html - alter default privileges - in schema api_v1_1_0 - grant select - -- this includes views - on tables - to api_fac_gov; - - alter default privileges - in schema api_v1_1_0_functions - grant select - -- this includes views - on tables - to api_fac_gov; - - -- Grant access to future sequences, if we have them - alter default privileges - in schema api_v1_1_0 - grant select, usage - on sequences - to api_fac_gov; - - end if; -end -$$ -; - ------------------------------------------------------ --- FUNCTIONS ------------------------------------------------------ - -CREATE OR REPLACE FUNCTION api_v1_1_0_functions.get_header(item text) RETURNS text - AS $get_header$ - declare res text; - begin - SELECT (current_setting('request.headers', true)::json)->>item into res; - return res; - end; -$get_header$ LANGUAGE plpgsql; - -create or replace function api_v1_1_0_functions.get_api_key_uuid() returns TEXT -as $gaku$ -declare uuid text; -begin - select api_v1_1_0_functions.get_header('x-api-user-id') into uuid; - return uuid; -end; -$gaku$ LANGUAGE plpgsql; - -create or replace function api_v1_1_0_functions.has_tribal_data_access() -returns boolean -as $has_tribal_data_access$ -DECLARE - uuid_header UUID; - key_exists boolean; -BEGIN - - SELECT api_v1_1_0_functions.get_api_key_uuid() INTO uuid_header; - SELECT - CASE WHEN EXISTS ( - SELECT key_id - FROM public.dissemination_TribalApiAccessKeyIds taaki - WHERE taaki.key_id = uuid_header::TEXT) - THEN 1::BOOLEAN - ELSE 0::BOOLEAN - END - INTO key_exists; - RAISE INFO 'api_v1_1_0 has_tribal % %', uuid_header, key_exists; - RETURN key_exists; -END; -$has_tribal_data_access$ LANGUAGE plpgsql; - --- If you change the constant defined by this function, --- you must regenerate the index. -CREATE OR REPLACE FUNCTION api_v1_1_0_functions.batch_size() - RETURNS int - LANGUAGE sql IMMUTABLE PARALLEL SAFE AS -'SELECT 20000'; -GRANT EXECUTE ON FUNCTION api_v1_1_0_functions.batch_size() TO api_fac_gov; - -CREATE OR REPLACE FUNCTION api_v1_1_0_functions.batch (id bigint) -returns bigint -as $batch$ -declare num bigint; -begin - select div(id, api_v1_1_0_functions.batch_size()) into num; - return num; -end; -$batch$ -language plpgsql immutable; -GRANT EXECUTE ON FUNCTION api_v1_1_0_functions.batch(bigint) TO api_fac_gov; - -CREATE OR REPLACE FUNCTION api_v1_1_0.batches (in_table text) -returns integer -as $batches$ -declare count integer; -declare batch_size bigint; -begin - select api_v1_1_0_functions.batch_size() into batch_size; - case - when in_table = 'general' then - select div(count(*), batch_size) into count - from public.dissemination_general; - when in_table = 'federal_awards' then - select div(count(*), batch_size) into count - from public.dissemination_federalaward; - else - count := 0; - end case; - RETURN count; -end; -$batches$ -language plpgsql; -GRANT EXECUTE ON FUNCTION api_v1_1_0.batches(text) TO api_fac_gov; - -CREATE OR REPLACE FUNCTION api_v1_1_0.get_general_batch (batch_no bigint) -returns setof dissemination_general -as $batches$ - select * from public.dissemination_general where api_v1_1_0_functions.batch(id) = batch_no; -$batches$ -language sql; -GRANT EXECUTE ON FUNCTION api_v1_1_0.get_general_batch(bigint) TO api_fac_gov; - -CREATE OR REPLACE FUNCTION api_v1_1_0.get_federal_award_batch (batch_no bigint) -returns setof dissemination_federalaward -as $batches$ - select * from public.dissemination_federalaward where api_v1_1_0_functions.batch(public.dissemination_federalaward.id) = batch_no; -$batches$ -language sql; -GRANT EXECUTE ON FUNCTION api_v1_1_0.get_federal_award_batch(bigint) TO api_fac_gov; - - --- We should consider dropping and regenerating this index --- every night after MV generation. --- drop index batch_by_id_dfa; -create index IF NOT EXISTS batch_by_id_dfa - on public.dissemination_federalaward - using btree(api_v1_1_0_functions.batch(public.dissemination_federalaward.id)); - --------------------------------------- --- finding_text +-- additional_eins --------------------------------------- -create view api_v1_1_0.findings_text as +create view api_v1_1_0.additional_eins as select gen.report_id, gen.auditee_uei, gen.audit_year, - ft.finding_ref_number, - ft.contains_chart_or_table, - ft.finding_text + --- + ein.additional_ein from - dissemination_findingtext ft, - dissemination_general gen + dissemination_general gen, + dissemination_additionalein ein where - ft.report_id = gen.report_id - and - (gen.is_public = true - or (gen.is_public = false and api_v1_1_0_functions.has_tribal_data_access())) - order by ft.id + gen.report_id = ein.report_id + order by ein.id ; --------------------------------------- @@ -224,31 +34,28 @@ create view api_v1_1_0.additional_ueis as order by uei.id ; + --------------------------------------- --- finding +-- corrective_action_plan --------------------------------------- -create view api_v1_1_0.findings as +create view api_v1_1_0.corrective_action_plans as select gen.report_id, gen.auditee_uei, gen.audit_year, - finding.award_reference, - finding.reference_number, - finding.is_material_weakness, - finding.is_modified_opinion, - finding.is_other_findings, - finding.is_other_matters, - finding.prior_finding_ref_numbers, - finding.is_questioned_costs, - finding.is_repeat_finding, - finding.is_significant_deficiency, - finding.type_requirement + --- + ct.finding_ref_number, + ct.contains_chart_or_table, + ct.planned_action from - dissemination_finding finding, - dissemination_general gen + dissemination_CAPText ct, + dissemination_General gen where - finding.report_id = gen.report_id - order by finding.id + ct.report_id = gen.report_id + and + (gen.is_public = true + or (gen.is_public = false and api_v1_1_0_functions.has_tribal_data_access())) + order by ct.id ; --------------------------------------- @@ -287,77 +94,55 @@ create view api_v1_1_0.federal_awards as order by award.id ; - --------------------------------------- --- corrective_action_plan +-- finding --------------------------------------- -create view api_v1_1_0.corrective_action_plans as +create view api_v1_1_0.findings as select gen.report_id, gen.auditee_uei, gen.audit_year, - --- - ct.finding_ref_number, - ct.contains_chart_or_table, - ct.planned_action + finding.award_reference, + finding.reference_number, + finding.is_material_weakness, + finding.is_modified_opinion, + finding.is_other_findings, + finding.is_other_matters, + finding.prior_finding_ref_numbers, + finding.is_questioned_costs, + finding.is_repeat_finding, + finding.is_significant_deficiency, + finding.type_requirement from - dissemination_CAPText ct, - dissemination_General gen + dissemination_finding finding, + dissemination_general gen where - ct.report_id = gen.report_id - and - (gen.is_public = true - or (gen.is_public = false and api_v1_1_0_functions.has_tribal_data_access())) - order by ct.id + finding.report_id = gen.report_id + order by finding.id ; --------------------------------------- --- notes_to_sefa +-- finding_text --------------------------------------- -create view api_v1_1_0.notes_to_sefa as +create view api_v1_1_0.findings_text as select gen.report_id, gen.auditee_uei, gen.audit_year, - --- - note.note_title as title, - note.accounting_policies, - note.is_minimis_rate_used, - note.rate_explained, - note.content, - note.contains_chart_or_table + ft.finding_ref_number, + ft.contains_chart_or_table, + ft.finding_text from - dissemination_general gen, - dissemination_note note + dissemination_findingtext ft, + dissemination_general gen where - note.report_id = gen.report_id - and + ft.report_id = gen.report_id + and (gen.is_public = true or (gen.is_public = false and api_v1_1_0_functions.has_tribal_data_access())) - order by note.id -; - ---------------------------------------- --- passthrough ---------------------------------------- -create view api_v1_1_0.passthrough as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - pass.award_reference, - pass.passthrough_id, - pass.passthrough_name - from - dissemination_general as gen, - dissemination_passthrough as pass - where - gen.report_id = pass.report_id - order by pass.id + order by ft.id ; - --------------------------------------- -- general --------------------------------------- @@ -441,6 +226,52 @@ create view api_v1_1_0.general as order by gen.id ; +--------------------------------------- +-- notes_to_sefa +--------------------------------------- +create view api_v1_1_0.notes_to_sefa as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + note.note_title as title, + note.accounting_policies, + note.is_minimis_rate_used, + note.rate_explained, + note.content, + note.contains_chart_or_table + from + dissemination_general gen, + dissemination_note note + where + note.report_id = gen.report_id + and + (gen.is_public = true + or (gen.is_public = false and api_v1_1_0_functions.has_tribal_data_access())) + order by note.id +; + +--------------------------------------- +-- passthrough +--------------------------------------- +create view api_v1_1_0.passthrough as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + pass.award_reference, + pass.passthrough_id, + pass.passthrough_name + from + dissemination_general as gen, + dissemination_passthrough as pass + where + gen.report_id = pass.report_id + order by pass.id +; + --------------------------------------- -- auditor (secondary auditor) --------------------------------------- @@ -467,18 +298,3 @@ create view api_v1_1_0.secondary_auditors as sa.report_id = gen.report_id order by sa.id ; - -create view api_v1_1_0.additional_eins as - select - gen.report_id, - gen.auditee_uei, - gen.audit_year, - --- - ein.additional_ein - from - dissemination_general gen, - dissemination_additionalein ein - where - gen.report_id = ein.report_id - order by ein.id -; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/124_api_v1_1_0_rpc.sql b/backend/dissemination/sql/fac-snapshot-db/post/124_api_v1_1_0_rpc.sql new file mode 100644 index 0000000000..2e48d88efe --- /dev/null +++ b/backend/dissemination/sql/fac-snapshot-db/post/124_api_v1_1_0_rpc.sql @@ -0,0 +1,46 @@ +CREATE OR REPLACE FUNCTION api_v1_1_0.request_file_access( + report_id TEXT +) RETURNS JSON LANGUAGE plpgsql AS +$$ +DECLARE + v_uuid_header TEXT; + v_access_uuid VARCHAR(200); + v_key_exists BOOLEAN; + v_key_added_date DATE; +BEGIN + + SELECT api_v1_1_0_functions.get_api_key_uuid() INTO v_uuid_header; + + -- Check if the provided API key exists in public.dissemination_TribalApiAccessKeyIds + SELECT + EXISTS( + SELECT 1 + FROM public.dissemination_tribalapiaccesskeyids + WHERE key_id = v_uuid_header + ) INTO v_key_exists; + + + -- Get the added date of the key from public.dissemination_TribalApiAccessKeyIds + SELECT date_added + INTO v_key_added_date + FROM public.dissemination_tribalapiaccesskeyids + WHERE key_id = v_uuid_header; + + + -- Check if the key is less than 6 months old + IF v_uuid_header IS NOT NULL AND v_key_exists AND v_key_added_date >= CURRENT_DATE - INTERVAL '6 months' THEN + -- Generate UUID (using PostgreSQL's gen_random_uuid function) + SELECT gen_random_uuid() INTO v_access_uuid; + + -- Inserting data into the one_time_access table + INSERT INTO public.dissemination_onetimeaccess (uuid, api_key_id, timestamp, report_id) + VALUES (v_access_uuid::UUID, v_uuid_header, CURRENT_TIMESTAMP, report_id); + + -- Return the UUID to the user + RETURN json_build_object('access_uuid', v_access_uuid); + ELSE + -- Return an error for unauthorized access + RETURN json_build_object('error', 'Unauthorized access or key older than 6 months')::JSON; + END IF; +END; +$$; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/129_api_v1_1_0_perms.sql b/backend/dissemination/sql/fac-snapshot-db/post/129_api_v1_1_0_perms.sql new file mode 100644 index 0000000000..8dab5a0c9a --- /dev/null +++ b/backend/dissemination/sql/fac-snapshot-db/post/129_api_v1_1_0_perms.sql @@ -0,0 +1,5 @@ +GRANT USAGE ON SCHEMA api_v1_1_0_functions TO api_fac_gov; +GRANT USAGE ON SCHEMA api_v1_1_0 TO api_fac_gov; + +GRANT SELECT ON ALL TABLES IN SCHEMA api_v1_1_0 to api_fac_gov; +GRANT SELECT, USAGE ON ALL SEQUENCES IN SCHEMA api_v1_1_0 TO api_fac_gov; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/140_api_v2_0_0_gate.sql b/backend/dissemination/sql/fac-snapshot-db/post/140_api_v2_0_0_gate.sql new file mode 100644 index 0000000000..f66ceb2c55 --- /dev/null +++ b/backend/dissemination/sql/fac-snapshot-db/post/140_api_v2_0_0_gate.sql @@ -0,0 +1,28 @@ +------------------------------------------------------------------ +-- GATE +------------------------------------------------------------------ +-- We only want the API to run if certain conditions are met. +-- We could try and encode that in the `bash` portion of the code. +-- Or, we could just gate things at the top of our SQL. +-- If the conditions are not met, we should exit noisily. +-- A cast to regclass will fail with an exception if the table +-- does not exist. +DO LANGUAGE plpgsql +$GATE$ + DECLARE + the_schema varchar := 'public_data_v1_0_0'; + the_table varchar := 'metadata'; + api_ver varchar := 'api_v2_0_0'; + BEGIN + IF EXISTS ( + SELECT FROM pg_tables + WHERE schemaname = the_schema + AND tablename = the_table + ) + THEN + RAISE info '% Gate condition met. Continuing.', api_ver; + ELSE + RAISE exception '% %.% not found.', api_ver, the_schema, the_table; + END IF; + END +$GATE$; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/141_api_v2_0_0_create_schema.sql b/backend/dissemination/sql/fac-snapshot-db/post/141_api_v2_0_0_create_schema.sql new file mode 100644 index 0000000000..9cfe24e77f --- /dev/null +++ b/backend/dissemination/sql/fac-snapshot-db/post/141_api_v2_0_0_create_schema.sql @@ -0,0 +1,2 @@ +CREATE SCHEMA IF NOT EXISTS api_v2_0_0; +CREATE SCHEMA IF NOT EXISTS api_v2_0_0_functions; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/142_api_v2_0_0_functions.sql b/backend/dissemination/sql/fac-snapshot-db/post/142_api_v2_0_0_functions.sql new file mode 100644 index 0000000000..721472c050 --- /dev/null +++ b/backend/dissemination/sql/fac-snapshot-db/post/142_api_v2_0_0_functions.sql @@ -0,0 +1,53 @@ +----------------------------------------------------- +-- get_header +-- Reaches into the headers provided through PostgREST in order to +-- grab a particular value, keyed by the header key (e.g. "x-api-key") +----------------------------------------------------- +CREATE OR REPLACE FUNCTION api_v2_0_0_functions.get_header(item text) RETURNS TEXT + AS $get_header$ + DECLARE res TEXT; + BEGIN + SELECT (current_setting('request.headers', true)::json)->>item INTO res; + RETURN res; + END; +$get_header$ LANGUAGE plpgsql; + +----------------------------------------------------- +-- get_api_key_uuid +-- Uses the get_header function to grab the user id provided by api.data.gov +----------------------------------------------------- +CREATE OR REPLACE FUNCTION api_v2_0_0_functions.get_api_key_uuid() RETURNS TEXT +AS $gaku$ +DECLARE uuid TEXT; +BEGIN + SELECT api_v2_0_0_functions.get_header('x-api-user-id') INTO uuid; + RETURN uuid; +end; +$gaku$ LANGUAGE plpgsql; + +----------------------------------------------------- +-- has_tribal_data_access +-- Determines whether the key id in question has been granted +-- tribal data access. Required for accessing all of the suppressed tables. +----------------------------------------------------- +CREATE OR REPLACE FUNCTION api_v2_0_0_functions.has_tribal_data_access() +RETURNS BOOLEAN +AS $has_tribal_data_access$ +DECLARE + uuid_header UUID; + key_exists BOOLEAN; +BEGIN + SELECT api_v2_0_0_functions.get_api_key_uuid() INTO uuid_header; + SELECT + CASE WHEN EXISTS ( + SELECT key_id + FROM public.dissemination_tribalapiaccesskeyids taaki + WHERE taaki.key_id = uuid_header::TEXT) + THEN 1::BOOLEAN + ELSE 0::BOOLEAN + END + INTO key_exists; + -- RAISE INFO 'api_v2_0_0 has_tribal % %', uuid_header, key_exists; + RETURN key_exists; +END; +$has_tribal_data_access$ LANGUAGE plpgsql; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/143_api_v2_0_0_views.sql b/backend/dissemination/sql/fac-snapshot-db/post/143_api_v2_0_0_views.sql new file mode 100644 index 0000000000..88aa11a736 --- /dev/null +++ b/backend/dissemination/sql/fac-snapshot-db/post/143_api_v2_0_0_views.sql @@ -0,0 +1,112 @@ +--------------------------------------- +-- additional_eins +--------------------------------------- +CREATE VIEW api_v2_0_0.additional_eins AS + SELECT * FROM public_data_v1_0_0.additional_eins ae + ORDER BY ae.id; + +--------------------------------------- +-- additional_ueis +--------------------------------------- +create view api_v2_0_0.additional_ueis AS + SELECT * FROM public_data_v1_0_0.additional_ueis au + ORDER BY au.id; + +--------------------------------------- +-- corrective_action_plan +--------------------------------------- +CREATE VIEW api_v2_0_0.corrective_action_plans AS + SELECT * FROM public_data_v1_0_0.corrective_action_plans cap + ORDER BY cap.id; + +--------------------------------------- +-- finding +--------------------------------------- +CREATE VIEW api_v2_0_0.findings as + SELECT * FROM public_data_v1_0_0.findings f + ORDER BY f.id; + +--------------------------------------- +-- finding_text +--------------------------------------- +CREATE VIEW api_v2_0_0.findings_text AS + SELECT * FROM public_data_v1_0_0.findings_text ft + ORDER BY ft.id; + +--------------------------------------- +-- federal award +--------------------------------------- +CREATE VIEW api_v2_0_0.federal_awards AS + SELECT * FROM public_data_v1_0_0.federal_awards fa + ORDER BY fa.id; + +--------------------------------------- +-- general +--------------------------------------- +CREATE VIEW api_v2_0_0.general AS + SELECT * FROM public_data_v1_0_0.general gen + ORDER BY gen.id; + +--------------------------------------- +-- notes_to_sefa +--------------------------------------- +CREATE VIEW api_v2_0_0.notes_to_sefa AS + SELECT * FROM public_data_v1_0_0.notes_to_sefa nts + ORDER BY nts.id; + +--------------------------------------- +-- passthrough +--------------------------------------- +CREATE VIEW api_v2_0_0.passthrough AS + SELECT * FROM public_data_v1_0_0.passthrough p + ORDER BY p.id; + +--------------------------------------- +-- auditor (secondary auditor) +--------------------------------------- +CREATE VIEW api_v2_0_0.secondary_auditors AS + SELECT * FROM public_data_v1_0_0.secondary_auditors sa + ORDER BY sa.id; + +--------------------------------------- +-- combined +--------------------------------------- +CREATE VIEW api_v2_0_0.combined AS + SELECT * FROM public_data_v1_0_0.combined comb + ORDER BY comb.seq; + +--------------------------------------- +-- metadata +--------------------------------------- +CREATE VIEW api_v2_0_0.metadata AS + SELECT * FROM public_data_v1_0_0.metadata; + +------------------------------------------------------------------------------ +-- SUPPRESSED ENDPOINTS +-- These require an API key that has tribal/suppressed data access. +------------------------------------------------------------------------------ + +--------------------------------------- +-- suppressed_corrective_action_plan +--------------------------------------- +CREATE VIEW api_v2_0_0.suppressed_corrective_action_plans AS + SELECT * FROM suppressed_data_v1_0_0.corrective_action_plans cap + WHERE api_v2_0_0_functions.has_tribal_data_access() + ORDER BY cap.id; + + +--------------------------------------- +-- suppressed_finding_text +--------------------------------------- +CREATE VIEW api_v2_0_0.suppressed_findings_text AS + SELECT * FROM suppressed_data_v1_0_0.findings_text ft + WHERE api_v2_0_0_functions.has_tribal_data_access() + ORDER BY ft.id; + +--------------------------------------- +-- suppressed_notes_to_sefa +--------------------------------------- +CREATE VIEW api_v2_0_0.suppressed_notes_to_sefa AS + SELECT * FROM suppressed_data_v1_0_0.notes_to_sefa nts + WHERE api_v2_0_0_functions.has_tribal_data_access() + ORDER BY nts.id; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/149_api_v2_0_0_perms.sql b/backend/dissemination/sql/fac-snapshot-db/post/149_api_v2_0_0_perms.sql new file mode 100644 index 0000000000..4dcacfcf64 --- /dev/null +++ b/backend/dissemination/sql/fac-snapshot-db/post/149_api_v2_0_0_perms.sql @@ -0,0 +1,7 @@ +----------------------------------------------------- +-- PERMISSIONS +----------------------------------------------------- +GRANT USAGE ON SCHEMA api_v2_0_0_functions TO api_fac_gov; +GRANT USAGE ON SCHEMA api_v2_0_0 TO api_fac_gov; + +GRANT SELECT ON ALL TABLES IN SCHEMA api_v2_0_0 TO api_fac_gov; diff --git a/backend/dissemination/sql/fac-snapshot-db/pre/000_api_v1_0_3.sql b/backend/dissemination/sql/fac-snapshot-db/pre/000_api_v1_0_3.sql deleted file mode 100644 index 5025af5fb4..0000000000 --- a/backend/dissemination/sql/fac-snapshot-db/pre/000_api_v1_0_3.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP SCHEMA IF EXISTS api_v1_0_3 CASCADE; -NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/fac-snapshot-db/pre/010_api_v1_1_0.sql b/backend/dissemination/sql/fac-snapshot-db/pre/010_api_v1_1_0.sql deleted file mode 100644 index d0e2e484d3..0000000000 --- a/backend/dissemination/sql/fac-snapshot-db/pre/010_api_v1_1_0.sql +++ /dev/null @@ -1,27 +0,0 @@ -begin; - -DROP SCHEMA IF EXISTS api_v1_1_0 CASCADE; --- DROP ROLE IF EXISTS authenticator; --- DROP ROLE IF EXISTS api_fac_gov; - -commit; - -notify pgrst, - 'reload schema'; -begin; - drop table if exists api_v1_1_0.metadata; - drop view if exists api_v1_1_0.general; - drop view if exists api_v1_1_0.auditor; - drop view if exists api_v1_1_0.federal_awards; - drop view if exists api_v1_1_0.findings; - drop view if exists api_v1_1_0.findings_text; - drop view if exists api_v1_1_0.corrective_action_plans; - drop view if exists api_v1_1_0.additional_ueis; - drop view if exists api_v1_1_0.notes_to_sefa; - drop view if exists api_v1_1_0.passthrough; - drop view if exists api_v1_1_0.secondary_auditors; - drop view if exists api_v1_1_0.additional_eins; -commit; - -notify pgrst, - 'reload schema'; diff --git a/backend/dissemination/sql/fac-snapshot-db/pre/070_schemas.sql b/backend/dissemination/sql/fac-snapshot-db/pre/010_roles.sql similarity index 87% rename from backend/dissemination/sql/fac-snapshot-db/pre/070_schemas.sql rename to backend/dissemination/sql/fac-snapshot-db/pre/010_roles.sql index 44eda52bae..0847b0ee0a 100644 --- a/backend/dissemination/sql/fac-snapshot-db/pre/070_schemas.sql +++ b/backend/dissemination/sql/fac-snapshot-db/pre/010_roles.sql @@ -9,7 +9,9 @@ BEGIN WHERE rolname = 'authenticator') THEN RAISE NOTICE 'ROLES Role "authenticator" already exists. Skipping.'; ELSE - CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; + CREATE ROLE authenticator + LOGIN NOINHERIT NOCREATEDB + NOCREATEROLE NOSUPERUSER; END IF; END $do$; @@ -31,6 +33,3 @@ END $do$; GRANT api_fac_gov TO authenticator; - -NOTIFY pgrst, 'reload schema'; - diff --git a/backend/dissemination/sql/fac-snapshot-db/pre/030_admin_api_v1_1_1.sql b/backend/dissemination/sql/fac-snapshot-db/pre/030_admin_api_v1_1_1.sql deleted file mode 100644 index 6cbdce2d39..0000000000 --- a/backend/dissemination/sql/fac-snapshot-db/pre/030_admin_api_v1_1_1.sql +++ /dev/null @@ -1,3 +0,0 @@ - -DROP SCHEMA IF EXISTS admin_api_v1_1_1 CASCADE; -notify pgrst, 'reload schema'; diff --git a/backend/dissemination/sql/fac-snapshot-db/pre/040_public_api_v1_0_0.sql b/backend/dissemination/sql/fac-snapshot-db/pre/040_public_api_v1_0_0.sql deleted file mode 100644 index c601de73f2..0000000000 --- a/backend/dissemination/sql/fac-snapshot-db/pre/040_public_api_v1_0_0.sql +++ /dev/null @@ -1,8 +0,0 @@ --- --- This drops all the views, too. ---- -DROP SCHEMA IF EXISTS public_api_v1_0_0 CASCADE; -DROP SCHEMA IF EXISTS public_api_v1_0_0_functions CASCADE; - -notify pgrst, - 'reload schema'; diff --git a/backend/dissemination/sql/fac-snapshot-db/pre/050_public_data_v1_0_0.sql b/backend/dissemination/sql/fac-snapshot-db/pre/050_public_data_v1_0_0.sql deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/backend/dissemination/sql/fac-snapshot-db/pre/060_schemas.sql b/backend/dissemination/sql/fac-snapshot-db/pre/060_schemas.sql index dce01c65ef..ba12cb7ec4 100644 --- a/backend/dissemination/sql/fac-snapshot-db/pre/060_schemas.sql +++ b/backend/dissemination/sql/fac-snapshot-db/pre/060_schemas.sql @@ -1,3 +1,64 @@ --- This has to exist so that `sling` can do its thing. --- So we create it here, as a late step in the teardown. +--------------------------------- +-- DROP +--------------------------------- +DROP SCHEMA IF EXISTS api_v1_0_3 CASCADE; +DROP SCHEMA IF EXISTS api_v1_1_0 CASCADE; +DROP SCHEMA IF EXISTS api_v1_1_1 CASCADE; + +DROP SCHEMA IF EXISTS admin_api_v1_1_0 CASCADE; +DROP SCHEMA IF EXISTS admin_api_v1_1_0_functions CASCADE; + +DROP SCHEMA IF EXISTS admin_api_v1_1_1 CASCADE; +DROP SCHEMA IF EXISTS admin_api_v1_1_1_functions CASCADE; + +DROP SCHEMA IF EXISTS api_v2_0_0 CASCADE; +DROP SCHEMA IF EXISTS api_v2_0_0_functions CASCADE; + +------------------------------------------------------------------ +-- CONDITIONAL +------------------------------------------------------------------ +-- On deploys, we don't want to tear down the public data. +-- So, we check to see if the metadata table is present, and if so, +-- leave the schemas in place. +DO LANGUAGE plpgsql +$GATE$ + DECLARE + the_schema varchar := 'public_data_v1_0_0'; + the_table varchar := 'metadata'; + BEGIN + IF EXISTS ( + SELECT FROM pg_tables + WHERE schemaname = the_schema + AND tablename = the_table + ) + THEN + RAISE info 'The metadata table exists; leaving public data schemas in place.'; + ELSE + RAISE info 'The metadata table does not exist; dropping schemas for a clean rebuild.'; + DROP SCHEMA IF EXISTS public_data_v1_0_0 CASCADE; + DROP SCHEMA IF EXISTS suppressed_data_v1_0_0 CASCADE; + END IF; + END +$GATE$; + +--------------------------------- +-- CREATE +--------------------------------- +-- Retired 20241024 +-- CREATE SCHEMA IF NOT EXISTS api_v1_0_3; +-- CREATE SCHEMA IF NOT EXISTS api_v1_0_3_functions; + +CREATE SCHEMA IF NOT EXISTS api_v1_1_0; +CREATE SCHEMA IF NOT EXISTS api_v1_1_0_functions; + +CREATE SCHEMA IF NOT EXISTS admin_api_v1_1_0; +CREATE SCHEMA IF NOT EXISTS admin_api_v1_1_0_functions; + CREATE SCHEMA IF NOT EXISTS public_data_v1_0_0; +CREATE SCHEMA IF NOT EXISTS public_data_v1_0_0_functions; +CREATE SCHEMA IF NOT EXISTS suppressed_data_v1_0_0; + +CREATE SCHEMA IF NOT EXISTS api_v2_0_0; +CREATE SCHEMA IF NOT EXISTS api_v2_0_0_functions; + + diff --git a/backend/dissemination/sql/public.rest b/backend/dissemination/sql/public.rest index adf0246f88..54a57f049f 100644 --- a/backend/dissemination/sql/public.rest +++ b/backend/dissemination/sql/public.rest @@ -16,14 +16,14 @@ Accept: application/vnd.pgrst.plan GET {{scheme}}://{{apiUrl}}/general?report_id=eq.2021-12-CENSUS-0000250449 authorization: {{authorization}} x-api-user-id: {{xApiUserId}} -accept-profile: public_api_v1_0_0 +accept-profile: api_v2_0_0 Accept: application/vnd.pgrst.plan ### GET {{scheme}}://{{apiUrl}}/federal_awards?batch_number=eq.200 authorization: {{authorization}} x-api-user-id: {{xApiUserId}} -accept-profile: public_api_v1_0_0 +accept-profile: api_v2_0_0 Accept: application/vnd.pgrst.plan @@ -31,23 +31,23 @@ Accept: application/vnd.pgrst.plan GET {{scheme}}://{{apiUrl}}/federal_awards?batch_number=eq.200&limit=1 authorization: {{authorization}} x-api-user-id: {{xApiUserId}} -accept-profile: public_api_v1_0_0 +accept-profile: api_v2_0_0 ### GET {{scheme}}://{{apiUrl}}/combined?report_id=eq.2021-12-CENSUS-0000250449 authorization: {{authorization}} x-api-user-id: {{xApiUserId}} -accept-profile: public_api_v1_0_0 +accept-profile: api_v2_0_0 Accept: application/vnd.pgrst.plan ### GET {{scheme}}://{{apiUrl}}/rpc/get_batch_federal_awards?_batch=3&limit=10 authorization: {{authorization}} x-api-user-id: {{xApiUserId}} -accept-profile: public_api_v1_0_0 +accept-profile: api_v2_0_0 ### GET {{scheme}}://{{apiUrl}}/metadata authorization: {{authorization}} x-api-user-id: {{xApiUserId}} -accept-profile: public_api_v1_0_0 +accept-profile: api_v2_0_0 diff --git a/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml b/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml index 224b75516e..aa74ea29b0 100644 --- a/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml +++ b/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml @@ -23,10 +23,6 @@ streams: SELECT gen.id as id, NEXTVAL('public_data_v1_0_0.seq_v1_0_0_general') AS seq, - -- Add in post - -- DIV(seq, 20000) AS batch_number, - -- What if we use this for partitioning, instead? - -- mod((gen.id - min(gen.id)), 20000) AS batch_number_2, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -115,77 +111,11 @@ streams: public.dissemination_general gen ORDER BY gen.id target_options: - pre_sql: | - DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_additional_eins; - DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_additional_ueis; - DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_combined; - DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_corrective_action_plans; - DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_federal_awards; - DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_findings_text; - DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_findings; - DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_general; - DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_notes_to_sefa; - DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_passthrough; - DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_secondary_auditors; - -- Now, create them. We get a start at 1 for every refresh this way. - CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_additional_eins START 1; - CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_additional_ueis START 1; - CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_combined START 1; - CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_corrective_action_plans START 1; - CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_federal_awards START 1; - CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_findings_text START 1; - CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_findings START 1; - CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_general START 1; - CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_notes_to_sefa START 1; - CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_passthrough START 1; - CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_secondary_auditors START 1; post_sql: | -- Add a clean batch number after the table is created. ALTER TABLE public_data_v1_0_0.general ADD COLUMN batch_number INTEGER; - -- Update that batch number as a modulus of 20K, our response - -- size limit in PostgREST. UPDATE public_data_v1_0_0.general SET batch_number=DIV(seq, 20000); - -- Create the indexes on the table. - CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_report_id_idx - on public_data_v1_0_0.general (report_id); - CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_report_id_fad - on public_data_v1_0_0.general (report_id, fac_accepted_date); - CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_report_id_fad - on public_data_v1_0_0.general (report_id, fac_accepted_date); - CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditee_certify_name_idx - ON public_data_v1_0_0.general - ((lower(auditee_certify_name))); - CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditee_name_idx - ON public_data_v1_0_0.general - ((lower(auditee_name))); - CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditor_certify_name_idx - ON public_data_v1_0_0.general - ((lower(auditor_certify_name))); - CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditor_contact_name_idx - ON public_data_v1_0_0.general - ((lower(auditor_contact_name))); - CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditor_firm_name_idx - ON public_data_v1_0_0.general - ((lower(auditor_firm_name))); - CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditee_email_idx - on public_data_v1_0_0.general ((lower(auditee_email))); - CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditor_email_idx - on public_data_v1_0_0.general ((lower(auditor_email))); - CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_start_date_idx - ON public_data_v1_0_0.general (fy_start_date); - CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_end_date_idx - ON public_data_v1_0_0.general (fy_end_date); - CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditee_uei_idx - ON public_data_v1_0_0.general (auditee_uei); - CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_auditee_ein_idx - ON public_data_v1_0_0.general (auditee_ein); - CREATE INDEX IF NOT EXISTS fac_snapshot_db_general_audit_year_idx - on public_data_v1_0_0.general (audit_year); - CREATE INDEX IF NOT EXISTS fac_snapshot_db_federal_batch_general_idx - ON public_data_v1_0_0.general (batch_number); - - # All additional EINs are public. public.additional_eins: @@ -194,7 +124,7 @@ streams: sql: | SELECT ein.id as id, - div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_additional_eins'), 20000) as batch_number, + div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_additional_eins'), 20000) as seq, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -207,6 +137,12 @@ streams: WHERE gen.report_id = ein.report_id ORDER BY ein.id + target_options: + post_sql: | + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.additional_eins + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.additional_eins SET batch_number=DIV(seq, 20000); # All of the additional UEI info is public info. public.dissemination_additional_ueis: @@ -215,7 +151,7 @@ streams: sql: | SELECT uei.id as id, - div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_additional_ueis'), 20000) as batch_number, + div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_additional_ueis'), 20000) as seq, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -228,6 +164,12 @@ streams: WHERE gen.report_id = uei.report_id ORDER BY uei.id + target_options: + post_sql: | + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.additional_eins + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.additional_eins SET batch_number=DIV(seq, 20000); # Corrective action plans are NOT always public. public.dissemination_corrective_action_plan: @@ -236,7 +178,7 @@ streams: sql: | SELECT cap.id as id, - div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_corrective_action_plans'), 20000) as batch_number, + div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_corrective_action_plans'), 20000) as seq, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -254,6 +196,12 @@ streams: -- Only include the public corrective action plans. gen.is_public = true ORDER BY cap.id + target_options: + post_sql: | + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.corrective_action_plans + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.corrective_action_plans SET batch_number=DIV(seq, 20000); # All Federal award data is public. public.dissemination_federalaward: @@ -262,7 +210,7 @@ streams: sql: | SELECT award.id as id, - div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_federal_awards'), 20000) as batch_number, + div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_federal_awards'), 20000) as seq, dg.report_id, dg.auditee_uei, dg.audit_year, @@ -294,6 +242,12 @@ streams: WHERE award.report_id = dg.report_id ORDER BY award.id + target_options: + post_sql: | + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.federal_awards + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.federal_awards SET batch_number=DIV(seq, 20000); # The findings table is public. public.dissemination_findings: @@ -302,7 +256,7 @@ streams: sql: | SELECT finding.id as id, - div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_findings'), 20000) as batch_number, + div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_findings'), 20000) as seq, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -325,6 +279,12 @@ streams: WHERE finding.report_id = gen.report_id ORDER BY finding.id + target_options: + post_sql: | + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.findings + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.findings SET batch_number=DIV(seq, 20000); # Findings text is NOT always public. public.dissemination_findingstext: @@ -333,7 +293,7 @@ streams: sql: | SELECT ft.id as id, - div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_findings_text'), 20000) as batch_number, + div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_findings_text'), 20000) as seq, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -349,6 +309,12 @@ streams: AND gen.is_public = true ORDER BY ft.id + target_options: + post_sql: | + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.findings_text + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.findings_text SET batch_number=DIV(seq, 20000); # The notes to SEFA are NOT all public. public.dissemination_notes: @@ -357,7 +323,7 @@ streams: sql: | SELECT note.id as id, - div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_notes_to_sefa'), 20000) as batch_number, + div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_notes_to_sefa'), 20000) as seq, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -378,6 +344,12 @@ streams: -- Some notes are not public. gen.is_public = true ORDER BY note.id + target_options: + post_sql: | + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.notes_to_sefa + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.notes_to_sefa SET batch_number=DIV(seq, 20000); # All passthrough information is public. public.dissemination_passthrough: @@ -386,7 +358,7 @@ streams: sql: | SELECT pass.id as id, - div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_passthrough'), 20000) as batch_number, + div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_passthrough'), 20000) as seq, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -401,6 +373,12 @@ streams: WHERE gen.report_id = pass.report_id ORDER BY pass.id + target_options: + post_sql: | + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.passthrough + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.passthrough SET batch_number=DIV(seq, 20000); # All secondary auditor info is public. public.secondary_auditors: @@ -409,7 +387,7 @@ streams: sql: | SELECT sa.id as id, - div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_secondary_auditors'), 20000) as batch_number, + div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_secondary_auditors'), 20000) as seq, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -431,6 +409,12 @@ streams: WHERE sa.report_id = gen.report_id ORDER BY sa.id + target_options: + post_sql: | + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.secondary_auditors + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.secondary_auditors SET batch_number=DIV(seq, 20000); public.combined: mode: full-refresh @@ -438,7 +422,7 @@ streams: sql: | SELECT dg.report_id, - div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_combined'), 20000) as batch_number, + div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_combined'), 20000) as seq, dfa.award_reference, df.reference_number, concat(dfa.federal_agency_prefix,'.',dfa.federal_award_extension) as aln, @@ -558,3 +542,9 @@ streams: LEFT JOIN public.dissemination_passthrough dp ON dfa.report_id = dp.report_id AND dfa.award_reference = dp.award_reference + target_options: + post_sql: | + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.combined + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.combined SET batch_number=DIV(seq, 20000); diff --git a/backend/dissemination/sql/sling/public_data_v1_0_0/public_metadata_v1_0_0.yaml b/backend/dissemination/sql/sling/public_data_v1_0_0/public_metadata_v1_0_0.yaml index 1d22a3c7aa..eb6735387c 100644 --- a/backend/dissemination/sql/sling/public_data_v1_0_0/public_metadata_v1_0_0.yaml +++ b/backend/dissemination/sql/sling/public_data_v1_0_0/public_metadata_v1_0_0.yaml @@ -21,53 +21,53 @@ streams: object: public_data_v1_0_0.metadata sql: | SELECT 'additional_eins' - AS table, COUNT(*) --, public_api_v1_0_0_functions.batches('additional_eins') + AS table, COUNT(*) --, api_v2_0_0_functions.batches('additional_eins') FROM public_data_v1_0_0.additional_eins UNION SELECT 'additional_ueis' - AS table, COUNT(*) --, public_api_v1_0_0_functions.batches('additional_ueis') + AS table, COUNT(*) --, api_v2_0_0_functions.batches('additional_ueis') FROM public_data_v1_0_0.additional_ueis UNION SELECT 'combined' - AS table, COUNT(*) --, public_api_v1_0_0_functions.batches('combined') + AS table, COUNT(*) --, api_v2_0_0_functions.batches('combined') FROM public_data_v1_0_0.combined UNION SELECT 'federal_awards' - AS table, COUNT(*) --, public_api_v1_0_0_functions.batches('federal_awards') + AS table, COUNT(*) --, api_v2_0_0_functions.batches('federal_awards') FROM public_data_v1_0_0.federal_awards UNION SELECT 'findings_text' - AS table, COUNT(*) --, public_api_v1_0_0_functions.batches('findings_text') + AS table, COUNT(*) --, api_v2_0_0_functions.batches('findings_text') FROM public_data_v1_0_0.findings_text UNION SELECT 'findings' - AS table, COUNT(*) --, public_api_v1_0_0_functions.batches('findings') + AS table, COUNT(*) --, api_v2_0_0_functions.batches('findings') FROM public_data_v1_0_0.findings UNION SELECT 'general' - AS table, COUNT(*) --, public_api_v1_0_0_functions.batches('general') + AS table, COUNT(*) --, api_v2_0_0_functions.batches('general') FROM public_data_v1_0_0.general UNION SELECT 'notes_to_sefa' - AS table, COUNT(*) --, public_api_v1_0_0_functions.batches('notes_to_sefa') + AS table, COUNT(*) --, api_v2_0_0_functions.batches('notes_to_sefa') FROM public_data_v1_0_0.notes_to_sefa UNION SELECT 'passthrough' - AS table, COUNT(*) --, public_api_v1_0_0_functions.batches('passthrough') + AS table, COUNT(*) --, api_v2_0_0_functions.batches('passthrough') FROM public_data_v1_0_0.passthrough UNION SELECT 'secondary_auditors' - AS table, COUNT(*) --, public_api_v1_0_0_functions.batches('secondary_auditors') + AS table, COUNT(*) --, api_v2_0_0_functions.batches('secondary_auditors') FROM public_data_v1_0_0.secondary_auditors UNION SELECT 'tribal_corrective_action_plans' - AS table, COUNT(*) --, public_api_v1_0_0_functions.batches('tribal_corrective_action_plans') + AS table, COUNT(*) --, api_v2_0_0_functions.batches('tribal_corrective_action_plans') FROM public_data_v1_0_0.tribal_corrective_action_plans UNION SELECT 'tribal_findings_text' - AS table, COUNT(*) --, public_api_v1_0_0_functions.batches('tribal_findings_text') + AS table, COUNT(*) --, api_v2_0_0_functions.batches('tribal_findings_text') FROM public_data_v1_0_0.tribal_findings_text UNION SELECT 'tribal_notes_to_sefa' - AS table, COUNT(*) --, public_api_v1_0_0_functions.batches('tribal_notes_to_sefa') + AS table, COUNT(*) --, api_v2_0_0_functions.batches('tribal_notes_to_sefa') FROM public_data_v1_0_0.tribal_notes_to_sefa diff --git a/backend/dissemination/sql/test_api.py b/backend/dissemination/sql/test_api.py new file mode 100644 index 0000000000..1c29fa88d0 --- /dev/null +++ b/backend/dissemination/sql/test_api.py @@ -0,0 +1,172 @@ +# +# Execute as a pytest. +# pytest -s --env local test_api.py +# + +import click +import os +import requests +import sys +import uuid + + +class EnvVars: + FAC_API_KEY = os.getenv("FAC_API_KEY") + FAC_API_KEY_ID = os.getenv("CYPRESS_API_GOV_USER_ID") + FAC_AUTH_BEARER = os.getenv("CYPRESS_API_GOV_JWT") + RECORDS_REQUESTED = 5 + + +def url(env): + EnvVars.FAC_API_KEY = os.getenv("FAC_API_KEY") + match env: + case "local": + return "http://localhost:3000" + case "preview": + return "https://api-preview.fac.gov" + case "dev": + return "https://api-dev.fac.gov" + case "staging": + return "https://api-staging.fac.gov" + case "production": + return "https://api.fac.gov" + case _: + print("No environment provided; exiting.") + sys.exit() + + +def headers(env): + if env in ["local"]: + if EnvVars.FAC_AUTH_BEARER == None: + print("FAC_AUTH_BEARER not set.") + sys.exit() + if EnvVars.FAC_API_KEY_ID == None: + print("FAC_API_KEY_ID not set.") + sys.exit() + return { + "authorization": f"bearer {EnvVars.FAC_AUTH_BEARER}", + "x-api-user-id": EnvVars.FAC_API_KEY_ID, + } + elif env in ["preview", "dev", "staging", "production"]: + if EnvVars.FAC_API_KEY == None: + print("FAC_API_KEY not set.") + sys.exit() + return { + "x-api-key": EnvVars.FAC_API_KEY, + } + else: + print("No environment matches for header construction.") + sys.exit() + + +def api(version): + return {"accept-profile": version} + + +def limit(start, end): + return {"Range-Unit": "items", "Range": f"{start}-{end}"} + + +# Asserts that an API response is: +# * A list +# * A list composed of objects that all contain the required keys +def good_resp(objs, keys): + assert isinstance(objs, list) + assert len(objs) == EnvVars.RECORDS_REQUESTED + for k in keys: + for o in objs: + # print(f"Checking {k} in {o}") + assert k in o + return True + + +# Constructs the base URL for making multiple API calls off of. +def cons(env, api_version): + def _helper(endpoint, keys): + base = url(env) + h = headers(env) | limit(0, EnvVars.RECORDS_REQUESTED - 1) | api(api_version) + r = requests.get(base + f"/{endpoint}", headers=h) + # print(r.request.url) + # print(r.request.headers) + good_resp(r.json(), keys) + + return _helper + + +# These tables are common to both the old API and +# the new public data API. +def common_tables(f): + f("general", ["report_id", "audit_year", "auditee_name"]) + f( + "federal_awards", + ["report_id", "amount_expended", "audit_report_type"], + ) + f("corrective_action_plans", ["report_id", "finding_ref_number", "auditee_uei"]) + + +def test_api_v1_0_3_not_exist(env): + f = cons(env, "api_v1_0_3") + try: + common_tables(f) + print("This schema/API should not exist.") + assert False + except: + pass + + +def test_api_v1_1_0(env): + f = cons(env, "api_v1_1_0") + common_tables(f) + + +def test_api_v2_0_0(env): + f = cons(env, "api_v2_0_0") + common_tables(f) + + +def test_suppressed_not_accessible_with_bad_key(env): + # Stash the token, and wipe it out, so the API + # calls will fail. + TEMP_FAC_API_KEY_ID = EnvVars.FAC_API_KEY_ID + EnvVars.FAC_API_KEY_ID = str(uuid.uuid4()) + f = cons(env, "api_v2_0_0") + failed_count = 0 + for thunk in [ + lambda: f( + "suppressed_notes_to_sefa", ["report_id", "content", "is_minimis_rate_used"] + ), + lambda: f("suppressed_findings_text", ["report_id", "finding_ref_number"]), + lambda: f( + "suppressed_corrective_action_plans", + ["report_id", "finding_ref_number", "planned_action"], + ), + ]: + try: + thunk() + except: + failed_count += 1 + assert failed_count == 3 + # Restore it in case we need it in later tests. + EnvVars.FAC_API_KEY_ID = TEMP_FAC_API_KEY_ID + + +def test_suppressed_accessible_with_good_key(env): + # Stash the token, and wipe it out, so the API + # calls will fail. + f = cons(env, "api_v2_0_0") + failed_count = 0 + for thunk in [ + lambda: f( + "suppressed_notes_to_sefa", ["report_id", "content", "is_minimis_rate_used"] + ), + lambda: f("suppressed_findings_text", ["report_id", "finding_ref_number"]), + lambda: f( + "suppressed_corrective_action_plans", + ["report_id", "finding_ref_number", "planned_action"], + ), + ]: + try: + thunk() + except: + failed_count += 1 + assert failed_count == 0 diff --git a/backend/docker-compose-db-only.yml b/backend/docker-compose-db-only.yml index c4ac82d8a9..14e962c4a8 100644 --- a/backend/docker-compose-db-only.yml +++ b/backend/docker-compose-db-only.yml @@ -58,7 +58,7 @@ services: PGRST_OPENAPI_SERVER_PROXY_URI: http://127.0.0.1:3000 PGRST_DB_ANON_ROLE: anon # See https://postgrest.org/en/stable/references/api/schemas.html#multiple-schemas for multiple schemas - PGRST_DB_SCHEMAS: "api_v1_0_3,api_v1_1_0,admin_api_v1_1_0,public_api_v1_0_0" + PGRST_DB_SCHEMAS: "api_v1_0_3,api_v1_1_0,admin_api_v1_1_0,api_v2_0_0" PGRST_JWT_SECRET: ${PGRST_JWT_SECRET:-32_chars_fallback_secret_testing} # Fallback value for testing environments # Enable this to inspect the DB plans for queries via EXPLAIN PGRST_DB_PLAN_ENABLED: 1 diff --git a/backend/docker-compose.yml b/backend/docker-compose.yml index 29d260faf7..13dc431571 100644 --- a/backend/docker-compose.yml +++ b/backend/docker-compose.yml @@ -78,7 +78,6 @@ services: DISABLE_AUTH: ${DISABLE_AUTH:-False} PGRST_JWT_SECRET: ${PGRST_JWT_SECRET:-32_chars_fallback_secret_testing} ENABLE_DEBUG_TOOLBAR: false - env_file: - ".env" ports: @@ -103,15 +102,35 @@ services: #--------------------------------------------- # Minio (S3 clone) #--------------------------------------------- + # minio: + # container_name: "minio" + # image: minio/minio + # command: server /tmp/minio --console-address ":9002" + # ports: + # - "9001:9000" + # - "9002:9002" + # volumes: + # - "minio-vol:/tmp/minio" minio: - container_name: "minio" - image: minio/minio - command: server /tmp/minio --console-address ":9002" + image: minio/minio:latest + command: server /data --console-address ":9002" + expose: + - "9001" + - "9002" ports: - - "9001:9000" - - "9002:9002" + - 9001:9000 + - 9002:9002 + environment: + MINIO_ROOT_USER: nutnutnut + MINIO_ROOT_PASSWORD: nutnutnut + MINIO_API_ROOT_ACCESS: on + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 5s + retries: 5 volumes: - - "minio-vol:/tmp/minio" + - minio-vol:/data #--------------------------------------------- # PostgREST API provider @@ -129,7 +148,7 @@ services: PGRST_OPENAPI_SERVER_PROXY_URI: http://127.0.0.1:3000 PGRST_DB_ANON_ROLE: anon # See https://postgrest.org/en/stable/references/api/schemas.html#multiple-schemas for multiple schemas - PGRST_DB_SCHEMAS: "api_v1_0_3,api_v1_1_0,admin_api_v1_1_0,public_api_v1_0_0" + PGRST_DB_SCHEMAS: "api_v1_0_3,api_v1_1_0,admin_api_v1_1_0,api_v2_0_0" PGRST_JWT_SECRET: ${PGRST_JWT_SECRET:-32_chars_fallback_secret_testing} # Fallback value for testing environments # Enable this to inspect the DB plans for queries via EXPLAIN PGRST_DB_PLAN_ENABLED: 1 diff --git a/backend/fac-backup-util.sh b/backend/fac-backup-util.sh index ab6e2165e4..9e66327fcd 100755 --- a/backend/fac-backup-util.sh +++ b/backend/fac-backup-util.sh @@ -49,6 +49,12 @@ elif [ "$run_option" == "deploy_backup" ]; then gonogo "db_to_db" AWSS3Sync "$s3_name" "$backup_s3_name" gonogo "s3_sync" +elif [ "$run_option" == "rds_backup" ]; then + GetUtil + InstallAWS + gonogo "install_aws" + RDSToRDS "$db_name" "$backup_db_name" "backup" + gonogo "db_to_db" elif [ "$run_option" == "scheduled_backup" ]; then GetUtil InstallAWS diff --git a/backend/run.sh b/backend/run.sh index a9206ab8d4..dc0bc4993d 100755 --- a/backend/run.sh +++ b/backend/run.sh @@ -8,8 +8,7 @@ source tools/migrate_app_tables.sh source tools/seed_cog_baseline.sh source tools/setup_env.sh -source tools/sling_first_run.sh -source tools/sling_to_sqlite_in_s3.sh +# source tools/sling_first_run.sh source tools/sql_pre_post.sh source tools/util_startup.sh @@ -34,8 +33,8 @@ gonogo "migrate_app_tables" # MOVE DATA TO SECOND DB # This runs sling and preps tables in the snapshot DB. # Only runs if the tables are not present (e.g. first deploy) -sling_first_run -gonogo "sling_first_run" +# sling_first_run +# gonogo "sling_first_run" ##### # SQL POST diff --git a/backend/tools/remove_api_from_fac_db.sh b/backend/tools/remove_api_from_fac_db.sh deleted file mode 100644 index 8c93a92bd1..0000000000 --- a/backend/tools/remove_api_from_fac_db.sh +++ /dev/null @@ -1,3 +0,0 @@ -# run this once in each env -# after we deploy the new API and everything -# is happy. diff --git a/backend/tools/setup_local_env.sh b/backend/tools/setup_local_env.sh index 64c6a685db..3cf4d209e5 100644 --- a/backend/tools/setup_local_env.sh +++ b/backend/tools/setup_local_env.sh @@ -6,21 +6,22 @@ function setup_local_env { startup_log "LOCAL_ENV" "We are in a local envirnoment." export AWS_PUBLIC_BUCKET_NAME="fac-public-s3" - export AWS_PRIVATE_BUCKET_NAME="fac-public-s3" + export AWS_PRIVATE_BUCKET_NAME="fac-private-s3" - export AWS_PRIVATE_ACCESS_KEY_ID="longtest" - export AWS_PRIVATE_SECRET_ACCESS_KEY="longtest" - export AWS_PRIVATE_ENDPOINT="http://minio:9001" + export AWS_PRIVATE_ACCESS_KEY_ID="nutnutnut" + export AWS_PRIVATE_SECRET_ACCESS_KEY="nutnutnut" + export AWS_S3_PRIVATE_ENDPOINT="http://minio:9001" - export AWS_PUBLIC_ACCESS_KEY_ID="longtest" - export AWS_PUBLIC_SECRET_ACCESS_KEY="longtest" - export AWS_PUBLIC_ENDPOINT="http://minio:9001" + export AWS_PUBLIC_ACCESS_KEY_ID="nutnutnut" + export AWS_PUBLIC_SECRET_ACCESS_KEY="nutnutnut" + export AWS_S3_PUBLIC_ENDPOINT="http://minio:9001" - mc alias set myminio "${AWS_S3_PRIVATE_ENDPOINT}" minioadmin minioadmin + mc alias set myminio ${AWS_S3_PRIVATE_ENDPOINT} ${AWS_PRIVATE_ACCESS_KEY_ID} ${AWS_PRIVATE_ACCESS_KEY_ID} + # until (mc config host add myminio $AWS_PRIVATE_ENDPOINT nutnutnut nutnutnut) do echo '...waiting...' && sleep 1; done; # Do nothing if the bucket already exists. # https://min.io/docs/minio/linux/reference/minio-mc/mc-mb.html - mc mb --ignore-existing myminio/fac-private-s3 - mc mb --ignore-existing myminio/fac-public-s3 + mc mb --ignore-existing myminio/${AWS_PUBLIC_BUCKET_NAME} + mc mb --ignore-existing myminio/${AWS_PRIVATE_BUCKET_NAME} # MCJ 20241016 FIXME: Is this even needed locally? I don't think so. # mc admin user svcacct add \ diff --git a/backend/tools/sling_first_run.sh b/backend/tools/sling_first_run.sh index aea89cc842..8e997a5df4 100644 --- a/backend/tools/sling_first_run.sh +++ b/backend/tools/sling_first_run.sh @@ -36,7 +36,7 @@ function sling_first_run() { startup_log "SLING_FIRST_RUN" "API tables don't exist; running sling." $SLING_EXE run -r dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml gonogo "sling public data for API tables" - $SLING_EXE run -r dissemination/sql/sling/public_data_v1_0_0/tribal_data_v1_0_0.yaml + $SLING_EXE run -r dissemination/sql/sling/public_data_v1_0_0/suppressed_data_v1_0_0.yaml gonogo "sling tribal data for API tables" $SLING_EXE run -r dissemination/sql/sling/public_data_v1_0_0/public_metadata_v1_0_0.yaml gonogo "sling metadata table" diff --git a/backend/util/api_perf_test/perf.py b/backend/util/api_perf_test/perf.py index 9c7043082c..cb3e1ac3b0 100644 --- a/backend/util/api_perf_test/perf.py +++ b/backend/util/api_perf_test/perf.py @@ -5,30 +5,25 @@ import math import json -URI = "http://localhost:8080" +URI = "http://localhost:3000" # GET {{scheme}}://{{apiUrl}}/general?report_id=eq.2021-12-CENSUS-0000250449 # authorization: {{authorization}} # x-api-user-id: {{xApiUserId}} -# accept-profile: public_api_v1_0_0 +# accept-profile: api_v2_0_0 # Accept: application/vnd.pgrst.plan -NOT_ACTUALLY_SECRET = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.zJrV44Lhr1Ck4vg1dMnldql0adLgut241jo0FbFXMlI" -MORE_SECRET = os.getenv("CYPRESS_API_GOV_JWT") -SEKRET = NOT_ACTUALLY_SECRET - def fetch_fa_exp(api_version): total_cost = 0 for offset in range(0, 4000000, 20000): print(f"fetch_fa_exp api {api_version} offset {offset}") - query = f"{URI}/federal_awards" # ?limit=20000&offset={offset} + query = f"{URI}/federal_awards?limit=20000&offset={offset}" headers = { "accept-profile": api_version, "accept": "application/vnd.pgrst.plan+json", "x-api-user-id": os.getenv("API_KEY_ID"), - "authorization": f"bearer {SEKRET}", - "range": f"{offset}-{offset+19999}", + "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", } resp = requests.get(query, headers=headers) @@ -45,17 +40,15 @@ def fetch_fa_by_year_exp(api_version): print( f"fetch_fa_by_year_exp api {api_version} ay {audit_year} offset {offset}" ) - query = f"{URI}/federal_awards?audit_year=eq.{audit_year}" # &limit=20000&offset={offset}" + query = f"{URI}/federal_awards?audit_year=eq.{audit_year}&limit=20000&offset={offset}" headers = { "accept-profile": api_version, "accept": "application/vnd.pgrst.plan+json", "x-api-user-id": os.getenv("API_KEY_ID"), - "authorization": f"bearer {SEKRET}", - "range": f"{offset}-{offset+19999}", + "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", } resp = requests.get(query, headers=headers) - pprint(resp) # We get back a list of one plan, and we want the total cost. total_cost += resp.json()[0]["Plan"]["Total Cost"] return math.floor(total_cost) @@ -68,7 +61,7 @@ def fetch_fa_time(api_version): headers = { "accept-profile": api_version, "x-api-user-id": os.getenv("API_KEY_ID"), - "authorization": f"bearer {SEKRET}", + "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", } t0 = time.time() resp = requests.get(query, headers=headers) @@ -84,12 +77,11 @@ def fetch_fa_time_by_year(api_version): for year in range(16, 24): for offset in range(0, 1000000, 20000): audit_year = f"20{year:02}" - query = f"{URI}/federal_awards?audit_year=eq.{audit_year}" # &limit=20000&offset={offset}" + query = f"{URI}/federal_awards?audit_year=eq.{audit_year}&limit=20000&offset={offset}" headers = { "accept-profile": api_version, "x-api-user-id": os.getenv("API_KEY_ID"), - "authorization": f"bearer {SEKRET}", - "range": f"{offset}-{offset+19999}", + "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", } t0 = time.time() resp = requests.get(query, headers=headers) @@ -107,10 +99,10 @@ def fetch_fa_batches_exp(): for batch_no in range(0, 235): query = f"{URI}/federal_awards?batch_number=eq.{batch_no}" headers = { - "accept-profile": "public_api_v1_0_0", + "accept-profile": "api_v2_0_0", "accept": "application/vnd.pgrst.plan+json", "x-api-user-id": os.getenv("API_KEY_ID"), - "authorization": f"bearer {SEKRET}", + "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", } resp = requests.get(query, headers=headers) @@ -125,9 +117,9 @@ def fetch_fa_batches_time(): print(f"batch number: {batch_no}") query = f"{URI}/federal_awards?batch_number=eq.{batch_no}" headers = { - "accept-profile": "public_api_v1_0_0", + "accept-profile": "api_v2_0_0", "x-api-user-id": os.getenv("API_KEY_ID"), - "authorization": f"bearer {SEKRET}", + "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", } t0 = time.time() resp = requests.get(query, headers=headers) @@ -152,16 +144,16 @@ def make_ratios(d1, d2): results3 = {} results4 = {} - # results1["api110_by_year"] = fetch_fa_by_year_exp("api_v1_1_0") - # results1["ap110"] = fetch_fa_exp("api_v1_1_0") - # results1["public100"] = fetch_fa_exp("public_api_v1_0_0") - # results1["public100_batches"] = fetch_fa_batches_exp() - # results1["public100_by_year"] = fetch_fa_by_year_exp("public_api_v1_0_0") + results1["api110_by_year"] = fetch_fa_by_year_exp("api_v1_1_0") + results1["ap110"] = fetch_fa_exp("api_v1_1_0") + results1["public100"] = fetch_fa_exp("api_v2_0_0") + results1["public100_batches"] = fetch_fa_batches_exp() + results1["public100_by_year"] = fetch_fa_by_year_exp("api_v2_0_0") print("Running timing tests... ~5m") - results3["public100_by_year"] = fetch_fa_time_by_year("public_api_v1_0_0") - results3["public100"] = fetch_fa_time("public_api_v1_0_0") + results3["public100_by_year"] = fetch_fa_time_by_year("api_v2_0_0") + results3["public100"] = fetch_fa_time("api_v2_0_0") results3["public100_batches"] = fetch_fa_batches_time() results3["ap110"] = fetch_fa_time("api_v1_1_0") results3["ap110_by_year"] = fetch_fa_time_by_year("api_v1_1_0") diff --git a/backend/util/fac-backup-and-sling.sh b/backend/util/fac-backup-and-sling.sh deleted file mode 100755 index f4d54550f3..0000000000 --- a/backend/util/fac-backup-and-sling.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/bin/bash -set -e -source tools/util_startup.sh -source tools/setup_env.sh -setup_env - -version=$1 -run_option=$2 -s3_name="fac-private-s3" -backup_s3_name="backups" -db_name="fac-db" -backup_db_name="fac-snapshot-db" -initial_date=$(date +%Y%m%d%H%M) -scheduled_date=$(date +%m-%d-%H) -daily_date=$(date +%m-%d) - -mkdir tmp && cd tmp || return - -GetUtil() { - curl -x "$https_proxy" -L "https://github.com/GSA-TTS/fac-backup-utility/releases/download/$version/gov.gsa.fac.cgov-util-$version-linux-amd64.tar.gz" -O - tar -xvf "gov.gsa.fac.cgov-util-$version-linux-amd64.tar.gz" && rm "gov.gsa.fac.cgov-util-$version-linux-amd64.tar.gz" -} - -GetSling() { - curl -x "$https_proxy" -LO 'https://github.com/slingdata-io/sling-cli/releases/latest/download/sling_linux_amd64.tar.gz' - tar xf sling_linux_amd64.tar.gz - rm -f sling_linux_amd64.tar.gz - chmod +x sling -} - -RunSling() { - ./sling run -r dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml - ./sling run -r dissemination/sql/sling/public_data_v1_0_0/tribal_data_v1_0_0.yaml -} - -InstallAWS() { - ./gov.gsa.fac.cgov-util install_aws -} - -AWSS3Sync() { - ./gov.gsa.fac.cgov-util s3_sync --source_s3 s3://"$1"/ --dest_s3 s3://"$2"/ -} - -RDSToS3Dump() { - ./gov.gsa.fac.cgov-util db_to_s3 --db "$1" --s3path s3://"$2"/"$3"/ -} - -RDSToRDS() { - ./gov.gsa.fac.cgov-util db_to_db --src_db "$1" --dest_db "$2" --operation "$3" -} - -if [ "$run_option" == "initial_backup" ]; then - GetUtil - InstallAWS - gonogo "install_aws" - GetSling - gonogo "insall_sling" - RDSToS3Dump "$db_name" "$backup_s3_name" "initial/$initial_date" - gonogo "db_to_s3" - RDSToRDS "$db_name" "$backup_db_name" "initial" - gonogo "db_to_db" - AWSS3Sync "$s3_name" "$backup_s3_name" - gonogo "s3_sync" - RunSling - gonogo "run_sling" -elif [ "$run_option" == "deploy_backup" ]; then - GetUtil - InstallAWS - gonogo "install_aws" - GetSling - gonogo "insall_sling" - RDSToRDS "$db_name" "$backup_db_name" "backup" - gonogo "db_to_db" - AWSS3Sync "$s3_name" "$backup_s3_name" - gonogo "s3_sync" - RunSling - gonogo "run_sling" -elif [ "$run_option" == "scheduled_backup" ]; then - GetUtil - InstallAWS - gonogo "install_aws" - RDSToS3Dump "$db_name" "$backup_s3_name" "scheduled/$scheduled_date" - gonogo "db_to_s3" - AWSS3Sync "$s3_name" "$backup_s3_name" - gonogo "s3_sync" -elif [ "$run_option" == "daily_backup" ]; then - GetUtil - InstallAWS - gonogo "install_aws" - GetSling - gonogo "insall_sling" - RDSToRDS "$db_name" "$backup_db_name" "backup" - gonogo "db_to_db" - RDSToS3Dump "$db_name" "$backup_s3_name" "daily/$daily_date" - gonogo "db_to_s3" - AWSS3Sync "$s3_name" "$backup_s3_name" - gonogo "s3_sync" - RunSling - gonogo "run_sling" -elif [ "$run_option" == "media_sync" ]; then - GetUtil - InstallAWS - gonogo "install_aws" - AWSS3Sync "$s3_name" "$backup_s3_name" - gonogo "s3_sync" -fi diff --git a/terraform/shared/modules/env/postgrest.tf b/terraform/shared/modules/env/postgrest.tf index 772612dac0..dbbe1fbfc2 100644 --- a/terraform/shared/modules/env/postgrest.tf +++ b/terraform/shared/modules/env/postgrest.tf @@ -32,7 +32,7 @@ resource "cloudfoundry_app" "postgrest" { environment = { PGRST_DB_URI : cloudfoundry_service_key.postgrest.credentials.uri - PGRST_DB_SCHEMAS : "api_v1_0_3,api_v1_1_0,admin_api_v1_1_0,public_api_v1_0_0" + PGRST_DB_SCHEMAS : "api_v1_0_3,api_v1_1_0,admin_api_v1_1_0,api_v2_0_0" PGRST_DB_ANON_ROLE : "anon" PGRST_JWT_SECRET : var.pgrst_jwt_secret PGRST_DB_MAX_ROWS : 20000 From 2ea604a68c52e42c31c876a465e43996c39a14f7 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Thu, 24 Oct 2024 18:05:48 -0400 Subject: [PATCH 37/89] Simulating pre-deploy backup Our local environment has to make sure that fac-snapshot-db looks "right" before we do anything. This means making sure we run cgov-util to do a snapshot before we do anything pre/post with SQL. --- .../post/010_remove_old_census_data.sql | 90 +++++++++++++++++++ .../post/010_copy_dissem_tables.sql | 86 ++++++++++++++++++ .../post/0144_api_v2_0_0_rpc.sql | 10 +-- .../post/020_public_data_v1_0_0_tables.sql | 26 +++--- .../021_suppressed_data_v1_0_0_tables.sql | 6 +- .../post/122_api_v1_1_0_functions.sql | 2 +- .../post/124_api_v1_1_0_rpc.sql | 10 +-- .../post/142_api_v2_0_0_functions.sql | 2 +- .../sql/fac-snapshot-db/pre/060_schemas.sql | 3 +- backend/run.sh | 11 ++- backend/tools/cgov_util_local_only.sh | 19 ++++ backend/tools/setup_cgov_env.py | 80 +++++++++++++++++ backend/tools/sql_pre_post.sh | 2 + 13 files changed, 317 insertions(+), 30 deletions(-) create mode 100644 backend/dissemination/sql/fac-db/post/010_remove_old_census_data.sql create mode 100644 backend/dissemination/sql/fac-snapshot-db/post/010_copy_dissem_tables.sql create mode 100644 backend/tools/cgov_util_local_only.sh create mode 100644 backend/tools/setup_cgov_env.py diff --git a/backend/dissemination/sql/fac-db/post/010_remove_old_census_data.sql b/backend/dissemination/sql/fac-db/post/010_remove_old_census_data.sql new file mode 100644 index 0000000000..846541e7a0 --- /dev/null +++ b/backend/dissemination/sql/fac-db/post/010_remove_old_census_data.sql @@ -0,0 +1,90 @@ + +-- but the tables are still in prod. This will remove them +-- from the prod environment, where they are no longer needed. +-- OLD RAW CENSUS DATA +-- our own data for this. We have removed this from the deploy, +-- Specifically, it was used in cog/over. However, we now use +-- This data was an early part of the data migration. +-------------------------------------- +-------------------------------------- +DROP TABLE IF EXISTS census_agency16 CASCADE; +DROP TABLE IF EXISTS census_agency17 CASCADE; +DROP TABLE IF EXISTS census_agency18 CASCADE; +DROP TABLE IF EXISTS census_agency19 CASCADE; +DROP TABLE IF EXISTS census_agency20 CASCADE; +DROP TABLE IF EXISTS census_agency21 CASCADE; +DROP TABLE IF EXISTS census_agency22 CASCADE; +DROP TABLE IF EXISTS census_captext_formatted19 CASCADE; +DROP TABLE IF EXISTS census_captext_formatted20 CASCADE; +DROP TABLE IF EXISTS census_captext_formatted21 CASCADE; +DROP TABLE IF EXISTS census_captext_formatted22 CASCADE; +DROP TABLE IF EXISTS census_captext19 CASCADE; +DROP TABLE IF EXISTS census_captext20 CASCADE; +DROP TABLE IF EXISTS census_captext21 CASCADE; +DROP TABLE IF EXISTS census_captext22 CASCADE; +DROP TABLE IF EXISTS census_cfda16 CASCADE; +DROP TABLE IF EXISTS census_cfda17 CASCADE; +DROP TABLE IF EXISTS census_cfda18 CASCADE; +DROP TABLE IF EXISTS census_cfda19 CASCADE; +DROP TABLE IF EXISTS census_cfda20 CASCADE; +DROP TABLE IF EXISTS census_cfda21 CASCADE; +DROP TABLE IF EXISTS census_cfda22 CASCADE; +DROP TABLE IF EXISTS census_cpas16 CASCADE; +DROP TABLE IF EXISTS census_cpas17 CASCADE; +DROP TABLE IF EXISTS census_cpas18 CASCADE; +DROP TABLE IF EXISTS census_cpas19 CASCADE; +DROP TABLE IF EXISTS census_cpas20 CASCADE; +DROP TABLE IF EXISTS census_cpas21 CASCADE; +DROP TABLE IF EXISTS census_cpas22 CASCADE; +DROP TABLE IF EXISTS census_duns16 CASCADE; +DROP TABLE IF EXISTS census_duns17 CASCADE; +DROP TABLE IF EXISTS census_duns18 CASCADE; +DROP TABLE IF EXISTS census_duns19 CASCADE; +DROP TABLE IF EXISTS census_duns20 CASCADE; +DROP TABLE IF EXISTS census_duns21 CASCADE; +DROP TABLE IF EXISTS census_duns22 CASCADE; +DROP TABLE IF EXISTS census_eins16 CASCADE; +DROP TABLE IF EXISTS census_eins17 CASCADE; +DROP TABLE IF EXISTS census_eins18 CASCADE; +DROP TABLE IF EXISTS census_eins19 CASCADE; +DROP TABLE IF EXISTS census_eins20 CASCADE; +DROP TABLE IF EXISTS census_eins21 CASCADE; +DROP TABLE IF EXISTS census_eins22 CASCADE; +DROP TABLE IF EXISTS census_findings16 CASCADE; +DROP TABLE IF EXISTS census_findings17 CASCADE; +DROP TABLE IF EXISTS census_findings18 CASCADE; +DROP TABLE IF EXISTS census_findings19 CASCADE; +DROP TABLE IF EXISTS census_findings20 CASCADE; +DROP TABLE IF EXISTS census_findings21 CASCADE; +DROP TABLE IF EXISTS census_findings22 CASCADE; +DROP TABLE IF EXISTS census_findingstext_formatted19 CASCADE; +DROP TABLE IF EXISTS census_findingstext_formatted20 CASCADE; +DROP TABLE IF EXISTS census_findingstext_formatted21 CASCADE; +DROP TABLE IF EXISTS census_findingstext_formatted22 CASCADE; +DROP TABLE IF EXISTS census_findingstext19 CASCADE; +DROP TABLE IF EXISTS census_findingstext20 CASCADE; +DROP TABLE IF EXISTS census_findingstext21 CASCADE; +DROP TABLE IF EXISTS census_findingstext22 CASCADE; +DROP TABLE IF EXISTS census_gen16 CASCADE; +DROP TABLE IF EXISTS census_gen17 CASCADE; +DROP TABLE IF EXISTS census_gen18 CASCADE; +DROP TABLE IF EXISTS census_gen19 CASCADE; +DROP TABLE IF EXISTS census_gen20 CASCADE; +DROP TABLE IF EXISTS census_gen21 CASCADE; +DROP TABLE IF EXISTS census_gen22 CASCADE; +DROP TABLE IF EXISTS census_notes19 CASCADE; +DROP TABLE IF EXISTS census_notes20 CASCADE; +DROP TABLE IF EXISTS census_notes21 CASCADE; +DROP TABLE IF EXISTS census_notes22 CASCADE; +DROP TABLE IF EXISTS census_passthrough16 CASCADE; +DROP TABLE IF EXISTS census_passthrough17 CASCADE; +DROP TABLE IF EXISTS census_passthrough18 CASCADE; +DROP TABLE IF EXISTS census_passthrough19 CASCADE; +DROP TABLE IF EXISTS census_passthrough20 CASCADE; +DROP TABLE IF EXISTS census_passthrough21 CASCADE; +DROP TABLE IF EXISTS census_passthrough22 CASCADE; +DROP TABLE IF EXISTS census_revisions19 CASCADE; +DROP TABLE IF EXISTS census_revisions20 CASCADE; +DROP TABLE IF EXISTS census_revisions21 CASCADE; +DROP TABLE IF EXISTS census_revisions22 CASCADE; +DROP TABLE IF EXISTS census_ueis22 CASCADE; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/010_copy_dissem_tables.sql b/backend/dissemination/sql/fac-snapshot-db/post/010_copy_dissem_tables.sql new file mode 100644 index 0000000000..6bc2e10cd0 --- /dev/null +++ b/backend/dissemination/sql/fac-snapshot-db/post/010_copy_dissem_tables.sql @@ -0,0 +1,86 @@ +--------------------------------------------- +-- COPY DISSEMINATION TABLES +-- WHY? Because our deploy process will want to make backups, +-- we will want to DROP and recreate those tables. If we +-- point our API at those tables (even in fac-snapshot-db), we +-- will have a problem at deploy-time. +-- +-- So, the fix is to take the copy that is in fac-snapshot-db (the +-- actual snapshot of the prod data) and make *one more copy*. We will +-- then point the API at that. We do this becasue we can (nightly) drop +-- the copy, tear down the API, make a fresh backup (and copy) and then +-- stand the API back up. +-- +-- It is a long way to go, but it kinda makes our entire data pipeline +-- "stateless," in a way. + +-- CREATE TABLE [Table to copy To] +-- AS [Table to copy From] + +CREATE OR REPLACE FUNCTION copy.create_copy_of_dissemination() + RETURNS VOID +AS +$ct$ +BEGIN + CREATE TABLE copy.dissemination_additionalein + AS TABLE public.dissemination_additionalein; + + CREATE TABLE copy.dissemination_additionaluei + AS TABLE public.dissemination_additionaluei; + + CREATE TABLE copy.dissemination_captext + AS TABLE public.dissemination_captext; + + CREATE TABLE copy.dissemination_federalaward + AS TABLE public.dissemination_federalaward; + + CREATE TABLE copy.dissemination_finding + AS TABLE public.dissemination_finding; + + CREATE TABLE copy.dissemination_findingtext + AS TABLE public.dissemination_findingtext; + + CREATE TABLE copy.dissemination_general + AS TABLE public.dissemination_general; + + CREATE TABLE copy.dissemination_invalidauditrecord + AS TABLE public.dissemination_invalidauditrecord; + + CREATE TABLE copy.dissemination_issuedescriptionrecord + AS TABLE public.dissemination_issuedescriptionrecord; + + CREATE TABLE copy.dissemination_migrationinspectionrecord + AS TABLE public.dissemination_migrationinspectionrecord; + + CREATE TABLE copy.dissemination_note + AS TABLE public.dissemination_note; + + CREATE TABLE copy.dissemination_passthrough + AS TABLE public.dissemination_passthrough; + + CREATE TABLE copy.dissemination_secondaryauditor + AS TABLE public.dissemination_secondaryauditor; +END +$ct$ +LANGUAGE plpgsql; + +DO LANGUAGE plpgsql +$GATE$ + DECLARE + the_schema varchar := 'copy'; + the_table varchar := 'dissemination_general'; + BEGIN + IF EXISTS ( + SELECT FROM pg_tables + WHERE schemaname = the_schema + AND tablename = the_table + ) + THEN + RAISE info 'We have a copy of dissem in fac-snapshot, so no need to make another.'; + ELSE + RAISE info 'Making a copy of dissemination_* in fac-snapshot.'; + PERFORM copy.create_copy_of_dissemination(); + END IF; + END +$GATE$; + diff --git a/backend/dissemination/sql/fac-snapshot-db/post/0144_api_v2_0_0_rpc.sql b/backend/dissemination/sql/fac-snapshot-db/post/0144_api_v2_0_0_rpc.sql index 5b7c925678..38e3a19621 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/0144_api_v2_0_0_rpc.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/0144_api_v2_0_0_rpc.sql @@ -11,19 +11,19 @@ BEGIN SELECT api_v2_0_0_functions.get_api_key_uuid() INTO v_uuid_header; - -- Check if the provided API key exists in public.dissemination_TribalApiAccessKeyIds + -- Check if the provided API key exists in copy.dissemination_TribalApiAccessKeyIds SELECT EXISTS( SELECT 1 - FROM public.dissemination_tribalapiaccesskeyids + FROM copy.dissemination_tribalapiaccesskeyids WHERE key_id = v_uuid_header ) INTO v_key_exists; - -- Get the added date of the key from public.dissemination_TribalApiAccessKeyIds + -- Get the added date of the key from copy.dissemination_TribalApiAccessKeyIds SELECT date_added INTO v_key_added_date - FROM public.dissemination_tribalapiaccesskeyids + FROM copy.dissemination_tribalapiaccesskeyids WHERE key_id = v_uuid_header; @@ -33,7 +33,7 @@ BEGIN SELECT gen_random_uuid() INTO v_access_uuid; -- Inserting data into the one_time_access table - INSERT INTO public.dissemination_onetimeaccess (uuid, api_key_id, timestamp, report_id) + INSERT INTO copy.dissemination_onetimeaccess (uuid, api_key_id, timestamp, report_id) VALUES (v_access_uuid::UUID, v_uuid_header, CURRENT_TIMESTAMP, report_id); -- Return the UUID to the user diff --git a/backend/dissemination/sql/fac-snapshot-db/post/020_public_data_v1_0_0_tables.sql b/backend/dissemination/sql/fac-snapshot-db/post/020_public_data_v1_0_0_tables.sql index 1ab1ae2c4c..717655ce29 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/020_public_data_v1_0_0_tables.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/020_public_data_v1_0_0_tables.sql @@ -95,7 +95,7 @@ BEGIN gen.total_amount_expended, gen.type_audit_code FROM - public.dissemination_general gen + copy.dissemination_general gen ORDER BY gen.id; -- Add a clean batch number after the table is created. @@ -126,7 +126,7 @@ BEGIN ein.additional_ein FROM public_data_v1_0_0.general gen, - public.dissemination_additionalein ein + copy.dissemination_additionalein ein WHERE gen.report_id = ein.report_id ORDER BY ein.id; @@ -159,7 +159,7 @@ BEGIN uei.additional_uei FROM public_data_v1_0_0.general gen, - public.dissemination_additionaluei uei + copy.dissemination_additionaluei uei WHERE gen.report_id = uei.report_id ORDER BY uei.id; @@ -195,7 +195,7 @@ BEGIN cap.planned_action FROM public_data_v1_0_0.general gen, - public.dissemination_captext cap + copy.dissemination_captext cap WHERE cap.report_id = gen.report_id AND @@ -250,7 +250,7 @@ BEGIN award.state_cluster_name FROM public_data_v1_0_0.general dg, - public.dissemination_federalaward award + copy.dissemination_federalaward award WHERE award.report_id = dg.report_id ORDER BY award.id; @@ -293,7 +293,7 @@ BEGIN finding.type_requirement FROM public_data_v1_0_0.general gen, - public.dissemination_finding finding + copy.dissemination_finding finding WHERE finding.report_id = gen.report_id ORDER BY finding.id; @@ -327,7 +327,7 @@ BEGIN ft.finding_text FROM public_data_v1_0_0.general gen, - public.dissemination_findingtext ft + copy.dissemination_findingtext ft WHERE ft.report_id = gen.report_id AND @@ -367,7 +367,7 @@ BEGIN note.rate_explained FROM public_data_v1_0_0.general gen, - public.dissemination_note note + copy.dissemination_note note WHERE note.report_id = gen.report_id AND @@ -405,7 +405,7 @@ BEGIN pass.passthrough_name FROM public_data_v1_0_0.general gen, - public.dissemination_passthrough pass + copy.dissemination_passthrough pass WHERE gen.report_id = pass.report_id ORDER BY pass.id; @@ -447,7 +447,7 @@ BEGIN sa.contact_title FROM public_data_v1_0_0.general gen, - public.dissemination_secondaryauditor sa + copy.dissemination_secondaryauditor sa WHERE sa.report_id = gen.report_id ORDER BY sa.id; @@ -582,13 +582,13 @@ BEGIN dp.passthrough_id, dp.passthrough_name FROM - public.dissemination_federalaward dfa + copy.dissemination_federalaward dfa LEFT JOIN public_data_v1_0_0.general dg ON dfa.report_id = dg.report_id - LEFT JOIN public.dissemination_finding df + LEFT JOIN copy.dissemination_finding df ON dfa.report_id = df.report_id AND dfa.award_reference = df.award_reference - LEFT JOIN public.dissemination_passthrough dp + LEFT JOIN copy.dissemination_passthrough dp ON dfa.report_id = dp.report_id AND dfa.award_reference = dp.award_reference ORDER BY seq diff --git a/backend/dissemination/sql/fac-snapshot-db/post/021_suppressed_data_v1_0_0_tables.sql b/backend/dissemination/sql/fac-snapshot-db/post/021_suppressed_data_v1_0_0_tables.sql index 389d13ef75..cfdbb6d9b0 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/021_suppressed_data_v1_0_0_tables.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/021_suppressed_data_v1_0_0_tables.sql @@ -19,7 +19,7 @@ BEGIN cap.planned_action FROM public_data_v1_0_0.general gen, - public.dissemination_captext cap + copy.dissemination_captext cap WHERE cap.report_id = gen.report_id AND @@ -50,7 +50,7 @@ BEGIN ft.finding_text FROM public_data_v1_0_0.general gen, - public.dissemination_findingtext ft + copy.dissemination_findingtext ft WHERE ft.report_id = gen.report_id AND @@ -85,7 +85,7 @@ BEGIN note.rate_explained FROM public_data_v1_0_0.general gen, - public.dissemination_note note + copy.dissemination_note note WHERE note.report_id = gen.report_id AND diff --git a/backend/dissemination/sql/fac-snapshot-db/post/122_api_v1_1_0_functions.sql b/backend/dissemination/sql/fac-snapshot-db/post/122_api_v1_1_0_functions.sql index 3abebc0b9b..62e648d496 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/122_api_v1_1_0_functions.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/122_api_v1_1_0_functions.sql @@ -41,7 +41,7 @@ BEGIN SELECT CASE WHEN EXISTS ( SELECT key_id - FROM public.dissemination_tribalapiaccesskeyids taaki + FROM copy.dissemination_tribalapiaccesskeyids taaki WHERE taaki.key_id = uuid_header::TEXT) THEN 1::BOOLEAN ELSE 0::BOOLEAN diff --git a/backend/dissemination/sql/fac-snapshot-db/post/124_api_v1_1_0_rpc.sql b/backend/dissemination/sql/fac-snapshot-db/post/124_api_v1_1_0_rpc.sql index 2e48d88efe..a5b749f6ce 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/124_api_v1_1_0_rpc.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/124_api_v1_1_0_rpc.sql @@ -11,19 +11,19 @@ BEGIN SELECT api_v1_1_0_functions.get_api_key_uuid() INTO v_uuid_header; - -- Check if the provided API key exists in public.dissemination_TribalApiAccessKeyIds + -- Check if the provided API key exists in copy.dissemination_TribalApiAccessKeyIds SELECT EXISTS( SELECT 1 - FROM public.dissemination_tribalapiaccesskeyids + FROM copy.dissemination_tribalapiaccesskeyids WHERE key_id = v_uuid_header ) INTO v_key_exists; - -- Get the added date of the key from public.dissemination_TribalApiAccessKeyIds + -- Get the added date of the key from copy.dissemination_TribalApiAccessKeyIds SELECT date_added INTO v_key_added_date - FROM public.dissemination_tribalapiaccesskeyids + FROM copy.dissemination_tribalapiaccesskeyids WHERE key_id = v_uuid_header; @@ -33,7 +33,7 @@ BEGIN SELECT gen_random_uuid() INTO v_access_uuid; -- Inserting data into the one_time_access table - INSERT INTO public.dissemination_onetimeaccess (uuid, api_key_id, timestamp, report_id) + INSERT INTO copy.dissemination_onetimeaccess (uuid, api_key_id, timestamp, report_id) VALUES (v_access_uuid::UUID, v_uuid_header, CURRENT_TIMESTAMP, report_id); -- Return the UUID to the user diff --git a/backend/dissemination/sql/fac-snapshot-db/post/142_api_v2_0_0_functions.sql b/backend/dissemination/sql/fac-snapshot-db/post/142_api_v2_0_0_functions.sql index 721472c050..a8ce2c47b5 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/142_api_v2_0_0_functions.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/142_api_v2_0_0_functions.sql @@ -41,7 +41,7 @@ BEGIN SELECT CASE WHEN EXISTS ( SELECT key_id - FROM public.dissemination_tribalapiaccesskeyids taaki + FROM copy.dissemination_tribalapiaccesskeyids taaki WHERE taaki.key_id = uuid_header::TEXT) THEN 1::BOOLEAN ELSE 0::BOOLEAN diff --git a/backend/dissemination/sql/fac-snapshot-db/pre/060_schemas.sql b/backend/dissemination/sql/fac-snapshot-db/pre/060_schemas.sql index ba12cb7ec4..5fee6d04f9 100644 --- a/backend/dissemination/sql/fac-snapshot-db/pre/060_schemas.sql +++ b/backend/dissemination/sql/fac-snapshot-db/pre/060_schemas.sql @@ -48,6 +48,8 @@ $GATE$; -- CREATE SCHEMA IF NOT EXISTS api_v1_0_3; -- CREATE SCHEMA IF NOT EXISTS api_v1_0_3_functions; +CREATE SCHEMA IF NOT EXISTS copy; + CREATE SCHEMA IF NOT EXISTS api_v1_1_0; CREATE SCHEMA IF NOT EXISTS api_v1_1_0_functions; @@ -61,4 +63,3 @@ CREATE SCHEMA IF NOT EXISTS suppressed_data_v1_0_0; CREATE SCHEMA IF NOT EXISTS api_v2_0_0; CREATE SCHEMA IF NOT EXISTS api_v2_0_0_functions; - diff --git a/backend/run.sh b/backend/run.sh index 5f9e65978a..005fcf5fab 100755 --- a/backend/run.sh +++ b/backend/run.sh @@ -9,6 +9,7 @@ source tools/util_startup.sh # This will choose the correct environment # for local envs (LOCAL or TESTING) and cloud.gov source tools/setup_env.sh +source tools/cgov_util_local_only.sh source tools/curation_audit_tracking_init.sh source tools/migrate_app_tables.sh source tools/seed_cog_baseline.sh @@ -22,6 +23,14 @@ source tools/util_startup.sh setup_env gonogo "setup_env" +##### +# SIMULATE DEPLOY BACKUP +# Before we deploy, we always get a copy of dissemination_* +# tables into fac-snapshot-db. We need to simulate this locally +# so that we can run SQL pre/post operations on fac-snapshot-db. +cgov_util_local_only +gonogo "cgov_util_local_only" + ##### # SQL PRE # We have SQL that we want to run before the migrations and sling are run. @@ -31,7 +40,7 @@ gonogo "sql_pre" ##### # MIGRATE APP TABLES -# migrate_app_tables +migrate_app_tables gonogo "migrate_app_tables" ##### diff --git a/backend/tools/cgov_util_local_only.sh b/backend/tools/cgov_util_local_only.sh new file mode 100644 index 0000000000..e340eb83ae --- /dev/null +++ b/backend/tools/cgov_util_local_only.sh @@ -0,0 +1,19 @@ +source tools/util_startup.sh + +function cgov_util_local_only() { + startup_log "CGOV_LOCAL_ONLY" "Making an initial 'backup'" + + check_table_exists $FAC_SNAPSHOT_URI 'public' 'dissemination_general' + local is_general_table=$FUNCTION_RESULT + if [ $is_general_table -ne 0 ]; then + # This is the first run. + startup_log "CGOV_LOCAL_ONLY" "Running cgov-util INITIAL." + $CGOV_UTIL_EXE db_to_db \ + --src_db fac-db \ + --dest_db fac-snapshot-db \ + --operation initial + fi + + startup_log "CGOV_LOCAL_ONLY" "Done" + return 0 +} diff --git a/backend/tools/setup_cgov_env.py b/backend/tools/setup_cgov_env.py new file mode 100644 index 0000000000..4e26cb3597 --- /dev/null +++ b/backend/tools/setup_cgov_env.py @@ -0,0 +1,80 @@ +import json +import os +from pprint import pprint + + +def get_vcap(): + # return json.loads(os.getenv("VCAP_SERVICES")) + obj = json.load(open("example_vcap.json")) + return obj + + +# eg "user-provided", "name", ""https-proxy-creds"" +def from_array(arr, key, value): + for o in arr: + if o[key] == value: + return o + raise ValueError("Nope") + + +def at_path(vcap, keys): + if isinstance(keys, str): + keys = [keys] + struct = vcap + for k in keys: + struct = struct[k] + return struct + + +def vcap_lookup(key): + vcap = get_vcap() + + +def setup_cgov_env(): + env = {} + env["SSL_CERT_FILE"] = "/etc/ssl/certs/ca-certificates.crt" + env["REQUESTS_CA_BUNDLE"] = "/etc/ssl/certs/ca-certificates.crt" + env["https_proxy"] = at_path( + from_array(at_path(get_vcap(), ["user-provided"]), "name", "https-proxy-creds"), + ["credentials", "uri"], + ) + env["smtp_proxy_domain"] = at_path( + from_array(at_path(get_vcap(), ["user-provided"]), "name", "smtp-proxy-creds"), + ["credentials", "domain"], + ) + env["smtp_proxy_port"] = at_path( + from_array(at_path(get_vcap(), ["user-provided"]), "name", "smtp-proxy-creds"), + ["credentials", "port"], + ) + + # S3_ENDPOINT_FOR_NO_PROXY="$(echo $VCAP_SERVICES | jq --raw-output --arg service_name "fac-public-s3" ".[][] | select(.name == \$service_name) | .credentials.endpoint")" + # S3_FIPS_ENDPOINT_FOR_NO_PROXY="$(echo $VCAP_SERVICES | jq --raw-output --arg service_name "fac-public-s3" ".[][] | select(.name == \$service_name) | .credentials.fips_endpoint")" + # S3_PRIVATE_ENDPOINT_FOR_NO_PROXY="$(echo $VCAP_SERVICES | jq --raw-output --arg service_name "fac-private-s3" ".[][] | select(.name == \$service_name) | .credentials.endpoint")" + # S3_PRIVATE_FIPS_ENDPOINT_FOR_NO_PROXY="$(echo $VCAP_SERVICES | jq --raw-output --arg service_name "fac-private-s3" ".[][] | select(.name == \$service_name) | .credentials.fips_endpoint")" + # export no_proxy="${S3_ENDPOINT_FOR_NO_PROXY},${S3_FIPS_ENDPOINT_FOR_NO_PROXY},${S3_PRIVATE_ENDPOINT_FOR_NO_PROXY},${S3_PRIVATE_FIPS_ENDPOINT_FOR_NO_PROXY},apps.internal" + + S3_ENDPOINT_FOR_NO_PROXY = at_path( + from_array(at_path(get_vcap(), ["s3"]), "name", "fac-public-s3"), + ["credentials", "endpoint"], + ) + S3_FIPS_ENDPOINT_FOR_NO_PROXY = at_path( + from_array(at_path(get_vcap(), ["s3"]), "name", "fac-public-s3"), + ["credentials", "fips_endpoint"], + ) + S3_PRIVATE_ENDPOINT_FOR_NO_PROXY = at_path( + from_array(at_path(get_vcap(), ["s3"]), "name", "fac-private-s3"), + ["credentials", "endpoint"], + ) + S3_PRIVATE_FIPS_ENDPOINT_FOR_NO_PROXY = at_path( + from_array(at_path(get_vcap(), ["s3"]), "name", "fac-private-s3"), + ["credentials", "fips_endpoint"], + ) + env["no_proxy"] = ( + f"{S3_ENDPOINT_FOR_NO_PROXY},{S3_FIPS_ENDPOINT_FOR_NO_PROXY},{S3_PRIVATE_ENDPOINT_FOR_NO_PROXY},{S3_PRIVATE_FIPS_ENDPOINT_FOR_NO_PROXY},apps.internal" + ) + + pprint(env) + + +if __name__ == "__main__": + setup_cgov_env() diff --git a/backend/tools/sql_pre_post.sh b/backend/tools/sql_pre_post.sh index 999dfb6a71..642aa7a73a 100644 --- a/backend/tools/sql_pre_post.sh +++ b/backend/tools/sql_pre_post.sh @@ -26,10 +26,12 @@ function run_sql_files { function sql_pre { + run_sql_files $FAC_DB_URI "SQL_PRE" "pre" run_sql_files $FAC_SNAPSHOT_URI "SQL_PRE" "pre" } function sql_post { + run_sql_files $FAC_DB_URI "SQL_POST" "post" run_sql_files $FAC_SNAPSHOT_URI "SQL_POST" "post" } From 8b5d43aed37aba90d0597ab8cf62ce601ff355c6 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Fri, 25 Oct 2024 07:51:18 -0400 Subject: [PATCH 38/89] Fixes Minio, copies data This introduces a new copy of the data, so that the tables we back up to every night do not become an issue. (That is, we want the API bound to the copy, not to the copies that get dumped every night.) --- backend/.gitignore | 1 + backend/config.json | 90 +++ .../post/010_remove_old_census_data.sql | 2 + .../post/010_copy_dissem_tables.sql | 213 ++++-- .../post/020_public_data_v1_0_0_tables.sql | 655 ----------------- .../post/020a_public_data_v1_0_0_general.sql | 129 ++++ .../post/020b_public_data_v1_0_0_tables.sql | 634 ++++++++++++++++ .../021_suppressed_data_v1_0_0_tables.sql | 258 ++++--- .../post/123_api_v1_1_0_views.sql | 38 +- .../sql/fac-snapshot-db/pre/060_schemas.sql | 16 +- .../sql/fac-snapshot-db/pre/090_sequences.sql | 70 +- .../bulk_data_export/create_stream_yaml.py | 51 ++ .../public_data_v1_0_0_single_csv.yaml | 684 ++++++++++++++++++ .../public_data_v1_0_0.yaml | 22 +- backend/tools/sling_bulk_export.sh | 12 + backend/tools/sling_first_run.SKIP | 49 ++ 16 files changed, 2067 insertions(+), 857 deletions(-) create mode 100644 backend/.gitignore create mode 100644 backend/config.json delete mode 100644 backend/dissemination/sql/fac-snapshot-db/post/020_public_data_v1_0_0_tables.sql create mode 100644 backend/dissemination/sql/fac-snapshot-db/post/020a_public_data_v1_0_0_general.sql create mode 100644 backend/dissemination/sql/fac-snapshot-db/post/020b_public_data_v1_0_0_tables.sql create mode 100644 backend/dissemination/sql/sling/bulk_data_export/create_stream_yaml.py create mode 100644 backend/dissemination/sql/sling/bulk_data_export/public_data_v1_0_0_single_csv.yaml create mode 100644 backend/tools/sling_bulk_export.sh create mode 100644 backend/tools/sling_first_run.SKIP diff --git a/backend/.gitignore b/backend/.gitignore new file mode 100644 index 0000000000..335ec9573d --- /dev/null +++ b/backend/.gitignore @@ -0,0 +1 @@ +*.tar.gz diff --git a/backend/config.json b/backend/config.json new file mode 100644 index 0000000000..2b1fc22521 --- /dev/null +++ b/backend/config.json @@ -0,0 +1,90 @@ +{ + "user-provided": [ + { + "label": "mc", + "name": "backups", + "tags": [], + "instance_guid": "UUIDALPHA1", + "instance_name": "backups", + "binding_guid": "UUIDALPHA2", + "binding_name": null, + "credentials": { + "access_key_id": "longtest", + "secret_access_key": "longtest", + "bucket": "backups", + "endpoint": "http://127.0.0.1:9001", + "admin_username": "minioadmin", + "admin_password": "minioadmin" + } + }, + { + "label": "mc", + "name": "fac-private-s3", + "tags": [], + "instance_guid": "UUIDALPHA1", + "instance_name": "fac-private-s3", + "binding_guid": "UUIDALPHA2", + "binding_name": null, + "credentials": { + "access_key_id": "longtest", + "secret_access_key": "longtest", + "bucket": "fac-private-s3", + "endpoint": "http://127.0.0.1:9001", + "admin_username": "minioadmin", + "admin_password": "minioadmin" + } + } + ], + "aws-rds": [ + { + "label": "fac-db", + "provider": null, + "plan": null, + "name": "fac-db", + "tags": [ + "database", + "docker" + ], + "instance_guid": "UUIDINDIA1", + "instance_name": "db", + "binding_guid": "UUIDINDIA2", + "binding_name": null, + "credentials": { + "db_name": "postgres", + "host": "db", + "name": "postgres", + "password": "", + "port": "5432", + "uri": "postgres://postgres@db:5432/postgres?sslmode=disable", + "username": "postgres" + }, + "syslog_drain_url": null, + "volume_mounts": [] + }, + { + "label": "fac-snapshot-db", + "provider": null, + "plan": null, + "name": "fac-snapshot-db", + "tags": [ + "database", + "docker" + ], + "instance_guid": "UUIDJULIET1", + "instance_name": "db", + "binding_guid": "UUIDJULIET2", + "binding_name": null, + "credentials": { + "db_name": "postgres", + "host": "db2", + "name": "postgres", + "password": "", + "port": "5432", + "uri": "postgres://postgres@db2:5432/postgres?sslmode=disable", + "username": "postgres" + }, + "syslog_drain_url": null, + "volume_mounts": [] + } + ] +} diff --git a/backend/dissemination/sql/fac-db/post/010_remove_old_census_data.sql b/backend/dissemination/sql/fac-db/post/010_remove_old_census_data.sql index 846541e7a0..8db01324df 100644 --- a/backend/dissemination/sql/fac-db/post/010_remove_old_census_data.sql +++ b/backend/dissemination/sql/fac-db/post/010_remove_old_census_data.sql @@ -5,6 +5,8 @@ -- our own data for this. We have removed this from the deploy, -- Specifically, it was used in cog/over. However, we now use -- This data was an early part of the data migration. +-- The cascade will get rid of any associated artifacts, which +-- we certainly do not want. -------------------------------------- -------------------------------------- DROP TABLE IF EXISTS census_agency16 CASCADE; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/010_copy_dissem_tables.sql b/backend/dissemination/sql/fac-snapshot-db/post/010_copy_dissem_tables.sql index 6bc2e10cd0..ae5e846050 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/010_copy_dissem_tables.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/010_copy_dissem_tables.sql @@ -17,70 +17,165 @@ -- CREATE TABLE [Table to copy To] -- AS [Table to copy From] -CREATE OR REPLACE FUNCTION copy.create_copy_of_dissemination() +CREATE OR REPLACE FUNCTION dissem_copy.create_dissemination_additionalein() RETURNS VOID -AS -$ct$ -BEGIN - CREATE TABLE copy.dissemination_additionalein - AS TABLE public.dissemination_additionalein; - - CREATE TABLE copy.dissemination_additionaluei - AS TABLE public.dissemination_additionaluei; - - CREATE TABLE copy.dissemination_captext - AS TABLE public.dissemination_captext; - - CREATE TABLE copy.dissemination_federalaward - AS TABLE public.dissemination_federalaward; - - CREATE TABLE copy.dissemination_finding - AS TABLE public.dissemination_finding; - - CREATE TABLE copy.dissemination_findingtext - AS TABLE public.dissemination_findingtext; - - CREATE TABLE copy.dissemination_general - AS TABLE public.dissemination_general; - - CREATE TABLE copy.dissemination_invalidauditrecord - AS TABLE public.dissemination_invalidauditrecord; - - CREATE TABLE copy.dissemination_issuedescriptionrecord - AS TABLE public.dissemination_issuedescriptionrecord; - - CREATE TABLE copy.dissemination_migrationinspectionrecord - AS TABLE public.dissemination_migrationinspectionrecord; - - CREATE TABLE copy.dissemination_note - AS TABLE public.dissemination_note; - - CREATE TABLE copy.dissemination_passthrough - AS TABLE public.dissemination_passthrough; - - CREATE TABLE copy.dissemination_secondaryauditor - AS TABLE public.dissemination_secondaryauditor; -END -$ct$ -LANGUAGE plpgsql; + AS + $ct$ + BEGIN + CREATE TABLE dissem_copy.dissemination_additionalein + AS SELECT * FROM public.dissemination_additionalein; + END + $ct$ + LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION dissem_copy.create_dissemination_additionaluei() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE dissem_copy.dissemination_additionaluei + AS SELECT * FROM public.dissemination_additionaluei; + END + $ct$ + LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION dissem_copy.create_dissemination_captext() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE dissem_copy.dissemination_captext + AS SELECT * FROM public.dissemination_captext; + END + $ct$ + LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION dissem_copy.create_dissemination_federalaward() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE dissem_copy.dissemination_federalaward + AS SELECT * FROM public.dissemination_federalaward; + END + $ct$ + LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION dissem_copy.create_dissemination_finding() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE dissem_copy.dissemination_finding + AS SELECT * FROM public.dissemination_finding; + END + $ct$ + LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION dissem_copy.create_dissemination_findingtext() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE dissem_copy.dissemination_findingtext + AS SELECT * FROM public.dissemination_findingtext; + END + $ct$ + LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION dissem_copy.create_dissemination_general() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE dissem_copy.dissemination_general + AS SELECT * FROM public.dissemination_general; + END + $ct$ + LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION dissem_copy.create_dissemination_invalidauditrecord() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE dissem_copy.dissemination_invalidauditrecord + AS SELECT * FROM public.dissemination_invalidauditrecord; + END + $ct$ + LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION dissem_copy.create_dissemination_migrationinspectionrecord() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE dissem_copy.dissemination_migrationinspectionrecord + AS SELECT * FROM public.dissemination_migrationinspectionrecord; + END + $ct$ + LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION dissem_copy.create_dissemination_note() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE dissem_copy.dissemination_note + AS SELECT * FROM public.dissemination_note; + END + $ct$ + LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION dissem_copy.create_dissemination_passthrough() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE dissem_copy.dissemination_passthrough + AS SELECT * FROM public.dissemination_passthrough; + END + $ct$ + LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION dissem_copy.create_dissemination_secondaryauditor() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE dissem_copy.dissemination_secondaryauditor + AS SELECT * FROM public.dissemination_secondaryauditor; + END + $ct$ + LANGUAGE plpgsql; DO LANGUAGE plpgsql $GATE$ - DECLARE - the_schema varchar := 'copy'; - the_table varchar := 'dissemination_general'; BEGIN - IF EXISTS ( - SELECT FROM pg_tables - WHERE schemaname = the_schema - AND tablename = the_table - ) - THEN - RAISE info 'We have a copy of dissem in fac-snapshot, so no need to make another.'; - ELSE - RAISE info 'Making a copy of dissemination_* in fac-snapshot.'; - PERFORM copy.create_copy_of_dissemination(); - END IF; + RAISE info 'create_dissemination_additionalein'; + PERFORM dissem_copy.create_dissemination_additionalein(); + RAISE info 'create_dissemination_additionaluei'; + PERFORM dissem_copy.create_dissemination_additionaluei(); + RAISE info 'create_dissemination_captext'; + PERFORM dissem_copy.create_dissemination_captext(); + RAISE info 'create_dissemination_federalaward'; + PERFORM dissem_copy.create_dissemination_federalaward(); + RAISE info 'create_dissemination_finding'; + PERFORM dissem_copy.create_dissemination_finding(); + RAISE info 'create_dissemination_findingtext'; + PERFORM dissem_copy.create_dissemination_findingtext(); + RAISE info 'create_dissemination_general'; + PERFORM dissem_copy.create_dissemination_general(); + RAISE info 'create_dissemination_invalidauditrecord'; + PERFORM dissem_copy.create_dissemination_invalidauditrecord(); + RAISE info 'create_dissemination_migrationinspectionrecord'; + PERFORM dissem_copy.create_dissemination_migrationinspectionrecord(); + RAISE info 'create_dissemination_note'; + PERFORM dissem_copy.create_dissemination_note(); + RAISE info 'create_dissemination_passthrough'; + PERFORM dissem_copy.create_dissemination_passthrough(); + RAISE info 'create_dissemination_secondaryauditor'; + PERFORM dissem_copy.create_dissemination_secondaryauditor(); END $GATE$; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/020_public_data_v1_0_0_tables.sql b/backend/dissemination/sql/fac-snapshot-db/post/020_public_data_v1_0_0_tables.sql deleted file mode 100644 index 717655ce29..0000000000 --- a/backend/dissemination/sql/fac-snapshot-db/post/020_public_data_v1_0_0_tables.sql +++ /dev/null @@ -1,655 +0,0 @@ ------------------------------------------------------------ --- general ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_general() - RETURNS VOID -AS -$ct$ -BEGIN - CREATE TABLE public_data_v1_0_0.general AS - SELECT - gen.id as id, - NEXTVAL('public_data_v1_0_0.seq_v1_0_0_general') AS seq, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.agencies_with_prior_findings, - gen.audit_period_covered, - gen.audit_type, - gen.auditee_address_line_1, - gen.auditee_certified_date, - gen.auditee_certify_name, - gen.auditee_certify_title, - gen.auditee_city, - gen.auditee_contact_name, - gen.auditee_contact_title, - gen.auditee_ein, - gen.auditee_email, - gen.auditee_name, - gen.auditee_phone, - gen.auditee_state, - gen.auditee_zip, - gen.auditor_address_line_1, - gen.auditor_certified_date, - gen.auditor_certify_name, - gen.auditor_certify_title, - gen.auditor_city, - gen.auditor_contact_name, - gen.auditor_contact_title, - gen.auditor_country, - gen.auditor_ein, - gen.auditor_email, - gen.auditor_firm_name, - gen.auditor_foreign_address, - gen.auditor_phone, - gen.auditor_state, - gen.auditor_zip, - gen.cognizant_agency, - gen.data_source, - gen.date_created, - gen.dollar_threshold, - gen.entity_type, - gen.fac_accepted_date, - gen.fy_end_date, - gen.fy_start_date, - gen.gaap_results, - gen.is_additional_ueis, - gen.is_aicpa_audit_guide_included, - gen.is_going_concern_included, - gen.is_internal_control_deficiency_disclosed, - gen.is_internal_control_material_weakness_disclosed, - gen.is_low_risk_auditee, - gen.is_material_noncompliance_disclosed, - CASE EXISTS - ( - SELECT - ein.report_id - FROM - dissemination_additionalein ein - WHERE - ein.report_id = gen.report_id - ) - WHEN FALSE THEN 'No' - ELSE 'Yes' - END AS is_multiple_eins, - gen.is_public, - CASE EXISTS - ( - SELECT - aud.report_id - FROM - dissemination_secondaryauditor aud - WHERE - aud.report_id = gen.report_id - ) - WHEN FALSE THEN 'No' - ELSE 'Yes' - END AS is_secondary_auditors, - gen.is_sp_framework_required, - gen.number_months, - gen.oversight_agency, - gen.ready_for_certification_date, - gen.sp_framework_basis, - gen.sp_framework_opinions, - gen.submitted_date, - gen.total_amount_expended, - gen.type_audit_code - FROM - copy.dissemination_general gen - ORDER BY gen.id; - - -- Add a clean batch number after the table is created. - ALTER TABLE public_data_v1_0_0.general - ADD COLUMN batch_number INTEGER; - UPDATE public_data_v1_0_0.general SET batch_number=DIV(seq, public.batch_size()); -END -$ct$ -LANGUAGE plpgsql; - ------------------------------------------------------------ --- addition_eins ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_additional_eins() - RETURNS VOID -AS -$ct$ -BEGIN - CREATE TABLE public_data_v1_0_0.additional_eins AS - SELECT - ein.id as id, - NEXTVAL('public_data_v1_0_0.seq_v1_0_0_additional_eins') AS seq, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - ein.additional_ein - FROM - public_data_v1_0_0.general gen, - copy.dissemination_additionalein ein - WHERE - gen.report_id = ein.report_id - ORDER BY ein.id; - - ALTER TABLE public_data_v1_0_0.additional_eins - ADD COLUMN batch_number INTEGER; - UPDATE public_data_v1_0_0.additional_eins SET batch_number=DIV(seq, public.batch_size()); -END -$ct$ -LANGUAGE plpgsql; - - ------------------------------------------------------------ --- additional_ueis ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_additional_ueis() - RETURNS VOID -AS -$ct$ -BEGIN - CREATE TABLE public_data_v1_0_0.additional_ueis AS - SELECT - uei.id as id, - NEXTVAL('public_data_v1_0_0.seq_v1_0_0_additional_ueis') AS seq, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - uei.additional_uei - FROM - public_data_v1_0_0.general gen, - copy.dissemination_additionaluei uei - WHERE - gen.report_id = uei.report_id - ORDER BY uei.id; - - -- Add a clean batch number after the table is created. - ALTER TABLE public_data_v1_0_0.additional_ueis - ADD COLUMN batch_number INTEGER; - UPDATE public_data_v1_0_0.additional_ueis SET batch_number=DIV(seq, public.batch_size()); -END -$ct$ -LANGUAGE plpgsql; - ------------------------------------------------------------ --- corrective_action_plans ------------------------------------------------------------ - -CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_corrective_action_plans() - RETURNS VOID -AS -$ct$ -BEGIN - CREATE TABLE public_data_v1_0_0.corrective_action_plans AS - SELECT - cap.id as id, - NEXTVAL('public_data_v1_0_0.seq_v1_0_0_corrective_action_plans') as seq, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - cap.contains_chart_or_table, - cap.finding_ref_number, - cap.planned_action - FROM - public_data_v1_0_0.general gen, - copy.dissemination_captext cap - WHERE - cap.report_id = gen.report_id - AND - -- Only include the public corrective action plans. - gen.is_public = true - ORDER BY cap.id; - - -- Add a clean batch number after the table is created. - ALTER TABLE public_data_v1_0_0.corrective_action_plans - ADD COLUMN batch_number INTEGER; - UPDATE public_data_v1_0_0.corrective_action_plans SET batch_number=DIV(seq, public.batch_size()); -END -$ct$ -LANGUAGE plpgsql; - ------------------------------------------------------------ --- federal_awards ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_federal_awards() - RETURNS VOID -AS -$ct$ -BEGIN - CREATE TABLE public_data_v1_0_0.federal_awards AS - SELECT - award.id as id, - NEXTVAL('public_data_v1_0_0.seq_v1_0_0_federal_awards') as seq, - dg.report_id, - dg.auditee_uei, - dg.audit_year, - dg.fac_accepted_date, - concat(award.federal_agency_prefix,'.',award.federal_award_extension) as aln, - --- - award.additional_award_identification, - award.amount_expended, - award.audit_report_type, - award.award_reference, - award.cluster_name, - award.cluster_total, - award.federal_agency_prefix, - award.federal_award_extension, - award.federal_program_name, - award.federal_program_total, - award.findings_count, - award.is_direct, - award.is_loan, - award.is_major, - award.is_passthrough_award, - award.loan_balance, - award.other_cluster_name, - award.passthrough_amount, - award.state_cluster_name - FROM - public_data_v1_0_0.general dg, - copy.dissemination_federalaward award - WHERE - award.report_id = dg.report_id - ORDER BY award.id; - - -- Add a clean batch number after the table is created. - ALTER TABLE public_data_v1_0_0.federal_awards - ADD COLUMN batch_number INTEGER; - UPDATE public_data_v1_0_0.federal_awards SET batch_number=DIV(seq, public.batch_size()); -END -$ct$ -LANGUAGE plpgsql; - ------------------------------------------------------------ --- findings ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_findings() - RETURNS VOID -AS -$ct$ -BEGIN - CREATE TABLE public_data_v1_0_0.findings AS - SELECT - finding.id as id, - NEXTVAL('public_data_v1_0_0.seq_v1_0_0_findings') as seq, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - finding.award_reference, - finding.is_material_weakness, - finding.is_modified_opinion, - finding.is_other_findings, - finding.is_other_matters, - finding.is_questioned_costs, - finding.is_repeat_finding, - finding.is_significant_deficiency, - finding.prior_finding_ref_numbers, - finding.reference_number, - finding.type_requirement - FROM - public_data_v1_0_0.general gen, - copy.dissemination_finding finding - WHERE - finding.report_id = gen.report_id - ORDER BY finding.id; - - -- Add a clean batch number after the table is created. - ALTER TABLE public_data_v1_0_0.findings - ADD COLUMN batch_number INTEGER; - UPDATE public_data_v1_0_0.findings SET batch_number=DIV(seq, public.batch_size()); -END -$ct$ -LANGUAGE plpgsql; - ------------------------------------------------------------ --- findings_text ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_findings_text() - RETURNS VOID -AS -$ct$ -BEGIN - CREATE TABLE public_data_v1_0_0.findings_text AS - SELECT - ft.id as id, - NEXTVAL('public_data_v1_0_0.seq_v1_0_0_findings_text') as seq, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - ft.finding_ref_number, - ft.contains_chart_or_table, - ft.finding_text - FROM - public_data_v1_0_0.general gen, - copy.dissemination_findingtext ft - WHERE - ft.report_id = gen.report_id - AND - gen.is_public = true - ORDER BY ft.id; - - -- Add a clean batch number after the table is created. - ALTER TABLE public_data_v1_0_0.findings_text - ADD COLUMN batch_number INTEGER; - UPDATE public_data_v1_0_0.findings_text SET batch_number=DIV(seq, public.batch_size()); -END -$ct$ -LANGUAGE plpgsql; - ------------------------------------------------------------ --- notes_to_sefa ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_notes_to_sefa() - RETURNS VOID -AS -$ct$ -BEGIN - CREATE TABLE public_data_v1_0_0.notes_to_sefa AS - SELECT - note.id as id, - NEXTVAL('public_data_v1_0_0.seq_v1_0_0_notes_to_sefa') as seq, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - note.accounting_policies, - note.contains_chart_or_table, - note.content, - note.is_minimis_rate_used, - note.note_title as title, - note.rate_explained - FROM - public_data_v1_0_0.general gen, - copy.dissemination_note note - WHERE - note.report_id = gen.report_id - AND - -- Some notes are not public. - gen.is_public = true - ORDER BY note.id; - - -- Add a clean batch number after the table is created. - ALTER TABLE public_data_v1_0_0.notes_to_sefa - ADD COLUMN batch_number INTEGER; - UPDATE public_data_v1_0_0.notes_to_sefa SET batch_number=DIV(seq, public.batch_size()); -END -$ct$ -LANGUAGE plpgsql; - ------------------------------------------------------------ --- passthrough ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_passthrough() - RETURNS VOID -AS -$ct$ -BEGIN - CREATE TABLE public_data_v1_0_0.passthrough AS - SELECT - pass.id as id, - NEXTVAL('public_data_v1_0_0.seq_v1_0_0_passthrough') as seq, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - pass.award_reference, - pass.passthrough_id, - pass.passthrough_name - FROM - public_data_v1_0_0.general gen, - copy.dissemination_passthrough pass - WHERE - gen.report_id = pass.report_id - ORDER BY pass.id; - - -- Add a clean batch number after the table is created. - ALTER TABLE public_data_v1_0_0.passthrough - ADD COLUMN batch_number INTEGER; - UPDATE public_data_v1_0_0.passthrough SET batch_number=DIV(seq, public.batch_size()); -END -$ct$ -LANGUAGE plpgsql; - ------------------------------------------------------------ --- secondary_auditors ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_secondary_auditors() - RETURNS VOID -AS -$ct$ -BEGIN - CREATE TABLE public_data_v1_0_0.secondary_auditors AS - SELECT - sa.id as id, - NEXTVAL('public_data_v1_0_0.seq_v1_0_0_secondary_auditors') as seq, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - sa.address_city, - sa.address_state, - sa.address_street, - sa.address_zipcode, - sa.auditor_ein, - sa.auditor_name, - sa.contact_email, - sa.contact_name, - sa.contact_phone, - sa.contact_title - FROM - public_data_v1_0_0.general gen, - copy.dissemination_secondaryauditor sa - WHERE - sa.report_id = gen.report_id - ORDER BY sa.id; - - -- Add a clean batch number after the table is created. - ALTER TABLE public_data_v1_0_0.secondary_auditors - ADD COLUMN batch_number INTEGER; - UPDATE public_data_v1_0_0.secondary_auditors SET batch_number=DIV(seq, public.batch_size()); -END -$ct$ -LANGUAGE plpgsql; - ------------------------------------------------------------ --- combined ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_combined() - RETURNS VOID -AS -$ct$ -BEGIN - CREATE TABLE public_data_v1_0_0.combined AS - SELECT - dg.report_id, - NEXTVAL('public_data_v1_0_0.seq_v1_0_0_combined') as seq, - dfa.award_reference, - df.reference_number, - concat(dfa.federal_agency_prefix,'.',dfa.federal_award_extension) as aln, - -- - -- general - -- - dg.id as general_row_id, - dg.auditee_uei, - dg.audit_year, - dg.agencies_with_prior_findings, - dg.audit_period_covered, - dg.audit_type, - dg.auditee_address_line_1, - dg.auditee_certified_date, - dg.auditee_certify_name, - dg.auditee_certify_title, - dg.auditee_city, - dg.auditee_contact_name, - dg.auditee_contact_title, - dg.auditee_ein, - dg.auditee_email, - dg.auditee_name, - dg.auditee_phone, - dg.auditee_state, - dg.auditee_zip, - dg.auditor_address_line_1, - dg.auditor_certified_date, - dg.auditor_certify_name, - dg.auditor_certify_title, - dg.auditor_city, - dg.auditor_contact_name, - dg.auditor_contact_title, - dg.auditor_country, - dg.auditor_ein, - dg.auditor_email, - dg.auditor_firm_name, - dg.auditor_foreign_address, - dg.auditor_phone, - dg.auditor_state, - dg.auditor_zip, - dg.cognizant_agency, - dg.data_source, - dg.date_created, - dg.dollar_threshold, - dg.entity_type, - dg.fac_accepted_date, - dg.fy_end_date, - dg.fy_start_date, - dg.gaap_results, - dg.is_additional_ueis, - dg.is_aicpa_audit_guide_included, - dg.is_going_concern_included, - dg.is_internal_control_deficiency_disclosed, - dg.is_internal_control_material_weakness_disclosed, - dg.is_low_risk_auditee, - dg.is_material_noncompliance_disclosed, - dg.is_multiple_eins, - dg.is_public, - dg.is_secondary_auditors, - dg.is_sp_framework_required, - dg.number_months, - dg.oversight_agency, - dg.ready_for_certification_date, - dg.sp_framework_basis, - dg.sp_framework_opinions, - dg.submitted_date, - dg.total_amount_expended, - dg.type_audit_code, - -- - -- federal_award - -- - dfa.id as federal_award_row_id, - dfa.additional_award_identification, - dfa.amount_expended, - dfa.audit_report_type, - dfa.cluster_name, - dfa.cluster_total, - dfa.federal_agency_prefix, - dfa.federal_award_extension, - dfa.federal_program_name, - dfa.federal_program_total, - dfa.findings_count, - dfa.is_direct, - dfa.is_loan, - dfa.is_major, - dfa.is_passthrough_award, - dfa.loan_balance, - dfa.other_cluster_name, - dfa.passthrough_amount, - dfa.state_cluster_name, - -- - -- finding - -- - df.id as finding_row_id, - df.is_material_weakness, - df.is_modified_opinion, - df.is_other_findings, - df.is_other_matters, - df.is_questioned_costs, - df.is_repeat_finding, - df.is_significant_deficiency, - df.prior_finding_ref_numbers, - df.type_requirement, - -- - -- passthrough - -- - dp.id as passthrough_row_id, - dp.passthrough_id, - dp.passthrough_name - FROM - copy.dissemination_federalaward dfa - LEFT JOIN public_data_v1_0_0.general dg - ON dfa.report_id = dg.report_id - LEFT JOIN copy.dissemination_finding df - ON dfa.report_id = df.report_id - AND dfa.award_reference = df.award_reference - LEFT JOIN copy.dissemination_passthrough dp - ON dfa.report_id = dp.report_id - AND dfa.award_reference = dp.award_reference - ORDER BY seq - -- FIXME Do not leave this limit in place. - LIMIT 1; - - -- Add a clean batch number after the table is created. - ALTER TABLE public_data_v1_0_0.combined - ADD COLUMN batch_number INTEGER; - UPDATE public_data_v1_0_0.combined SET batch_number=DIV(seq, public.batch_size()); -END -$ct$ -LANGUAGE plpgsql; - - ------------------------------------------------------------ --- CONDITIONAL TABLE CREATION --- We make this conditional at startup/on deploy. --- The reason is that every time we deploy, this would tear down --- the entire API, interrupting service. We only do that nightly, if we can. --- However, on a clean deploy or a first deploy to a 2-DB config, we will --- need this to run. ------------------------------------------------------------ -DO LANGUAGE plpgsql -$GATE$ - DECLARE - the_schema varchar := 'public_data_v1_0_0'; - the_table varchar := 'metadata'; - api_ver varchar := 'api_v2_0_0'; - BEGIN - IF EXISTS ( - SELECT FROM pg_tables - WHERE schemaname = the_schema - AND tablename = the_table - ) - THEN - RAISE info '% Gate condition met. Skipping table creation.', api_ver; - ELSE - RAISE info '% %.% not found. Creating tables', api_ver, the_schema, the_table; - RAISE info 'Creating general'; - PERFORM public_data_v1_0_0_functions.create_general(); - RAISE info 'Creating additional_eins'; - PERFORM public_data_v1_0_0_functions.create_additional_eins(); - RAISE info 'Creating additional_ueis'; - PERFORM public_data_v1_0_0_functions.create_additional_ueis(); - RAISE info 'Creating corrective_action_plans'; - PERFORM public_data_v1_0_0_functions.create_corrective_action_plans(); - RAISE info 'Creating federal_awards'; - PERFORM public_data_v1_0_0_functions.create_federal_awards(); - RAISE info 'Creating findings'; - PERFORM public_data_v1_0_0_functions.create_findings(); - RAISE info 'Creating findings_text'; - PERFORM public_data_v1_0_0_functions.create_findings_text(); - RAISE info 'Creating notes_to_sefa'; - PERFORM public_data_v1_0_0_functions.create_notes_to_sefa(); - RAISE info 'Creating passthrough'; - PERFORM public_data_v1_0_0_functions.create_passthrough(); - RAISE info 'Creating secondary_auditors'; - PERFORM public_data_v1_0_0_functions.create_secondary_auditors(); - RAISE info 'Creating combined'; - PERFORM public_data_v1_0_0_functions.create_combined(); - END IF; - END -$GATE$; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/020a_public_data_v1_0_0_general.sql b/backend/dissemination/sql/fac-snapshot-db/post/020a_public_data_v1_0_0_general.sql new file mode 100644 index 0000000000..1a9a5ebb23 --- /dev/null +++ b/backend/dissemination/sql/fac-snapshot-db/post/020a_public_data_v1_0_0_general.sql @@ -0,0 +1,129 @@ +----------------------------------------------------------- +-- general +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_general() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE public_data_v1_0_0.general AS + SELECT + gen.id as id, + NEXTVAL('public_data_v1_0_0.seq_general') AS seq, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.agencies_with_prior_findings, + gen.audit_period_covered, + gen.audit_type, + gen.auditee_address_line_1, + gen.auditee_certified_date, + gen.auditee_certify_name, + gen.auditee_certify_title, + gen.auditee_city, + gen.auditee_contact_name, + gen.auditee_contact_title, + gen.auditee_ein, + gen.auditee_email, + gen.auditee_name, + gen.auditee_phone, + gen.auditee_state, + gen.auditee_zip, + gen.auditor_address_line_1, + gen.auditor_certified_date, + gen.auditor_certify_name, + gen.auditor_certify_title, + gen.auditor_city, + gen.auditor_contact_name, + gen.auditor_contact_title, + gen.auditor_country, + gen.auditor_ein, + gen.auditor_email, + gen.auditor_firm_name, + gen.auditor_foreign_address, + gen.auditor_phone, + gen.auditor_state, + gen.auditor_zip, + gen.cognizant_agency, + gen.data_source, + gen.date_created, + gen.dollar_threshold, + gen.entity_type, + gen.fac_accepted_date, + gen.fy_end_date, + gen.fy_start_date, + gen.gaap_results, + gen.is_additional_ueis, + gen.is_aicpa_audit_guide_included, + gen.is_going_concern_included, + gen.is_internal_control_deficiency_disclosed, + gen.is_internal_control_material_weakness_disclosed, + gen.is_low_risk_auditee, + gen.is_material_noncompliance_disclosed, + CASE EXISTS + ( + SELECT + ein.report_id + FROM + dissemination_additionalein ein + WHERE + ein.report_id = gen.report_id + ) + WHEN FALSE THEN 'No' + ELSE 'Yes' + END AS is_multiple_eins, + gen.is_public, + CASE EXISTS + ( + SELECT + aud.report_id + FROM + dissemination_secondaryauditor aud + WHERE + aud.report_id = gen.report_id + ) + WHEN FALSE THEN 'No' + ELSE 'Yes' + END AS is_secondary_auditors, + gen.is_sp_framework_required, + gen.number_months, + gen.oversight_agency, + gen.ready_for_certification_date, + gen.sp_framework_basis, + gen.sp_framework_opinions, + gen.submitted_date, + gen.total_amount_expended, + gen.type_audit_code + FROM + dissem_copy.dissemination_general gen + ORDER BY gen.id; + + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.general + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.general SET batch_number=DIV(seq, public.batch_size()); + END + $ct$ + LANGUAGE plpgsql; + +DO LANGUAGE plpgsql +$GATE$ + DECLARE + the_schema varchar := 'public_data_v1_0_0'; + the_table varchar := 'metadata'; + api_ver varchar := 'api_v2_0_0'; + BEGIN + IF EXISTS ( + SELECT FROM pg_tables + WHERE schemaname = the_schema + AND tablename = the_table + ) + THEN + RAISE info '% Gate condition met. Skipping table creation.', api_ver; + ELSE + RAISE info '% %.% not found. Creating tables', api_ver, the_schema, the_table; + RAISE info 'Creating general'; + PERFORM public_data_v1_0_0_functions.create_general(); + END IF; + END +$GATE$; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/020b_public_data_v1_0_0_tables.sql b/backend/dissemination/sql/fac-snapshot-db/post/020b_public_data_v1_0_0_tables.sql new file mode 100644 index 0000000000..a4eafb3bc3 --- /dev/null +++ b/backend/dissemination/sql/fac-snapshot-db/post/020b_public_data_v1_0_0_tables.sql @@ -0,0 +1,634 @@ + +----------------------------------------------------------- +-- addition_eins +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_additional_eins() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE public_data_v1_0_0.additional_eins AS + SELECT + ein.id as id, + NEXTVAL('public_data_v1_0_0.seq_additional_eins') AS seq, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + ein.additional_ein + FROM + public_data_v1_0_0.general gen, + dissem_copy.dissemination_additionalein ein + WHERE + gen.report_id = ein.report_id + ORDER BY ein.id; + + ALTER TABLE public_data_v1_0_0.additional_eins + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.additional_eins SET batch_number=DIV(seq, public.batch_size()); + END + $ct$ + LANGUAGE plpgsql; + + +----------------------------------------------------------- +-- additional_ueis +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_additional_ueis() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE public_data_v1_0_0.additional_ueis AS + SELECT + uei.id as id, + NEXTVAL('public_data_v1_0_0.seq_additional_ueis') AS seq, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + uei.additional_uei + FROM + public_data_v1_0_0.general gen, + dissem_copy.dissemination_additionaluei uei + WHERE + gen.report_id = uei.report_id + ORDER BY uei.id; + + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.additional_ueis + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.additional_ueis SET batch_number=DIV(seq, public.batch_size()); + END + $ct$ + LANGUAGE plpgsql; + +----------------------------------------------------------- +-- corrective_action_plans +----------------------------------------------------------- + +CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_corrective_action_plans() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE public_data_v1_0_0.corrective_action_plans AS + SELECT + cap.id as id, + NEXTVAL('public_data_v1_0_0.seq_corrective_action_plans') as seq, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + cap.contains_chart_or_table, + cap.finding_ref_number, + cap.planned_action + FROM + public_data_v1_0_0.general gen, + dissem_copy.dissemination_captext cap + WHERE + cap.report_id = gen.report_id + AND + -- Only include the public corrective action plans. + gen.is_public = true + ORDER BY cap.id; + + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.corrective_action_plans + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.corrective_action_plans SET batch_number=DIV(seq, public.batch_size()); + END + $ct$ + LANGUAGE plpgsql; + +----------------------------------------------------------- +-- federal_awards +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_federal_awards() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE public_data_v1_0_0.federal_awards AS + SELECT + award.id as id, + NEXTVAL('public_data_v1_0_0.seq_federal_awards') as seq, + dg.report_id, + dg.auditee_uei, + dg.audit_year, + dg.fac_accepted_date, + concat(award.federal_agency_prefix,'.',award.federal_award_extension) as aln, + --- + award.additional_award_identification, + award.amount_expended, + award.audit_report_type, + award.award_reference, + award.cluster_name, + award.cluster_total, + award.federal_agency_prefix, + award.federal_award_extension, + award.federal_program_name, + award.federal_program_total, + award.findings_count, + award.is_direct, + award.is_loan, + award.is_major, + award.is_passthrough_award, + award.loan_balance, + award.other_cluster_name, + award.passthrough_amount, + award.state_cluster_name + FROM + public_data_v1_0_0.general dg, + dissem_copy.dissemination_federalaward award + WHERE + award.report_id = dg.report_id + ORDER BY award.id; + + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.federal_awards + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.federal_awards SET batch_number=DIV(seq, public.batch_size()); + END + $ct$ + LANGUAGE plpgsql; + +----------------------------------------------------------- +-- findings +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_findings() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE public_data_v1_0_0.findings AS + SELECT + finding.id as id, + NEXTVAL('public_data_v1_0_0.seq_findings') as seq, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + finding.award_reference, + finding.is_material_weakness, + finding.is_modified_opinion, + finding.is_other_findings, + finding.is_other_matters, + finding.is_questioned_costs, + finding.is_repeat_finding, + finding.is_significant_deficiency, + finding.prior_finding_ref_numbers, + finding.reference_number, + finding.type_requirement + FROM + public_data_v1_0_0.general gen, + dissem_copy.dissemination_finding finding + WHERE + finding.report_id = gen.report_id + ORDER BY finding.id; + + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.findings + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.findings SET batch_number=DIV(seq, public.batch_size()); + END + $ct$ + LANGUAGE plpgsql; + +----------------------------------------------------------- +-- findings_text +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_findings_text() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE public_data_v1_0_0.findings_text AS + SELECT + ft.id as id, + NEXTVAL('public_data_v1_0_0.seq_findings_text') as seq, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + ft.finding_ref_number, + ft.contains_chart_or_table, + ft.finding_text + FROM + public_data_v1_0_0.general gen, + dissem_copy.dissemination_findingtext ft + WHERE + ft.report_id = gen.report_id + AND + gen.is_public = true + ORDER BY ft.id; + + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.findings_text + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.findings_text SET batch_number=DIV(seq, public.batch_size()); + END + $ct$ + LANGUAGE plpgsql; + +----------------------------------------------------------- +-- notes_to_sefa +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_notes_to_sefa() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE public_data_v1_0_0.notes_to_sefa AS + SELECT + note.id as id, + NEXTVAL('public_data_v1_0_0.seq_notes_to_sefa') as seq, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + note.accounting_policies, + note.contains_chart_or_table, + note.content, + note.is_minimis_rate_used, + note.note_title as title, + note.rate_explained + FROM + public_data_v1_0_0.general gen, + dissem_copy.dissemination_note note + WHERE + note.report_id = gen.report_id + AND + -- Some notes are not public. + gen.is_public = true + ORDER BY note.id; + + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.notes_to_sefa + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.notes_to_sefa SET batch_number=DIV(seq, public.batch_size()); + END + $ct$ + LANGUAGE plpgsql; + +----------------------------------------------------------- +-- passthrough +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_passthrough() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE public_data_v1_0_0.passthrough AS + SELECT + pass.id as id, + NEXTVAL('public_data_v1_0_0.seq_passthrough') as seq, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + pass.award_reference, + pass.passthrough_id, + pass.passthrough_name + FROM + public_data_v1_0_0.general gen, + dissem_copy.dissemination_passthrough pass + WHERE + gen.report_id = pass.report_id + ORDER BY pass.id; + + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.passthrough + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.passthrough SET batch_number=DIV(seq, public.batch_size()); + END + $ct$ + LANGUAGE plpgsql; + +----------------------------------------------------------- +-- secondary_auditors +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_secondary_auditors() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE public_data_v1_0_0.secondary_auditors AS + SELECT + sa.id as id, + NEXTVAL('public_data_v1_0_0.seq_secondary_auditors') as seq, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + sa.address_city, + sa.address_state, + sa.address_street, + sa.address_zipcode, + sa.auditor_ein, + sa.auditor_name, + sa.contact_email, + sa.contact_name, + sa.contact_phone, + sa.contact_title + FROM + public_data_v1_0_0.general gen, + dissem_copy.dissemination_secondaryauditor sa + WHERE + sa.report_id = gen.report_id + ORDER BY sa.id; + + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.secondary_auditors + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.secondary_auditors SET batch_number=DIV(seq, public.batch_size()); + END + $ct$ + LANGUAGE plpgsql; + +----------------------------------------------------------- +-- combined +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_combined() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE public_data_v1_0_0.combined AS + SELECT + dg.report_id, + NEXTVAL('public_data_v1_0_0.seq_combined') as seq, + dfa.award_reference, + df.reference_number, + concat(dfa.federal_agency_prefix,'.',dfa.federal_award_extension) as aln, + -- + -- general + -- + dg.id as general_row_id, + dg.auditee_uei, + dg.audit_year, + dg.agencies_with_prior_findings, + dg.audit_period_covered, + dg.audit_type, + dg.auditee_address_line_1, + dg.auditee_certified_date, + dg.auditee_certify_name, + dg.auditee_certify_title, + dg.auditee_city, + dg.auditee_contact_name, + dg.auditee_contact_title, + dg.auditee_ein, + dg.auditee_email, + dg.auditee_name, + dg.auditee_phone, + dg.auditee_state, + dg.auditee_zip, + dg.auditor_address_line_1, + dg.auditor_certified_date, + dg.auditor_certify_name, + dg.auditor_certify_title, + dg.auditor_city, + dg.auditor_contact_name, + dg.auditor_contact_title, + dg.auditor_country, + dg.auditor_ein, + dg.auditor_email, + dg.auditor_firm_name, + dg.auditor_foreign_address, + dg.auditor_phone, + dg.auditor_state, + dg.auditor_zip, + dg.cognizant_agency, + dg.data_source, + dg.date_created, + dg.dollar_threshold, + dg.entity_type, + dg.fac_accepted_date, + dg.fy_end_date, + dg.fy_start_date, + dg.gaap_results, + dg.is_additional_ueis, + dg.is_aicpa_audit_guide_included, + dg.is_going_concern_included, + dg.is_internal_control_deficiency_disclosed, + dg.is_internal_control_material_weakness_disclosed, + dg.is_low_risk_auditee, + dg.is_material_noncompliance_disclosed, + dg.is_multiple_eins, + dg.is_public, + dg.is_secondary_auditors, + dg.is_sp_framework_required, + dg.number_months, + dg.oversight_agency, + dg.ready_for_certification_date, + dg.sp_framework_basis, + dg.sp_framework_opinions, + dg.submitted_date, + dg.total_amount_expended, + dg.type_audit_code, + -- + -- federal_award + -- + dfa.id as federal_award_row_id, + dfa.additional_award_identification, + dfa.amount_expended, + dfa.audit_report_type, + dfa.cluster_name, + dfa.cluster_total, + dfa.federal_agency_prefix, + dfa.federal_award_extension, + dfa.federal_program_name, + dfa.federal_program_total, + dfa.findings_count, + dfa.is_direct, + dfa.is_loan, + dfa.is_major, + dfa.is_passthrough_award, + dfa.loan_balance, + dfa.other_cluster_name, + dfa.passthrough_amount, + dfa.state_cluster_name, + -- + -- finding + -- + df.id as finding_row_id, + df.is_material_weakness, + df.is_modified_opinion, + df.is_other_findings, + df.is_other_matters, + df.is_questioned_costs, + df.is_repeat_finding, + df.is_significant_deficiency, + df.prior_finding_ref_numbers, + df.type_requirement, + -- + -- passthrough + -- + dp.id as passthrough_row_id, + dp.passthrough_id, + dp.passthrough_name + FROM + dissem_copy.dissemination_federalaward dfa + LEFT JOIN public_data_v1_0_0.general dg + ON dfa.report_id = dg.report_id + LEFT JOIN dissem_copy.dissemination_finding df + ON dfa.report_id = df.report_id + AND dfa.award_reference = df.award_reference + LEFT JOIN dissem_copy.dissemination_passthrough dp + ON dfa.report_id = dp.report_id + AND dfa.award_reference = dp.award_reference + ORDER BY seq + ; + + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.combined + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.combined SET batch_number=DIV(seq, public.batch_size()); + END + $ct$ + LANGUAGE plpgsql; + +----------------------------------------------------------- +-- migration_inspection_record +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_migration_inspection_record() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE public_data_v1_0_0.migration_inspection_record AS + SELECT + mir.id AS id, + NEXTVAL('public_data_v1_0_0.seq_migration_inspection_record') as seq, + mir.audit_year, + mir.dbkey, + mir.report_id, + mir.run_datetime, + mir.additional_ein, + mir.additional_uei, + mir.cap_text, + mir.federal_award, + mir.finding, + mir.finding_text, + mir.general, + mir.note, + mir.passthrough, + mir.secondary_auditor + FROM + public_data_v1_0_0.general gen, + dissem_copy.dissemination_migrationinspectionrecord mir + WHERE + mir.report_id = gen.report_id + AND + gen.is_public = true + ; + + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.migration_inspection_record + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.migration_inspection_record SET batch_number=DIV(seq, public.batch_size()); + END + $ct$ + LANGUAGE plpgsql; + +----------------------------------------------------------- +-- invalid_audit_record +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_invalid_audit_record() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE public_data_v1_0_0.invalid_audit_record AS + SELECT + iar.id AS id, + NEXTVAL('public_data_v1_0_0.seq_invalid_audit_record') as seq, + iar.audit_year, + iar.dbkey, + iar.report_id, + iar.run_datetime, + iar.additional_ein, + iar.additional_uei, + iar.cap_text, + iar.federal_award, + iar.finding, + iar.finding_text, + iar.general, + iar.note, + iar.passthrough, + iar.secondary_auditor + FROM + dissem_copy.dissemination_invalidauditrecord iar, + public_data_v1_0_0.general gen + WHERE + iar.report_id = gen.report_id + AND + gen.is_public = true + ; + + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.invalid_audit_record + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.invalid_audit_record SET batch_number=DIV(seq, public.batch_size()); + END + $ct$ + LANGUAGE plpgsql; + +----------------------------------------------------------- +-- CONDITIONAL TABLE CREATION +-- We make this conditional at startup/on deploy. +-- The reason is that every time we deploy, this would tear down +-- the entire API, interrupting service. We only do that nightly, if we can. +-- However, on a clean deploy or a first deploy to a 2-DB config, we will +-- need this to run. +----------------------------------------------------------- +DO LANGUAGE plpgsql +$GATE$ + DECLARE + the_schema varchar := 'public_data_v1_0_0'; + the_table varchar := 'metadata'; + api_ver varchar := 'api_v2_0_0'; + BEGIN + IF EXISTS ( + SELECT FROM pg_tables + WHERE schemaname = the_schema + AND tablename = the_table + ) + THEN + RAISE info '% Gate condition met. Skipping table creation.', api_ver; + ELSE + RAISE info '% %.% not found. Creating tables', api_ver, the_schema, the_table; + RAISE info 'Creating additional_eins'; + PERFORM public_data_v1_0_0_functions.create_additional_eins(); + RAISE info 'Creating additional_ueis'; + PERFORM public_data_v1_0_0_functions.create_additional_ueis(); + RAISE info 'Creating corrective_action_plans'; + PERFORM public_data_v1_0_0_functions.create_corrective_action_plans(); + RAISE info 'Creating federal_awards'; + PERFORM public_data_v1_0_0_functions.create_federal_awards(); + RAISE info 'Creating findings'; + PERFORM public_data_v1_0_0_functions.create_findings(); + RAISE info 'Creating findings_text'; + PERFORM public_data_v1_0_0_functions.create_findings_text(); + RAISE info 'Creating notes_to_sefa'; + PERFORM public_data_v1_0_0_functions.create_notes_to_sefa(); + RAISE info 'Creating passthrough'; + PERFORM public_data_v1_0_0_functions.create_passthrough(); + RAISE info 'Creating secondary_auditors'; + PERFORM public_data_v1_0_0_functions.create_secondary_auditors(); + RAISE info 'Creating combined'; + PERFORM public_data_v1_0_0_functions.create_combined(); + RAISE info 'Creating migration_inspection_record'; + PERFORM public_data_v1_0_0_functions.create_migration_inspection_record(); + RAISE info 'Create invalid_audit_record'; + PERFORM public_data_v1_0_0_functions.create_invalid_audit_record(); + END IF; + END +$GATE$; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/021_suppressed_data_v1_0_0_tables.sql b/backend/dissemination/sql/fac-snapshot-db/post/021_suppressed_data_v1_0_0_tables.sql index cfdbb6d9b0..39708881fd 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/021_suppressed_data_v1_0_0_tables.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/021_suppressed_data_v1_0_0_tables.sql @@ -1,108 +1,192 @@ ----------------------------------------------------------- --- general +-- corrective_action_plans ----------------------------------------------------------- CREATE OR REPLACE FUNCTION suppressed_data_v1_0_0.create_corrective_action_plans() RETURNS VOID -AS -$ct$ -BEGIN - CREATE TABLE suppressed_data_v1_0_0.corrective_action_plans AS - SELECT - cap.id as id, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - cap.contains_chart_or_table, - cap.finding_ref_number, - cap.planned_action - FROM - public_data_v1_0_0.general gen, - copy.dissemination_captext cap - WHERE - cap.report_id = gen.report_id - AND - -- Only include the suppressed corrective action plans. - gen.is_public = false - ORDER BY cap.id; -END -$ct$ -LANGUAGE plpgsql; + AS + $ct$ + BEGIN + CREATE TABLE suppressed_data_v1_0_0.corrective_action_plans AS + SELECT + cap.id as id, + NEXTVAL('suppressed_data_v1_0_0.seq_corrective_action_plans') as seq, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + cap.contains_chart_or_table, + cap.finding_ref_number, + cap.planned_action + FROM + public_data_v1_0_0.general gen, + dissem_copy.dissemination_captext cap + WHERE + cap.report_id = gen.report_id + AND + -- Only include the suppressed corrective action plans. + gen.is_public = false + ORDER BY cap.id; + END + $ct$ + LANGUAGE plpgsql; ----------------------------------------------------------- --- general +-- findings_text ----------------------------------------------------------- CREATE OR REPLACE FUNCTION suppressed_data_v1_0_0.create_findings_text() RETURNS VOID -AS -$ct$ -BEGIN - CREATE TABLE suppressed_data_v1_0_0.findings_text AS - SELECT - ft.id as id, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - ft.finding_ref_number, - ft.contains_chart_or_table, - ft.finding_text - FROM - public_data_v1_0_0.general gen, - copy.dissemination_findingtext ft - WHERE - ft.report_id = gen.report_id - AND - -- Findings text is not always public - gen.is_public = false - ORDER BY ft.id; -END -$ct$ -LANGUAGE plpgsql; + AS + $ct$ + BEGIN + CREATE TABLE suppressed_data_v1_0_0.findings_text AS + SELECT + ft.id as id, + NEXTVAL('suppressed_data_v1_0_0.seq_findings_text') as seq, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + ft.finding_ref_number, + ft.contains_chart_or_table, + ft.finding_text + FROM + public_data_v1_0_0.general gen, + dissem_copy.dissemination_findingtext ft + WHERE + ft.report_id = gen.report_id + AND + -- Findings text is not always public + gen.is_public = false + ORDER BY ft.id; + END + $ct$ + LANGUAGE plpgsql; ----------------------------------------------------------- --- general +-- notes_to_sefa ----------------------------------------------------------- CREATE OR REPLACE FUNCTION suppressed_data_v1_0_0.create_notes_to_sefa() RETURNS VOID -AS -$ct$ -BEGIN - CREATE TABLE suppressed_data_v1_0_0.notes_to_sefa AS - SELECT - note.id as id, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.fac_accepted_date, - --- - note.accounting_policies, - note.contains_chart_or_table, - note.content, - note.is_minimis_rate_used, - note.note_title as title, - note.rate_explained - FROM + AS + $ct$ + BEGIN + CREATE TABLE suppressed_data_v1_0_0.notes_to_sefa AS + SELECT + note.id as id, + NEXTVAL('suppressed_data_v1_0_0.seq_notes_to_sefa') as seq, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.fac_accepted_date, + --- + note.accounting_policies, + note.contains_chart_or_table, + note.content, + note.is_minimis_rate_used, + note.note_title as title, + note.rate_explained + FROM + public_data_v1_0_0.general gen, + dissem_copy.dissemination_note note + WHERE + note.report_id = gen.report_id + AND + -- Some notes are not public. + gen.is_public = false + ORDER BY note.id; + END + $ct$ + LANGUAGE plpgsql; + +----------------------------------------------------------- +-- migration_inspection_record +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION suppressed_data_v1_0_0.create_migration_inspection_record() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE suppressed_data_v1_0_0.migration_inspection_record AS + SELECT + mir.id AS id, + NEXTVAL('suppressed_data_v1_0_0.seq_migration_inspection_record') as seq, + mir.audit_year, + mir.dbkey, + mir.report_id, + mir.run_datetime, + mir.additional_ein, + mir.additional_uei, + mir.cap_text, + mir.federal_award, + mir.finding, + mir.finding_text, + mir.general, + mir.note, + mir.passthrough, + mir.secondary_auditor + FROM public_data_v1_0_0.general gen, - copy.dissemination_note note - WHERE - note.report_id = gen.report_id + dissem_copy.dissemination_migrationinspectionrecord mir + WHERE + mir.report_id = gen.report_id AND - -- Some notes are not public. gen.is_public = false - ORDER BY note.id; -END -$ct$ -LANGUAGE plpgsql; + ; + + -- Add a clean batch number after the table is created. + ALTER TABLE suppressed_data_v1_0_0.migration_inspection_record + ADD COLUMN batch_number INTEGER; + UPDATE suppressed_data_v1_0_0.migration_inspection_record SET batch_number=DIV(seq, public.batch_size()); + END + $ct$ + LANGUAGE plpgsql; + +----------------------------------------------------------- +-- invalid_audit_record +----------------------------------------------------------- +CREATE OR REPLACE FUNCTION suppressed_data_v1_0_0.create_invalid_audit_record() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE suppressed_data_v1_0_0.invalid_audit_record AS + SELECT + iar.id AS id, + NEXTVAL('suppressed_data_v1_0_0.seq_invalid_audit_record') as seq, + iar.audit_year, + iar.dbkey, + iar.report_id, + iar.run_datetime, + iar.additional_ein, + iar.additional_uei, + iar.cap_text, + iar.federal_award, + iar.finding, + iar.finding_text, + iar.general, + iar.note, + iar.passthrough, + iar.secondary_auditor + FROM + dissem_copy.dissemination_invalidauditrecord iar, + dissem_copy.dissemination_general gen + WHERE + iar.report_id = gen.report_id + AND + gen.is_public = true + ; + + -- Add a clean batch number after the table is created. + ALTER TABLE suppressed_data_v1_0_0.invalid_audit_record + ADD COLUMN batch_number INTEGER; + UPDATE suppressed_data_v1_0_0.invalid_audit_record SET batch_number=DIV(seq, public.batch_size()); + END + $ct$ + LANGUAGE plpgsql; ----------------------------------------------------------- -- CONDITIONAL TABLE CREATION --- We make this conditional at startup/on deploy. --- The reason is that every time we deploy, this would tear down --- the entire API, interrupting service. We only do that nightly, if we can. --- However, on a clean deploy or a first deploy to a 2-DB config, we will --- need this to run. ----------------------------------------------------------- DO LANGUAGE plpgsql $GATE$ @@ -125,6 +209,10 @@ $GATE$ PERFORM suppressed_data_v1_0_0.create_findings_text(); RAISE info 'Creating suppressed notes_to_sefa'; PERFORM suppressed_data_v1_0_0.create_notes_to_sefa(); + RAISE info 'Creating migration_inspection_record'; + PERFORM suppressed_data_v1_0_0.create_migration_inspection_record(); + RAISE info 'Create invalid_audit_record'; + PERFORM suppressed_data_v1_0_0.create_invalid_audit_record(); END IF; END $GATE$; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/123_api_v1_1_0_views.sql b/backend/dissemination/sql/fac-snapshot-db/post/123_api_v1_1_0_views.sql index f3d9f28dff..1565041224 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/123_api_v1_1_0_views.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/123_api_v1_1_0_views.sql @@ -9,8 +9,8 @@ create view api_v1_1_0.additional_eins as --- ein.additional_ein from - dissemination_general gen, - dissemination_additionalein ein + dissem_copy.dissemination_general gen, + dissem_copy.dissemination_additionalein ein where gen.report_id = ein.report_id order by ein.id @@ -27,8 +27,8 @@ create view api_v1_1_0.additional_ueis as --- uei.additional_uei from - dissemination_general gen, - dissemination_additionaluei uei + dissem_copy.dissemination_general gen, + dissem_copy.dissemination_additionaluei uei where gen.report_id = uei.report_id order by uei.id @@ -48,8 +48,8 @@ create view api_v1_1_0.corrective_action_plans as ct.contains_chart_or_table, ct.planned_action from - dissemination_CAPText ct, - dissemination_General gen + dissem_copy.dissemination_CAPText ct, + dissem_copy.dissemination_General gen where ct.report_id = gen.report_id and @@ -87,8 +87,8 @@ create view api_v1_1_0.federal_awards as award.is_passthrough_award, award.passthrough_amount from - dissemination_federalaward award, - dissemination_general gen + dissem_copy.dissemination_federalaward award, + dissem_copy.dissemination_general gen where award.report_id = gen.report_id order by award.id @@ -114,8 +114,8 @@ create view api_v1_1_0.findings as finding.is_significant_deficiency, finding.type_requirement from - dissemination_finding finding, - dissemination_general gen + dissem_copy.dissemination_finding finding, + dissem_copy.dissemination_general gen where finding.report_id = gen.report_id order by finding.id @@ -133,8 +133,8 @@ create view api_v1_1_0.findings_text as ft.contains_chart_or_table, ft.finding_text from - dissemination_findingtext ft, - dissemination_general gen + dissem_copy.dissemination_findingtext ft, + dissem_copy.dissemination_general gen where ft.report_id = gen.report_id and @@ -222,7 +222,7 @@ create view api_v1_1_0.general as ELSE 'Yes' END AS is_secondary_auditors from - dissemination_general gen + dissem_copy.dissemination_general gen order by gen.id ; @@ -242,8 +242,8 @@ create view api_v1_1_0.notes_to_sefa as note.content, note.contains_chart_or_table from - dissemination_general gen, - dissemination_note note + dissem_copy.dissemination_general gen, + dissem_copy.dissemination_note note where note.report_id = gen.report_id and @@ -265,8 +265,8 @@ create view api_v1_1_0.passthrough as pass.passthrough_id, pass.passthrough_name from - dissemination_general as gen, - dissemination_passthrough as pass + dissem_copy.dissemination_general as gen, + dissem_copy.dissemination_passthrough as pass where gen.report_id = pass.report_id order by pass.id @@ -292,8 +292,8 @@ create view api_v1_1_0.secondary_auditors as sa.address_state, sa.address_zipcode from - dissemination_General gen, - dissemination_SecondaryAuditor sa + dissem_copy.dissemination_General gen, + dissem_copy.dissemination_SecondaryAuditor sa where sa.report_id = gen.report_id order by sa.id diff --git a/backend/dissemination/sql/fac-snapshot-db/pre/060_schemas.sql b/backend/dissemination/sql/fac-snapshot-db/pre/060_schemas.sql index 5fee6d04f9..89baa719dd 100644 --- a/backend/dissemination/sql/fac-snapshot-db/pre/060_schemas.sql +++ b/backend/dissemination/sql/fac-snapshot-db/pre/060_schemas.sql @@ -14,6 +14,14 @@ DROP SCHEMA IF EXISTS admin_api_v1_1_1_functions CASCADE; DROP SCHEMA IF EXISTS api_v2_0_0 CASCADE; DROP SCHEMA IF EXISTS api_v2_0_0_functions CASCADE; +------------- +-- ALWAYS +-- This is the start of the pipeline. +-- It is a copy of the backed up dissemination tables in +-- fac-snapshot-db. Always drop the schema and make a new copy. +------------- +DROP SCHEMA IF EXISTS dissem_copy CASCADE; + ------------------------------------------------------------------ -- CONDITIONAL ------------------------------------------------------------------ @@ -48,17 +56,17 @@ $GATE$; -- CREATE SCHEMA IF NOT EXISTS api_v1_0_3; -- CREATE SCHEMA IF NOT EXISTS api_v1_0_3_functions; -CREATE SCHEMA IF NOT EXISTS copy; - CREATE SCHEMA IF NOT EXISTS api_v1_1_0; CREATE SCHEMA IF NOT EXISTS api_v1_1_0_functions; -CREATE SCHEMA IF NOT EXISTS admin_api_v1_1_0; -CREATE SCHEMA IF NOT EXISTS admin_api_v1_1_0_functions; +-- Retired 20241024 +-- CREATE SCHEMA IF NOT EXISTS admin_api_v1_1_0; +-- CREATE SCHEMA IF NOT EXISTS admin_api_v1_1_0_functions; CREATE SCHEMA IF NOT EXISTS public_data_v1_0_0; CREATE SCHEMA IF NOT EXISTS public_data_v1_0_0_functions; CREATE SCHEMA IF NOT EXISTS suppressed_data_v1_0_0; +CREATE SCHEMA IF NOT EXISTS dissem_copy; CREATE SCHEMA IF NOT EXISTS api_v2_0_0; CREATE SCHEMA IF NOT EXISTS api_v2_0_0_functions; diff --git a/backend/dissemination/sql/fac-snapshot-db/pre/090_sequences.sql b/backend/dissemination/sql/fac-snapshot-db/pre/090_sequences.sql index c2dbbd95e5..0a2b124878 100644 --- a/backend/dissemination/sql/fac-snapshot-db/pre/090_sequences.sql +++ b/backend/dissemination/sql/fac-snapshot-db/pre/090_sequences.sql @@ -1,25 +1,47 @@ --- We need these sequences for bringing the public tables back. --- That is, sling needs them. So, lets build them right now. -DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_additional_eins; -DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_additional_ueis; -DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_combined; -DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_corrective_action_plans; -DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_federal_awards; -DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_findings_text; -DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_findings; -DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_general; -DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_notes_to_sefa; -DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_passthrough; -DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_v1_0_0_secondary_auditors; +-- Sequences are used to provide a foundation for batching, which +-- is needed for fast download of data. +---------------------------------------------------------- +-- PUBLIC DATA TABLES +---------------------------------------------------------- +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_additional_eins; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_additional_ueis; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_combined; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_corrective_action_plans; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_federal_awards; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_findings_text; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_findings; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_general; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_notes_to_sefa; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_passthrough; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_secondary_auditors; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_migration_inspection_record; +DROP SEQUENCE IF EXISTS public_data_v1_0_0.seq_invalid_audit_record; -CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_additional_eins START 1; -CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_additional_ueis START 1; -CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_combined START 1; -CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_corrective_action_plans START 1; -CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_federal_awards START 1; -CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_findings_text START 1; -CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_findings START 1; -CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_general START 1; -CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_notes_to_sefa START 1; -CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_passthrough START 1; -CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_v1_0_0_secondary_auditors START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_additional_eins START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_additional_ueis START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_combined START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_corrective_action_plans START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_federal_awards START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_findings_text START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_findings START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_general START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_notes_to_sefa START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_passthrough START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_secondary_auditors START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_migration_inspection_record START 1; +CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_invalid_audit_record START 1; + +---------------------------------------------------------- +-- SUPPRESSED DATA TABLES +---------------------------------------------------------- +DROP SEQUENCE IF EXISTS suppressed_data_v1_0_0.seq_corrective_action_plans; +DROP SEQUENCE IF EXISTS suppressed_data_v1_0_0.seq_findings_text; +DROP SEQUENCE IF EXISTS suppressed_data_v1_0_0.seq_notes_to_sefa; +DROP SEQUENCE IF EXISTS suppressed_data_v1_0_0.seq_migration_inspection_record; +DROP SEQUENCE IF EXISTS suppressed_data_v1_0_0.seq_invalid_audit_record; + +CREATE SEQUENCE IF NOT EXISTS suppressed_data_v1_0_0.seq_corrective_action_plans START 1; +CREATE SEQUENCE IF NOT EXISTS suppressed_data_v1_0_0.seq_findings_text START 1; +CREATE SEQUENCE IF NOT EXISTS suppressed_data_v1_0_0.seq_notes_to_sefa START 1; +CREATE SEQUENCE IF NOT EXISTS suppressed_data_v1_0_0.seq_migration_inspection_record START 1; +CREATE SEQUENCE IF NOT EXISTS suppressed_data_v1_0_0.seq_invalid_audit_record START 1; diff --git a/backend/dissemination/sql/sling/bulk_data_export/create_stream_yaml.py b/backend/dissemination/sql/sling/bulk_data_export/create_stream_yaml.py new file mode 100644 index 0000000000..7e2c6d0a2b --- /dev/null +++ b/backend/dissemination/sql/sling/bulk_data_export/create_stream_yaml.py @@ -0,0 +1,51 @@ +# Run this as + +# python create_stream_yaml.py > public_data_v1_0_0_single_csv.yaml + +# to regenerate the sling file. + +import yaml +from pprint import pprint + +obj = {"streams": dict()} + +obj["source"] = "FAC_SNAPSHOT_URI" +obj["target"] = "BULK_DATA_EXPORT" +obj["defaults"] = { + "target_options": { + "format": "csv", + "compression": "gzip", + "file_max_rows": 0, + } +} + +schema = "public_data_v1_0_0" + +tables = [ + "additional_eins", + "additional_ueis", + "combined", + "corrective_action_plans", + "federal_awards", + "findings", + "findings_text", + "general", + "notes_to_sefa", + "passthrough", + "secondary_auditors", +] + +years = range(2016, 2031) + + +for t in tables: + ndx = 0 + for y in years: + obj["streams"][f"{schema}.{t}.{ndx}"] = { + "object": f"bulk_export/{{MM}}/{t}_{y}.csv", + "sql": f"SELECT * FROM {schema}.{t} WHERE audit_year = '{y}'", + "mode": "full-refresh", + } + ndx += 1 + +print(yaml.dump(obj)) diff --git a/backend/dissemination/sql/sling/bulk_data_export/public_data_v1_0_0_single_csv.yaml b/backend/dissemination/sql/sling/bulk_data_export/public_data_v1_0_0_single_csv.yaml new file mode 100644 index 0000000000..2e85bfc856 --- /dev/null +++ b/backend/dissemination/sql/sling/bulk_data_export/public_data_v1_0_0_single_csv.yaml @@ -0,0 +1,684 @@ +defaults: + target_options: + compression: gzip + file_max_rows: 0 + format: csv +source: FAC_SNAPSHOT_URI +streams: + public_data_v1_0_0.additional_eins.0: + mode: full-refresh + object: bulk_export/{MM}/additional_eins_2016.csv + sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2016' + public_data_v1_0_0.additional_eins.1: + mode: full-refresh + object: bulk_export/{MM}/additional_eins_2017.csv + sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2017' + public_data_v1_0_0.additional_eins.10: + mode: full-refresh + object: bulk_export/{MM}/additional_eins_2026.csv + sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2026' + public_data_v1_0_0.additional_eins.11: + mode: full-refresh + object: bulk_export/{MM}/additional_eins_2027.csv + sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2027' + public_data_v1_0_0.additional_eins.12: + mode: full-refresh + object: bulk_export/{MM}/additional_eins_2028.csv + sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2028' + public_data_v1_0_0.additional_eins.13: + mode: full-refresh + object: bulk_export/{MM}/additional_eins_2029.csv + sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2029' + public_data_v1_0_0.additional_eins.14: + mode: full-refresh + object: bulk_export/{MM}/additional_eins_2030.csv + sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2030' + public_data_v1_0_0.additional_eins.2: + mode: full-refresh + object: bulk_export/{MM}/additional_eins_2018.csv + sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2018' + public_data_v1_0_0.additional_eins.3: + mode: full-refresh + object: bulk_export/{MM}/additional_eins_2019.csv + sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2019' + public_data_v1_0_0.additional_eins.4: + mode: full-refresh + object: bulk_export/{MM}/additional_eins_2020.csv + sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2020' + public_data_v1_0_0.additional_eins.5: + mode: full-refresh + object: bulk_export/{MM}/additional_eins_2021.csv + sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2021' + public_data_v1_0_0.additional_eins.6: + mode: full-refresh + object: bulk_export/{MM}/additional_eins_2022.csv + sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2022' + public_data_v1_0_0.additional_eins.7: + mode: full-refresh + object: bulk_export/{MM}/additional_eins_2023.csv + sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2023' + public_data_v1_0_0.additional_eins.8: + mode: full-refresh + object: bulk_export/{MM}/additional_eins_2024.csv + sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2024' + public_data_v1_0_0.additional_eins.9: + mode: full-refresh + object: bulk_export/{MM}/additional_eins_2025.csv + sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2025' + public_data_v1_0_0.additional_ueis.0: + mode: full-refresh + object: bulk_export/{MM}/additional_ueis_2016.csv + sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2016' + public_data_v1_0_0.additional_ueis.1: + mode: full-refresh + object: bulk_export/{MM}/additional_ueis_2017.csv + sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2017' + public_data_v1_0_0.additional_ueis.10: + mode: full-refresh + object: bulk_export/{MM}/additional_ueis_2026.csv + sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2026' + public_data_v1_0_0.additional_ueis.11: + mode: full-refresh + object: bulk_export/{MM}/additional_ueis_2027.csv + sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2027' + public_data_v1_0_0.additional_ueis.12: + mode: full-refresh + object: bulk_export/{MM}/additional_ueis_2028.csv + sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2028' + public_data_v1_0_0.additional_ueis.13: + mode: full-refresh + object: bulk_export/{MM}/additional_ueis_2029.csv + sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2029' + public_data_v1_0_0.additional_ueis.14: + mode: full-refresh + object: bulk_export/{MM}/additional_ueis_2030.csv + sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2030' + public_data_v1_0_0.additional_ueis.2: + mode: full-refresh + object: bulk_export/{MM}/additional_ueis_2018.csv + sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2018' + public_data_v1_0_0.additional_ueis.3: + mode: full-refresh + object: bulk_export/{MM}/additional_ueis_2019.csv + sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2019' + public_data_v1_0_0.additional_ueis.4: + mode: full-refresh + object: bulk_export/{MM}/additional_ueis_2020.csv + sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2020' + public_data_v1_0_0.additional_ueis.5: + mode: full-refresh + object: bulk_export/{MM}/additional_ueis_2021.csv + sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2021' + public_data_v1_0_0.additional_ueis.6: + mode: full-refresh + object: bulk_export/{MM}/additional_ueis_2022.csv + sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2022' + public_data_v1_0_0.additional_ueis.7: + mode: full-refresh + object: bulk_export/{MM}/additional_ueis_2023.csv + sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2023' + public_data_v1_0_0.additional_ueis.8: + mode: full-refresh + object: bulk_export/{MM}/additional_ueis_2024.csv + sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2024' + public_data_v1_0_0.additional_ueis.9: + mode: full-refresh + object: bulk_export/{MM}/additional_ueis_2025.csv + sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2025' + public_data_v1_0_0.combined.0: + mode: full-refresh + object: bulk_export/{MM}/combined_2016.csv + sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2016' + public_data_v1_0_0.combined.1: + mode: full-refresh + object: bulk_export/{MM}/combined_2017.csv + sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2017' + public_data_v1_0_0.combined.10: + mode: full-refresh + object: bulk_export/{MM}/combined_2026.csv + sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2026' + public_data_v1_0_0.combined.11: + mode: full-refresh + object: bulk_export/{MM}/combined_2027.csv + sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2027' + public_data_v1_0_0.combined.12: + mode: full-refresh + object: bulk_export/{MM}/combined_2028.csv + sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2028' + public_data_v1_0_0.combined.13: + mode: full-refresh + object: bulk_export/{MM}/combined_2029.csv + sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2029' + public_data_v1_0_0.combined.14: + mode: full-refresh + object: bulk_export/{MM}/combined_2030.csv + sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2030' + public_data_v1_0_0.combined.2: + mode: full-refresh + object: bulk_export/{MM}/combined_2018.csv + sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2018' + public_data_v1_0_0.combined.3: + mode: full-refresh + object: bulk_export/{MM}/combined_2019.csv + sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2019' + public_data_v1_0_0.combined.4: + mode: full-refresh + object: bulk_export/{MM}/combined_2020.csv + sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2020' + public_data_v1_0_0.combined.5: + mode: full-refresh + object: bulk_export/{MM}/combined_2021.csv + sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2021' + public_data_v1_0_0.combined.6: + mode: full-refresh + object: bulk_export/{MM}/combined_2022.csv + sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2022' + public_data_v1_0_0.combined.7: + mode: full-refresh + object: bulk_export/{MM}/combined_2023.csv + sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2023' + public_data_v1_0_0.combined.8: + mode: full-refresh + object: bulk_export/{MM}/combined_2024.csv + sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2024' + public_data_v1_0_0.combined.9: + mode: full-refresh + object: bulk_export/{MM}/combined_2025.csv + sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2025' + public_data_v1_0_0.corrective_action_plans.0: + mode: full-refresh + object: bulk_export/{MM}/corrective_action_plans_2016.csv + sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year + = '2016' + public_data_v1_0_0.corrective_action_plans.1: + mode: full-refresh + object: bulk_export/{MM}/corrective_action_plans_2017.csv + sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year + = '2017' + public_data_v1_0_0.corrective_action_plans.10: + mode: full-refresh + object: bulk_export/{MM}/corrective_action_plans_2026.csv + sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year + = '2026' + public_data_v1_0_0.corrective_action_plans.11: + mode: full-refresh + object: bulk_export/{MM}/corrective_action_plans_2027.csv + sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year + = '2027' + public_data_v1_0_0.corrective_action_plans.12: + mode: full-refresh + object: bulk_export/{MM}/corrective_action_plans_2028.csv + sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year + = '2028' + public_data_v1_0_0.corrective_action_plans.13: + mode: full-refresh + object: bulk_export/{MM}/corrective_action_plans_2029.csv + sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year + = '2029' + public_data_v1_0_0.corrective_action_plans.14: + mode: full-refresh + object: bulk_export/{MM}/corrective_action_plans_2030.csv + sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year + = '2030' + public_data_v1_0_0.corrective_action_plans.2: + mode: full-refresh + object: bulk_export/{MM}/corrective_action_plans_2018.csv + sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year + = '2018' + public_data_v1_0_0.corrective_action_plans.3: + mode: full-refresh + object: bulk_export/{MM}/corrective_action_plans_2019.csv + sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year + = '2019' + public_data_v1_0_0.corrective_action_plans.4: + mode: full-refresh + object: bulk_export/{MM}/corrective_action_plans_2020.csv + sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year + = '2020' + public_data_v1_0_0.corrective_action_plans.5: + mode: full-refresh + object: bulk_export/{MM}/corrective_action_plans_2021.csv + sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year + = '2021' + public_data_v1_0_0.corrective_action_plans.6: + mode: full-refresh + object: bulk_export/{MM}/corrective_action_plans_2022.csv + sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year + = '2022' + public_data_v1_0_0.corrective_action_plans.7: + mode: full-refresh + object: bulk_export/{MM}/corrective_action_plans_2023.csv + sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year + = '2023' + public_data_v1_0_0.corrective_action_plans.8: + mode: full-refresh + object: bulk_export/{MM}/corrective_action_plans_2024.csv + sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year + = '2024' + public_data_v1_0_0.corrective_action_plans.9: + mode: full-refresh + object: bulk_export/{MM}/corrective_action_plans_2025.csv + sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year + = '2025' + public_data_v1_0_0.federal_awards.0: + mode: full-refresh + object: bulk_export/{MM}/federal_awards_2016.csv + sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2016' + public_data_v1_0_0.federal_awards.1: + mode: full-refresh + object: bulk_export/{MM}/federal_awards_2017.csv + sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2017' + public_data_v1_0_0.federal_awards.10: + mode: full-refresh + object: bulk_export/{MM}/federal_awards_2026.csv + sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2026' + public_data_v1_0_0.federal_awards.11: + mode: full-refresh + object: bulk_export/{MM}/federal_awards_2027.csv + sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2027' + public_data_v1_0_0.federal_awards.12: + mode: full-refresh + object: bulk_export/{MM}/federal_awards_2028.csv + sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2028' + public_data_v1_0_0.federal_awards.13: + mode: full-refresh + object: bulk_export/{MM}/federal_awards_2029.csv + sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2029' + public_data_v1_0_0.federal_awards.14: + mode: full-refresh + object: bulk_export/{MM}/federal_awards_2030.csv + sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2030' + public_data_v1_0_0.federal_awards.2: + mode: full-refresh + object: bulk_export/{MM}/federal_awards_2018.csv + sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2018' + public_data_v1_0_0.federal_awards.3: + mode: full-refresh + object: bulk_export/{MM}/federal_awards_2019.csv + sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2019' + public_data_v1_0_0.federal_awards.4: + mode: full-refresh + object: bulk_export/{MM}/federal_awards_2020.csv + sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2020' + public_data_v1_0_0.federal_awards.5: + mode: full-refresh + object: bulk_export/{MM}/federal_awards_2021.csv + sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2021' + public_data_v1_0_0.federal_awards.6: + mode: full-refresh + object: bulk_export/{MM}/federal_awards_2022.csv + sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2022' + public_data_v1_0_0.federal_awards.7: + mode: full-refresh + object: bulk_export/{MM}/federal_awards_2023.csv + sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2023' + public_data_v1_0_0.federal_awards.8: + mode: full-refresh + object: bulk_export/{MM}/federal_awards_2024.csv + sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2024' + public_data_v1_0_0.federal_awards.9: + mode: full-refresh + object: bulk_export/{MM}/federal_awards_2025.csv + sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2025' + public_data_v1_0_0.findings.0: + mode: full-refresh + object: bulk_export/{MM}/findings_2016.csv + sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2016' + public_data_v1_0_0.findings.1: + mode: full-refresh + object: bulk_export/{MM}/findings_2017.csv + sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2017' + public_data_v1_0_0.findings.10: + mode: full-refresh + object: bulk_export/{MM}/findings_2026.csv + sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2026' + public_data_v1_0_0.findings.11: + mode: full-refresh + object: bulk_export/{MM}/findings_2027.csv + sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2027' + public_data_v1_0_0.findings.12: + mode: full-refresh + object: bulk_export/{MM}/findings_2028.csv + sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2028' + public_data_v1_0_0.findings.13: + mode: full-refresh + object: bulk_export/{MM}/findings_2029.csv + sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2029' + public_data_v1_0_0.findings.14: + mode: full-refresh + object: bulk_export/{MM}/findings_2030.csv + sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2030' + public_data_v1_0_0.findings.2: + mode: full-refresh + object: bulk_export/{MM}/findings_2018.csv + sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2018' + public_data_v1_0_0.findings.3: + mode: full-refresh + object: bulk_export/{MM}/findings_2019.csv + sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2019' + public_data_v1_0_0.findings.4: + mode: full-refresh + object: bulk_export/{MM}/findings_2020.csv + sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2020' + public_data_v1_0_0.findings.5: + mode: full-refresh + object: bulk_export/{MM}/findings_2021.csv + sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2021' + public_data_v1_0_0.findings.6: + mode: full-refresh + object: bulk_export/{MM}/findings_2022.csv + sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2022' + public_data_v1_0_0.findings.7: + mode: full-refresh + object: bulk_export/{MM}/findings_2023.csv + sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2023' + public_data_v1_0_0.findings.8: + mode: full-refresh + object: bulk_export/{MM}/findings_2024.csv + sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2024' + public_data_v1_0_0.findings.9: + mode: full-refresh + object: bulk_export/{MM}/findings_2025.csv + sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2025' + public_data_v1_0_0.findings_text.0: + mode: full-refresh + object: bulk_export/{MM}/findings_text_2016.csv + sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2016' + public_data_v1_0_0.findings_text.1: + mode: full-refresh + object: bulk_export/{MM}/findings_text_2017.csv + sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2017' + public_data_v1_0_0.findings_text.10: + mode: full-refresh + object: bulk_export/{MM}/findings_text_2026.csv + sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2026' + public_data_v1_0_0.findings_text.11: + mode: full-refresh + object: bulk_export/{MM}/findings_text_2027.csv + sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2027' + public_data_v1_0_0.findings_text.12: + mode: full-refresh + object: bulk_export/{MM}/findings_text_2028.csv + sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2028' + public_data_v1_0_0.findings_text.13: + mode: full-refresh + object: bulk_export/{MM}/findings_text_2029.csv + sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2029' + public_data_v1_0_0.findings_text.14: + mode: full-refresh + object: bulk_export/{MM}/findings_text_2030.csv + sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2030' + public_data_v1_0_0.findings_text.2: + mode: full-refresh + object: bulk_export/{MM}/findings_text_2018.csv + sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2018' + public_data_v1_0_0.findings_text.3: + mode: full-refresh + object: bulk_export/{MM}/findings_text_2019.csv + sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2019' + public_data_v1_0_0.findings_text.4: + mode: full-refresh + object: bulk_export/{MM}/findings_text_2020.csv + sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2020' + public_data_v1_0_0.findings_text.5: + mode: full-refresh + object: bulk_export/{MM}/findings_text_2021.csv + sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2021' + public_data_v1_0_0.findings_text.6: + mode: full-refresh + object: bulk_export/{MM}/findings_text_2022.csv + sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2022' + public_data_v1_0_0.findings_text.7: + mode: full-refresh + object: bulk_export/{MM}/findings_text_2023.csv + sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2023' + public_data_v1_0_0.findings_text.8: + mode: full-refresh + object: bulk_export/{MM}/findings_text_2024.csv + sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2024' + public_data_v1_0_0.findings_text.9: + mode: full-refresh + object: bulk_export/{MM}/findings_text_2025.csv + sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2025' + public_data_v1_0_0.general.0: + mode: full-refresh + object: bulk_export/{MM}/general_2016.csv + sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2016' + public_data_v1_0_0.general.1: + mode: full-refresh + object: bulk_export/{MM}/general_2017.csv + sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2017' + public_data_v1_0_0.general.10: + mode: full-refresh + object: bulk_export/{MM}/general_2026.csv + sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2026' + public_data_v1_0_0.general.11: + mode: full-refresh + object: bulk_export/{MM}/general_2027.csv + sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2027' + public_data_v1_0_0.general.12: + mode: full-refresh + object: bulk_export/{MM}/general_2028.csv + sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2028' + public_data_v1_0_0.general.13: + mode: full-refresh + object: bulk_export/{MM}/general_2029.csv + sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2029' + public_data_v1_0_0.general.14: + mode: full-refresh + object: bulk_export/{MM}/general_2030.csv + sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2030' + public_data_v1_0_0.general.2: + mode: full-refresh + object: bulk_export/{MM}/general_2018.csv + sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2018' + public_data_v1_0_0.general.3: + mode: full-refresh + object: bulk_export/{MM}/general_2019.csv + sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2019' + public_data_v1_0_0.general.4: + mode: full-refresh + object: bulk_export/{MM}/general_2020.csv + sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2020' + public_data_v1_0_0.general.5: + mode: full-refresh + object: bulk_export/{MM}/general_2021.csv + sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2021' + public_data_v1_0_0.general.6: + mode: full-refresh + object: bulk_export/{MM}/general_2022.csv + sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2022' + public_data_v1_0_0.general.7: + mode: full-refresh + object: bulk_export/{MM}/general_2023.csv + sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2023' + public_data_v1_0_0.general.8: + mode: full-refresh + object: bulk_export/{MM}/general_2024.csv + sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2024' + public_data_v1_0_0.general.9: + mode: full-refresh + object: bulk_export/{MM}/general_2025.csv + sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2025' + public_data_v1_0_0.notes_to_sefa.0: + mode: full-refresh + object: bulk_export/{MM}/notes_to_sefa_2016.csv + sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2016' + public_data_v1_0_0.notes_to_sefa.1: + mode: full-refresh + object: bulk_export/{MM}/notes_to_sefa_2017.csv + sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2017' + public_data_v1_0_0.notes_to_sefa.10: + mode: full-refresh + object: bulk_export/{MM}/notes_to_sefa_2026.csv + sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2026' + public_data_v1_0_0.notes_to_sefa.11: + mode: full-refresh + object: bulk_export/{MM}/notes_to_sefa_2027.csv + sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2027' + public_data_v1_0_0.notes_to_sefa.12: + mode: full-refresh + object: bulk_export/{MM}/notes_to_sefa_2028.csv + sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2028' + public_data_v1_0_0.notes_to_sefa.13: + mode: full-refresh + object: bulk_export/{MM}/notes_to_sefa_2029.csv + sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2029' + public_data_v1_0_0.notes_to_sefa.14: + mode: full-refresh + object: bulk_export/{MM}/notes_to_sefa_2030.csv + sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2030' + public_data_v1_0_0.notes_to_sefa.2: + mode: full-refresh + object: bulk_export/{MM}/notes_to_sefa_2018.csv + sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2018' + public_data_v1_0_0.notes_to_sefa.3: + mode: full-refresh + object: bulk_export/{MM}/notes_to_sefa_2019.csv + sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2019' + public_data_v1_0_0.notes_to_sefa.4: + mode: full-refresh + object: bulk_export/{MM}/notes_to_sefa_2020.csv + sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2020' + public_data_v1_0_0.notes_to_sefa.5: + mode: full-refresh + object: bulk_export/{MM}/notes_to_sefa_2021.csv + sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2021' + public_data_v1_0_0.notes_to_sefa.6: + mode: full-refresh + object: bulk_export/{MM}/notes_to_sefa_2022.csv + sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2022' + public_data_v1_0_0.notes_to_sefa.7: + mode: full-refresh + object: bulk_export/{MM}/notes_to_sefa_2023.csv + sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2023' + public_data_v1_0_0.notes_to_sefa.8: + mode: full-refresh + object: bulk_export/{MM}/notes_to_sefa_2024.csv + sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2024' + public_data_v1_0_0.notes_to_sefa.9: + mode: full-refresh + object: bulk_export/{MM}/notes_to_sefa_2025.csv + sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2025' + public_data_v1_0_0.passthrough.0: + mode: full-refresh + object: bulk_export/{MM}/passthrough_2016.csv + sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2016' + public_data_v1_0_0.passthrough.1: + mode: full-refresh + object: bulk_export/{MM}/passthrough_2017.csv + sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2017' + public_data_v1_0_0.passthrough.10: + mode: full-refresh + object: bulk_export/{MM}/passthrough_2026.csv + sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2026' + public_data_v1_0_0.passthrough.11: + mode: full-refresh + object: bulk_export/{MM}/passthrough_2027.csv + sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2027' + public_data_v1_0_0.passthrough.12: + mode: full-refresh + object: bulk_export/{MM}/passthrough_2028.csv + sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2028' + public_data_v1_0_0.passthrough.13: + mode: full-refresh + object: bulk_export/{MM}/passthrough_2029.csv + sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2029' + public_data_v1_0_0.passthrough.14: + mode: full-refresh + object: bulk_export/{MM}/passthrough_2030.csv + sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2030' + public_data_v1_0_0.passthrough.2: + mode: full-refresh + object: bulk_export/{MM}/passthrough_2018.csv + sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2018' + public_data_v1_0_0.passthrough.3: + mode: full-refresh + object: bulk_export/{MM}/passthrough_2019.csv + sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2019' + public_data_v1_0_0.passthrough.4: + mode: full-refresh + object: bulk_export/{MM}/passthrough_2020.csv + sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2020' + public_data_v1_0_0.passthrough.5: + mode: full-refresh + object: bulk_export/{MM}/passthrough_2021.csv + sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2021' + public_data_v1_0_0.passthrough.6: + mode: full-refresh + object: bulk_export/{MM}/passthrough_2022.csv + sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2022' + public_data_v1_0_0.passthrough.7: + mode: full-refresh + object: bulk_export/{MM}/passthrough_2023.csv + sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2023' + public_data_v1_0_0.passthrough.8: + mode: full-refresh + object: bulk_export/{MM}/passthrough_2024.csv + sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2024' + public_data_v1_0_0.passthrough.9: + mode: full-refresh + object: bulk_export/{MM}/passthrough_2025.csv + sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2025' + public_data_v1_0_0.secondary_auditors.0: + mode: full-refresh + object: bulk_export/{MM}/secondary_auditors_2016.csv + sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2016' + public_data_v1_0_0.secondary_auditors.1: + mode: full-refresh + object: bulk_export/{MM}/secondary_auditors_2017.csv + sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2017' + public_data_v1_0_0.secondary_auditors.10: + mode: full-refresh + object: bulk_export/{MM}/secondary_auditors_2026.csv + sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2026' + public_data_v1_0_0.secondary_auditors.11: + mode: full-refresh + object: bulk_export/{MM}/secondary_auditors_2027.csv + sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2027' + public_data_v1_0_0.secondary_auditors.12: + mode: full-refresh + object: bulk_export/{MM}/secondary_auditors_2028.csv + sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2028' + public_data_v1_0_0.secondary_auditors.13: + mode: full-refresh + object: bulk_export/{MM}/secondary_auditors_2029.csv + sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2029' + public_data_v1_0_0.secondary_auditors.14: + mode: full-refresh + object: bulk_export/{MM}/secondary_auditors_2030.csv + sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2030' + public_data_v1_0_0.secondary_auditors.2: + mode: full-refresh + object: bulk_export/{MM}/secondary_auditors_2018.csv + sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2018' + public_data_v1_0_0.secondary_auditors.3: + mode: full-refresh + object: bulk_export/{MM}/secondary_auditors_2019.csv + sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2019' + public_data_v1_0_0.secondary_auditors.4: + mode: full-refresh + object: bulk_export/{MM}/secondary_auditors_2020.csv + sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2020' + public_data_v1_0_0.secondary_auditors.5: + mode: full-refresh + object: bulk_export/{MM}/secondary_auditors_2021.csv + sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2021' + public_data_v1_0_0.secondary_auditors.6: + mode: full-refresh + object: bulk_export/{MM}/secondary_auditors_2022.csv + sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2022' + public_data_v1_0_0.secondary_auditors.7: + mode: full-refresh + object: bulk_export/{MM}/secondary_auditors_2023.csv + sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2023' + public_data_v1_0_0.secondary_auditors.8: + mode: full-refresh + object: bulk_export/{MM}/secondary_auditors_2024.csv + sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2024' + public_data_v1_0_0.secondary_auditors.9: + mode: full-refresh + object: bulk_export/{MM}/secondary_auditors_2025.csv + sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2025' +target: BULK_DATA_EXPORT + diff --git a/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml b/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml index aa74ea29b0..d8741960c0 100644 --- a/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml +++ b/backend/dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml @@ -22,7 +22,7 @@ streams: -- All of the general table is public data. SELECT gen.id as id, - NEXTVAL('public_data_v1_0_0.seq_v1_0_0_general') AS seq, + NEXTVAL('public_data_v1_0_0.general') AS seq, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -124,7 +124,7 @@ streams: sql: | SELECT ein.id as id, - div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_additional_eins'), 20000) as seq, + div(NEXTVAL('public_data_v1_0_0.additional_eins'), 20000) as seq, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -151,7 +151,7 @@ streams: sql: | SELECT uei.id as id, - div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_additional_ueis'), 20000) as seq, + div(NEXTVAL('public_data_v1_0_0.additional_ueis'), 20000) as seq, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -178,7 +178,7 @@ streams: sql: | SELECT cap.id as id, - div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_corrective_action_plans'), 20000) as seq, + div(NEXTVAL('public_data_v1_0_0.corrective_action_plans'), 20000) as seq, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -210,7 +210,7 @@ streams: sql: | SELECT award.id as id, - div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_federal_awards'), 20000) as seq, + div(NEXTVAL('public_data_v1_0_0.federal_awards'), 20000) as seq, dg.report_id, dg.auditee_uei, dg.audit_year, @@ -256,7 +256,7 @@ streams: sql: | SELECT finding.id as id, - div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_findings'), 20000) as seq, + div(NEXTVAL('public_data_v1_0_0.findings'), 20000) as seq, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -293,7 +293,7 @@ streams: sql: | SELECT ft.id as id, - div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_findings_text'), 20000) as seq, + div(NEXTVAL('public_data_v1_0_0.findings_text'), 20000) as seq, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -323,7 +323,7 @@ streams: sql: | SELECT note.id as id, - div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_notes_to_sefa'), 20000) as seq, + div(NEXTVAL('public_data_v1_0_0.notes_to_sefa'), 20000) as seq, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -358,7 +358,7 @@ streams: sql: | SELECT pass.id as id, - div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_passthrough'), 20000) as seq, + div(NEXTVAL('public_data_v1_0_0.passthrough'), 20000) as seq, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -387,7 +387,7 @@ streams: sql: | SELECT sa.id as id, - div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_secondary_auditors'), 20000) as seq, + div(NEXTVAL('public_data_v1_0_0.secondary_auditors'), 20000) as seq, gen.report_id, gen.auditee_uei, gen.audit_year, @@ -422,7 +422,7 @@ streams: sql: | SELECT dg.report_id, - div(NEXTVAL('public_data_v1_0_0.seq_v1_0_0_combined'), 20000) as seq, + div(NEXTVAL('public_data_v1_0_0.combined'), 20000) as seq, dfa.award_reference, df.reference_number, concat(dfa.federal_agency_prefix,'.',dfa.federal_award_extension) as aln, diff --git a/backend/tools/sling_bulk_export.sh b/backend/tools/sling_bulk_export.sh new file mode 100644 index 0000000000..2e97e9cc51 --- /dev/null +++ b/backend/tools/sling_bulk_export.sh @@ -0,0 +1,12 @@ +source tools/util_startup.sh + +function sling_bulk_export() { + startup_log "SLING_BULK_EXPORT" "Slinging data CSVs" + + SLING_ALLOW_EMPTY=1 + $SLING_EXE run -r dissemination/sql/sling/bulk_data_export/public_data_v1_0_0_single_csv.yaml + gonogo "sling_bulk_export" + + startup_log "SLING_BULK_EXPORT" "Done" + return 0 +} diff --git a/backend/tools/sling_first_run.SKIP b/backend/tools/sling_first_run.SKIP new file mode 100644 index 0000000000..8e997a5df4 --- /dev/null +++ b/backend/tools/sling_first_run.SKIP @@ -0,0 +1,49 @@ +source tools/util_startup.sh + +function sling_first_run() { + startup_log "SLING_FIRST_RUN" "Slinging data to fac-snapshot if needed" + # check_table_exists might return 1. + # It is possible set -e is kicking us out when the function + # returns an error? So, try set +e. + set +e + # The deploy action does a DB->DB backup with cgov-util. + # This should only ever actually run when testing locally. + # In the cloud environment, it will skip the actul run. + check_table_exists $FAC_SNAPSHOT_URI 'public' 'dissemination_general' + local is_general_table=$FUNCTION_RESULT + if [ $is_general_table -ne 0 ]; then + # This is the first run. + startup_log "SLING_FIRST_RUN" "Running cgov-util INITIAL." + $CGOV_UTIL_EXE db_to_db \ + --src_db fac-db \ + --dest_db fac-snapshot-db \ + --operation initial + fi + + # If the metadata table exists, it means sling has run to completion. + check_table_exists $FAC_SNAPSHOT_URI 'public_data_v1_0_0' 'metadata' + local is_metadata_table=$FUNCTION_RESULT + echo "SLING_FIRST_RUN public_data_v1_0_0.metadata = $is_metadata_table" + + # We need to load some functions for sling to complete, because + # we use those functions as part of the metadata table generation. + local base_path='dissemination/sql' + local location='sling' + + # Only run sling if the tables in the secondary DB do not exist. + if [ $is_metadata_table -ne 0 ]; then + SLING_ALLOW_EMPTY=1 + startup_log "SLING_FIRST_RUN" "API tables don't exist; running sling." + $SLING_EXE run -r dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml + gonogo "sling public data for API tables" + $SLING_EXE run -r dissemination/sql/sling/public_data_v1_0_0/suppressed_data_v1_0_0.yaml + gonogo "sling tribal data for API tables" + $SLING_EXE run -r dissemination/sql/sling/public_data_v1_0_0/public_metadata_v1_0_0.yaml + gonogo "sling metadata table" + startup_log "SLING_FIRST_RUN" "Successfully ran sling to generate tables." + else + startup_log "SLING_FIRST_RUN" "API tables exist; skipping sling." + fi + startup_log "SLING_FIRST_RUN" "Done" + return 0 +} From 1656badeee0e1bd5a65ffa5e57f9341a3552ac45 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Fri, 25 Oct 2024 08:34:50 -0400 Subject: [PATCH 39/89] Using a VCAP_SERVICES locally This lets the local config (at least for bash purposes) look almost exactly like the cloud.gov setup. --- backend/Dockerfile | 1 + backend/config.json | 90 -------- backend/config/settings.py | 6 +- .../config/vcap_services_for_containers.json | 118 ++++++++++ backend/docker-compose.yml | 4 +- backend/run.sh | 12 +- backend/tools/setup_cgov_env.sh | 12 -- backend/tools/setup_local_env.sh | 89 ++++++-- backend/tools/sling_first_run.sh | 49 ----- backend/tools/util_startup.sh | 39 +++- .../load_public_dissem_data/data/config.json | 204 ++++++++++-------- 11 files changed, 363 insertions(+), 261 deletions(-) delete mode 100644 backend/config.json create mode 100644 backend/config/vcap_services_for_containers.json delete mode 100644 backend/tools/sling_first_run.sh diff --git a/backend/Dockerfile b/backend/Dockerfile index c9ab79ae02..2e53152aca 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -18,6 +18,7 @@ RUN apt-get -yq update && \ gcc \ gnupg \ gnupg2 \ + jq \ postgresql-client \ wget diff --git a/backend/config.json b/backend/config.json deleted file mode 100644 index 2b1fc22521..0000000000 --- a/backend/config.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "user-provided": [ - { - "label": "mc", - "name": "backups", - "tags": [], - "instance_guid": "UUIDALPHA1", - "instance_name": "backups", - "binding_guid": "UUIDALPHA2", - "binding_name": null, - "credentials": { - "access_key_id": "longtest", - "secret_access_key": "longtest", - "bucket": "backups", - "endpoint": "http://127.0.0.1:9001", - "admin_username": "minioadmin", - "admin_password": "minioadmin" - } - }, - { - "label": "mc", - "name": "fac-private-s3", - "tags": [], - "instance_guid": "UUIDALPHA1", - "instance_name": "fac-private-s3", - "binding_guid": "UUIDALPHA2", - "binding_name": null, - "credentials": { - "access_key_id": "longtest", - "secret_access_key": "longtest", - "bucket": "fac-private-s3", - "endpoint": "http://127.0.0.1:9001", - "admin_username": "minioadmin", - "admin_password": "minioadmin" - } - } - ], - "aws-rds": [ - { - "label": "fac-db", - "provider": null, - "plan": null, - "name": "fac-db", - "tags": [ - "database", - "docker" - ], - "instance_guid": "UUIDINDIA1", - "instance_name": "db", - "binding_guid": "UUIDINDIA2", - "binding_name": null, - "credentials": { - "db_name": "postgres", - "host": "db", - "name": "postgres", - "password": "", - "port": "5432", - "uri": "postgres://postgres@db:5432/postgres?sslmode=disable", - "username": "postgres" - }, - "syslog_drain_url": null, - "volume_mounts": [] - }, - { - "label": "fac-snapshot-db", - "provider": null, - "plan": null, - "name": "fac-snapshot-db", - "tags": [ - "database", - "docker" - ], - "instance_guid": "UUIDJULIET1", - "instance_name": "db", - "binding_guid": "UUIDJULIET2", - "binding_name": null, - "credentials": { - "db_name": "postgres", - "host": "db2", - "name": "postgres", - "password": "", - "port": "5432", - "uri": "postgres://postgres@db2:5432/postgres?sslmode=disable", - "username": "postgres" - }, - "syslog_drain_url": null, - "volume_mounts": [] - } - ] -} diff --git a/backend/config/settings.py b/backend/config/settings.py index 4f1eda5c61..0e42c17c96 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -265,9 +265,11 @@ # MinIO only matters for local development and GitHub action environments. # These should match what we're setting in backend/run.sh - AWS_PRIVATE_ACCESS_KEY_ID = os.environ.get("AWS_PRIVATE_ACCESS_KEY_ID", "nutnutnut") + AWS_PRIVATE_ACCESS_KEY_ID = os.environ.get( + "AWS_PRIVATE_ACCESS_KEY_ID", "singleauditclearinghouse" + ) AWS_PRIVATE_SECRET_ACCESS_KEY = os.environ.get( - "AWS_PRIVATE_SECRET_ACCESS_KEY", "longtest" + "AWS_PRIVATE_SECRET_ACCESS_KEY", "singleauditclearinghouse" ) AWS_S3_PRIVATE_ENDPOINT = os.environ.get( "AWS_S3_PRIVATE_ENDPOINT", "http://minio:9000" diff --git a/backend/config/vcap_services_for_containers.json b/backend/config/vcap_services_for_containers.json new file mode 100644 index 0000000000..484ec221b6 --- /dev/null +++ b/backend/config/vcap_services_for_containers.json @@ -0,0 +1,118 @@ +{ + "s3": [ + { + "label": "s3", + "provider": "minio-local", + "plan": "basic", + "name": "fac-private-s3", + "tags": [ + "AWS", + "S3", + "object-storage" + ], + "instance_guid": "UUIDALPHA1", + "instance_name": "fac-private-s3", + "binding_guid": "UUIDALPHA2", + "binding_name": null, + "credentials": { + "uri": "http://minio:9000", + "port": 9000, + "insecure_skip_verify": false, + "access_key_id": "singleauditclearinghouse", + "secret_access_key": "singleauditclearinghouse", + "region": "us-east-1", + "bucket": "fac-private-s3", + "endpoint": "http://minio:9000", + "fips_endpoint": "http://minio:9000", + "additional_buckets": [] + }, + "syslog_drain_url": "https://ALPHA.drain.url", + "volume_mounts": [ + "no_mounts" + ] + }, + { + "label": "s3", + "provider": "minio-local", + "plan": "basic", + "name": "fac-public-s3", + "tags": [ + "AWS", + "S3", + "object-storage" + ], + "instance_guid": "UUIDALPHA1", + "instance_name": "fac-public-s3", + "binding_guid": "UUIDALPHA2", + "binding_name": null, + "credentials": { + "uri": "http://minio:9000", + "port": 9000, + "insecure_skip_verify": false, + "access_key_id": "singleauditclearinghouse", + "secret_access_key": "singleauditclearinghouse", + "region": "us-east-1", + "bucket": "fac-public-s3", + "endpoint": "http://minio:9000", + "fips_endpoint": "http://minio:9000", + "additional_buckets": [] + }, + "syslog_drain_url": "https://ALPHA.drain.url", + "volume_mounts": [ + "no_mounts" + ] + } + ], + "aws-rds": [ + { + "label": "fac-db", + "provider": null, + "plan": null, + "name": "fac-db", + "tags": [ + "database", + "docker" + ], + "instance_guid": "UUIDINDIA1", + "instance_name": "fac-db", + "binding_guid": "UUIDINDIA2", + "binding_name": null, + "credentials": { + "db_name": "postgres", + "host": "db", + "name": "postgres", + "password": "", + "port": "5432", + "uri": "postgres://postgres@db:5432/postgres?sslmode=disable", + "username": "postgres" + }, + "syslog_drain_url": null, + "volume_mounts": [] + }, + { + "label": "fac-snapshot-db", + "provider": null, + "plan": null, + "name": "fac-snapshot-db", + "tags": [ + "database", + "docker" + ], + "instance_guid": "UUIDJULIET1", + "instance_name": "fac-snapshot-db", + "binding_guid": "UUIDJULIET2", + "binding_name": null, + "credentials": { + "db_name": "postgres", + "host": "db2", + "name": "postgres", + "password": "", + "port": "5432", + "uri": "postgres://postgres@db2:5432/postgres?sslmode=disable", + "username": "postgres" + }, + "syslog_drain_url": null, + "volume_mounts": [] + } + ] +} diff --git a/backend/docker-compose.yml b/backend/docker-compose.yml index 13dc431571..3a8b65914e 100644 --- a/backend/docker-compose.yml +++ b/backend/docker-compose.yml @@ -121,8 +121,8 @@ services: - 9001:9000 - 9002:9002 environment: - MINIO_ROOT_USER: nutnutnut - MINIO_ROOT_PASSWORD: nutnutnut + MINIO_ROOT_USER: singleauditclearinghouse + MINIO_ROOT_PASSWORD: singleauditclearinghouse MINIO_API_ROOT_ACCESS: on healthcheck: test: ["CMD", "mc", "ready", "local"] diff --git a/backend/run.sh b/backend/run.sh index 005fcf5fab..9b2918effa 100755 --- a/backend/run.sh +++ b/backend/run.sh @@ -1,8 +1,7 @@ #!/bin/bash # Source everything; everything is now a function. -# Remember: bash has no idea if a function exists, -# so a typo in a function name will fail silently. Similarly, +# Remember: bash has no idea if a function exists. Similarly, # bash has horrible scoping, so use of `local` in functions is # critical for cleanliness in the startup script. source tools/util_startup.sh @@ -11,10 +10,10 @@ source tools/util_startup.sh source tools/setup_env.sh source tools/cgov_util_local_only.sh source tools/curation_audit_tracking_init.sh +source tools/sling_bulk_export.sh source tools/migrate_app_tables.sh source tools/seed_cog_baseline.sh source tools/setup_env.sh -# source tools/sling_first_run.sh source tools/sql_pre_post.sh source tools/util_startup.sh @@ -50,6 +49,13 @@ gonogo "migrate_app_tables" sql_post gonogo "sql_post" +##### +# BULK EXPORT +# Creates CSV and XLSX exports of all of the +# public data in the fac-private-s3 bucket. +sling_bulk_export +gonogo "sling_bulk_export" + ##### # CURATION AUDIT TRACKING curation_audit_tracking_init diff --git a/backend/tools/setup_cgov_env.sh b/backend/tools/setup_cgov_env.sh index 9dc60bfa0f..60a785052f 100644 --- a/backend/tools/setup_cgov_env.sh +++ b/backend/tools/setup_cgov_env.sh @@ -2,18 +2,6 @@ source tools/util_startup.sh # Aliases need to be outside of function scope -export _GET_AWS_RESULT="NONE" - -function get_aws_s3() { - local bucket="$1" - local key="$2" - _GET_AWS_RESULT=$(echo $VCAP_SERVICES | \ - jq --arg bucket "$bucket" '.s3 | map(select(.instance_name==$bucket))' | \ - jq .[] | \ - jq --arg key "$key" '.credentials | .[$key]') - return 0 -} - function setup_cgov_env { startup_log "CGOV_ENV" "We are in a cloud.gov envirnoment." diff --git a/backend/tools/setup_local_env.sh b/backend/tools/setup_local_env.sh index 3cf4d209e5..0bff2c9630 100644 --- a/backend/tools/setup_local_env.sh +++ b/backend/tools/setup_local_env.sh @@ -5,19 +5,69 @@ function setup_local_env { if [[ "${ENV}" == "LOCAL" || "${ENV}" == "TESTING" ]]; then startup_log "LOCAL_ENV" "We are in a local envirnoment." - export AWS_PUBLIC_BUCKET_NAME="fac-public-s3" - export AWS_PRIVATE_BUCKET_NAME="fac-private-s3" + # Load a fake VCAP_SERVICES file into the environment variable, + # so we can mimic the cloud.gov setup. + export VCAP_SERVICES=$(cat config/vcap_services_for_containers.json) + check_env_var_not_empty "VCAP_SERVICES" - export AWS_PRIVATE_ACCESS_KEY_ID="nutnutnut" - export AWS_PRIVATE_SECRET_ACCESS_KEY="nutnutnut" - export AWS_S3_PRIVATE_ENDPOINT="http://minio:9001" + # export AWS_PUBLIC_ACCESS_KEY_ID="singleauditclearinghouse" + # export AWS_PUBLIC_SECRET_ACCESS_KEY="singleauditclearinghouse" + # export AWS_S3_PUBLIC_ENDPOINT="http://minio:9000" - export AWS_PUBLIC_ACCESS_KEY_ID="nutnutnut" - export AWS_PUBLIC_SECRET_ACCESS_KEY="nutnutnut" - export AWS_S3_PUBLIC_ENDPOINT="http://minio:9001" - mc alias set myminio ${AWS_S3_PRIVATE_ENDPOINT} ${AWS_PRIVATE_ACCESS_KEY_ID} ${AWS_PRIVATE_ACCESS_KEY_ID} - # until (mc config host add myminio $AWS_PRIVATE_ENDPOINT nutnutnut nutnutnut) do echo '...waiting...' && sleep 1; done; + # https://stackoverflow.com/questions/48712545/break-jq-query-string-into-lines + # jq is fine with line breaks in strings. Just don't escape them. + # Makes long queries more readable. Maybe. + + # export AWS_PUBLIC_BUCKET_NAME="fac-public-s3" + # export AWS_PRIVATE_BUCKET_NAME="fac-private-s3" + + export AWS_PRIVATE_BUCKET_NAME=$(echo $VCAP_SERVICES \ + | jq --raw-output '.s3 + | map(select(.instance_name + | contains("fac-private-s3"))) + | .[] .credentials.bucket') + check_env_var_not_empty "AWS_PRIVATE_BUCKET_NAME" + + export AWS_PUBLIC_BUCKET_NAME=$(echo $VCAP_SERVICES \ + | jq --raw-output '.s3 + | map(select(.instance_name + | contains("fac-public-s3"))) + | .[] .credentials.bucket') + + + # export AWS_PRIVATE_ACCESS_KEY_ID="singleauditclearinghouse" + # export AWS_PRIVATE_SECRET_ACCESS_KEY="singleauditclearinghouse" + # export AWS_S3_PRIVATE_ENDPOINT="http://minio:9000" + + get_aws_s3 "fac-private-s3" "access_key_id" + export AWS_PRIVATE_ACCESS_KEY_ID=$_GET_AWS_RESULT + check_env_var_not_empty "AWS_PRIVATE_ACCESS_KEY_ID" + + get_aws_s3 "fac-private-s3" "secret_access_key" + export AWS_PRIVATE_SECRET_ACCESS_KEY=$_GET_AWS_RESULT + check_env_var_not_empty "AWS_PRIVATE_SECRET_ACCESS_KEY" + + get_aws_s3 "fac-private-s3" "endpoint" + export AWS_S3_PRIVATE_ENDPOINT=$_GET_AWS_RESULT + check_env_var_not_empty "AWS_S3_PRIVATE_ENDPOINT" + + get_aws_s3 "fac-public-s3" "access_key_id" + export AWS_PUBLIC_ACCESS_KEY_ID=$_GET_AWS_RESULT + check_env_var_not_empty "AWS_PUBLIC_ACCESS_KEY_ID" + + get_aws_s3 "fac-public-s3" "secret_access_key" + export AWS_PUBLIC_SECRET_ACCESS_KEY=$_GET_AWS_RESULT + check_env_var_not_empty "AWS_PUBLIC_SECRET_ACCESS_KEY" + + get_aws_s3 "fac-public-s3" "endpoint" + export AWS_S3_PUBLIC_ENDPOINT=$_GET_AWS_RESULT + check_env_var_not_empty "AWS_S3_PUBLIC_ENDPOINT" + + #export MC_HOST_=https://::@ + export MC_HOST_myminio="http://${AWS_PRIVATE_ACCESS_KEY_ID}:${AWS_PRIVATE_SECRET_ACCESS_KEY}@minio:9000" + # mc alias set myminio ${AWS_S3_PRIVATE_ENDPOINT} ${AWS_PRIVATE_ACCESS_KEY_ID} ${AWS_PRIVATE_ACCESS_KEY_ID} + # until (mc config host add myminio $AWS_PRIVATE_ENDPOINT singleauditclearinghouse singleauditclearinghouse) do echo '...waiting...' && sleep 1; done; # Do nothing if the bucket already exists. # https://min.io/docs/minio/linux/reference/minio-mc/mc-mb.html mc mb --ignore-existing myminio/${AWS_PUBLIC_BUCKET_NAME} @@ -35,9 +85,7 @@ function setup_local_env { export PSQL_EXE='psql --single-transaction -v ON_ERROR_STOP=on' export PSQL_EXE_NO_TXN='psql -v ON_ERROR_STOP=on' - export SLING_EXE='/bin/sling' - export CGOV_UTIL_EXE='/bin/cgov-util' - + # Locally, we need to pull in sling. # In production, it gets pulled in via the build/deploy process. curl -LO 'https://github.com/slingdata-io/sling-cli/releases/latest/download/sling_linux_amd64.tar.gz' @@ -52,6 +100,21 @@ function setup_local_env { chmod 755 gov.gsa.fac.cgov-util mv gov.gsa.fac.cgov-util /bin/cgov-util + export SLING_EXE='/bin/sling' + export CGOV_UTIL_EXE='/bin/cgov-util' + + show_env_var "AWS_S3_PRIVATE_ENDPOINT" + + $SLING_EXE conns set \ + BULK_DATA_EXPORT \ + type=s3 \ + bucket="${AWS_PRIVATE_BUCKET_NAME}" \ + access_key_id="${AWS_PRIVATE_ACCESS_KEY_ID}" \ + secret_access_key="${AWS_PRIVATE_SECRET_ACCESS_KEY}" \ + endpoint="${AWS_S3_PRIVATE_ENDPOINT}" + $SLING_EXE conns test BULK_DATA_EXPORT + gonogo "local_minio_conns_test" + # We need a config.json in the directory we are running # things from (or PWD). cp util/load_public_dissem_data/data/config.json . diff --git a/backend/tools/sling_first_run.sh b/backend/tools/sling_first_run.sh deleted file mode 100644 index 8e997a5df4..0000000000 --- a/backend/tools/sling_first_run.sh +++ /dev/null @@ -1,49 +0,0 @@ -source tools/util_startup.sh - -function sling_first_run() { - startup_log "SLING_FIRST_RUN" "Slinging data to fac-snapshot if needed" - # check_table_exists might return 1. - # It is possible set -e is kicking us out when the function - # returns an error? So, try set +e. - set +e - # The deploy action does a DB->DB backup with cgov-util. - # This should only ever actually run when testing locally. - # In the cloud environment, it will skip the actul run. - check_table_exists $FAC_SNAPSHOT_URI 'public' 'dissemination_general' - local is_general_table=$FUNCTION_RESULT - if [ $is_general_table -ne 0 ]; then - # This is the first run. - startup_log "SLING_FIRST_RUN" "Running cgov-util INITIAL." - $CGOV_UTIL_EXE db_to_db \ - --src_db fac-db \ - --dest_db fac-snapshot-db \ - --operation initial - fi - - # If the metadata table exists, it means sling has run to completion. - check_table_exists $FAC_SNAPSHOT_URI 'public_data_v1_0_0' 'metadata' - local is_metadata_table=$FUNCTION_RESULT - echo "SLING_FIRST_RUN public_data_v1_0_0.metadata = $is_metadata_table" - - # We need to load some functions for sling to complete, because - # we use those functions as part of the metadata table generation. - local base_path='dissemination/sql' - local location='sling' - - # Only run sling if the tables in the secondary DB do not exist. - if [ $is_metadata_table -ne 0 ]; then - SLING_ALLOW_EMPTY=1 - startup_log "SLING_FIRST_RUN" "API tables don't exist; running sling." - $SLING_EXE run -r dissemination/sql/sling/public_data_v1_0_0/public_data_v1_0_0.yaml - gonogo "sling public data for API tables" - $SLING_EXE run -r dissemination/sql/sling/public_data_v1_0_0/suppressed_data_v1_0_0.yaml - gonogo "sling tribal data for API tables" - $SLING_EXE run -r dissemination/sql/sling/public_data_v1_0_0/public_metadata_v1_0_0.yaml - gonogo "sling metadata table" - startup_log "SLING_FIRST_RUN" "Successfully ran sling to generate tables." - else - startup_log "SLING_FIRST_RUN" "API tables exist; skipping sling." - fi - startup_log "SLING_FIRST_RUN" "Done" - return 0 -} diff --git a/backend/tools/util_startup.sh b/backend/tools/util_startup.sh index a5ba18cc5d..a4c37ac346 100644 --- a/backend/tools/util_startup.sh +++ b/backend/tools/util_startup.sh @@ -65,6 +65,41 @@ function run_sql () { local db_uri="$1" local path="$2" echo "BEGIN run_sql < $path" - $PSQL_EXE $db_uri < $path - gonogo "GONOGO run_sql < $path" + if [[ "$file" == *"notxn"* ]]; then + $PSQL_EXE_NO_TXN $db_uri < $path; + gonogo "GONOGO run_sql < $path" + else + $PSQL_EXE $db_uri < $path; + gonogo "GONOGO run_sql < $path" + fi +} + +export _GET_AWS_RESULT="NONE" + +function get_aws_s3() { + local bucket="$1" + local key="$2" + _GET_AWS_RESULT=$(echo $VCAP_SERVICES | \ + jq --arg bucket "$bucket" '.s3 | map(select(.instance_name==$bucket))' | \ + jq .[] | \ + jq --raw-output --arg key "$key" '.credentials | .[$key]') + return 0 +} + +function check_env_var_not_empty() { + local var_name="$1" + local var_value="${!var_name}" + if [ -z "${var_value}" ]; + then + echo "CHECK_ENV_VAR ${var_name} has value '${var_value}'. Exiting."; + exit 255; + else + echo "CHECK_ENV_VAR ${var_name} is OK." + fi +} + +function show_env_var() { + local var_name="$1" + local var_value="${!var_name}" + echo "SHOW_ENV_VAR ${var_name} has value '${var_value}'."; } diff --git a/backend/util/load_public_dissem_data/data/config.json b/backend/util/load_public_dissem_data/data/config.json index 2b1fc22521..af567bf84d 100644 --- a/backend/util/load_public_dissem_data/data/config.json +++ b/backend/util/load_public_dissem_data/data/config.json @@ -1,90 +1,118 @@ { - "user-provided": [ - { - "label": "mc", - "name": "backups", - "tags": [], - "instance_guid": "UUIDALPHA1", - "instance_name": "backups", - "binding_guid": "UUIDALPHA2", - "binding_name": null, - "credentials": { - "access_key_id": "longtest", - "secret_access_key": "longtest", - "bucket": "backups", - "endpoint": "http://127.0.0.1:9001", - "admin_username": "minioadmin", - "admin_password": "minioadmin" - } - }, - { - "label": "mc", - "name": "fac-private-s3", - "tags": [], - "instance_guid": "UUIDALPHA1", - "instance_name": "fac-private-s3", - "binding_guid": "UUIDALPHA2", - "binding_name": null, - "credentials": { - "access_key_id": "longtest", - "secret_access_key": "longtest", - "bucket": "fac-private-s3", - "endpoint": "http://127.0.0.1:9001", - "admin_username": "minioadmin", - "admin_password": "minioadmin" - } - } - ], - "aws-rds": [ - { - "label": "fac-db", - "provider": null, - "plan": null, - "name": "fac-db", - "tags": [ - "database", - "docker" - ], - "instance_guid": "UUIDINDIA1", - "instance_name": "db", - "binding_guid": "UUIDINDIA2", - "binding_name": null, - "credentials": { - "db_name": "postgres", - "host": "db", - "name": "postgres", - "password": "", - "port": "5432", - "uri": "postgres://postgres@db:5432/postgres?sslmode=disable", - "username": "postgres" - }, - "syslog_drain_url": null, - "volume_mounts": [] - }, - { - "label": "fac-snapshot-db", - "provider": null, - "plan": null, - "name": "fac-snapshot-db", - "tags": [ - "database", - "docker" - ], - "instance_guid": "UUIDJULIET1", - "instance_name": "db", - "binding_guid": "UUIDJULIET2", - "binding_name": null, - "credentials": { - "db_name": "postgres", - "host": "db2", - "name": "postgres", - "password": "", - "port": "5432", - "uri": "postgres://postgres@db2:5432/postgres?sslmode=disable", - "username": "postgres" - }, - "syslog_drain_url": null, - "volume_mounts": [] - } - ] + "s3": [ + { + "label": "s3", + "provider": "minio-local", + "plan": "basic", + "name": "fac-private-s3", + "tags": [ + "AWS", + "S3", + "object-storage" + ], + "instance_guid": "UUIDALPHA1", + "instance_name": "fac-private-s3", + "binding_guid": "UUIDALPHA2", + "binding_name": null, + "credentials": { + "uri": "http://minio:9000", + "port": 9000, + "insecure_skip_verify": false, + "access_key_id": "singleauditclearinghouse", + "secret_access_key": "singleauditclearinghouse", + "region": "us-east-1", + "bucket": "fac-private-s3", + "endpoint": "minio", + "fips_endpoint": "minio", + "additional_buckets": [] + }, + "syslog_drain_url": "https://ALPHA.drain.url", + "volume_mounts": [ + "no_mounts" + ] + }, + { + "label": "s3", + "provider": "minio-local", + "plan": "basic", + "name": "fac-public-s3", + "tags": [ + "AWS", + "S3", + "object-storage" + ], + "instance_guid": "UUIDALPHA1", + "instance_name": "fac-public-s3", + "binding_guid": "UUIDALPHA2", + "binding_name": null, + "credentials": { + "uri": "http://minio:9000", + "port": 9000, + "insecure_skip_verify": false, + "access_key_id": "singleauditclearinghouse", + "secret_access_key": "singleauditclearinghouse", + "region": "us-east-1", + "bucket": "fac-public-s3", + "endpoint": "minio", + "fips_endpoint": "minio", + "additional_buckets": [] + }, + "syslog_drain_url": "https://ALPHA.drain.url", + "volume_mounts": [ + "no_mounts" + ] + } + ], + "aws-rds": [ + { + "label": "fac-db", + "provider": null, + "plan": null, + "name": "fac-db", + "tags": [ + "database", + "docker" + ], + "instance_guid": "UUIDINDIA1", + "instance_name": "db", + "binding_guid": "UUIDINDIA2", + "binding_name": null, + "credentials": { + "db_name": "postgres", + "host": "db", + "name": "postgres", + "password": "", + "port": "5432", + "uri": "postgres://postgres@db:5432/postgres?sslmode=disable", + "username": "postgres" + }, + "syslog_drain_url": null, + "volume_mounts": [] + }, + { + "label": "fac-snapshot-db", + "provider": null, + "plan": null, + "name": "fac-snapshot-db", + "tags": [ + "database", + "docker" + ], + "instance_guid": "UUIDJULIET1", + "instance_name": "db", + "binding_guid": "UUIDJULIET2", + "binding_name": null, + "credentials": { + "db_name": "postgres", + "host": "db2", + "name": "postgres", + "password": "", + "port": "5432", + "uri": "postgres://postgres@db2:5432/postgres?sslmode=disable", + "username": "postgres" + }, + "syslog_drain_url": null, + "volume_mounts": [] + } + ] } From 6f5b2058b4e20e5fd57db44fcd8d1921828cf31d Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Fri, 25 Oct 2024 09:24:25 -0400 Subject: [PATCH 40/89] Minor change, move to /tmp This keeps the container from dropping products like cgov-util into our local filesystem. --- backend/.gitignore | 1 + backend/tools/setup_local_env.sh | 27 ++++++++++++++++----------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/backend/.gitignore b/backend/.gitignore index 335ec9573d..1d9488fd11 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -1 +1,2 @@ *.tar.gz +config.json diff --git a/backend/tools/setup_local_env.sh b/backend/tools/setup_local_env.sh index 0bff2c9630..d5f96d777a 100644 --- a/backend/tools/setup_local_env.sh +++ b/backend/tools/setup_local_env.sh @@ -88,23 +88,28 @@ function setup_local_env { # Locally, we need to pull in sling. # In production, it gets pulled in via the build/deploy process. - curl -LO 'https://github.com/slingdata-io/sling-cli/releases/latest/download/sling_linux_amd64.tar.gz' - tar xf sling_linux_amd64.tar.gz - rm -f sling_linux_amd64.tar.gz - chmod +x sling - mv sling /bin/sling + pushd /tmp + curl -LO 'https://github.com/slingdata-io/sling-cli/releases/latest/download/sling_linux_amd64.tar.gz' + tar xf sling_linux_amd64.tar.gz + rm -f sling_linux_amd64.tar.gz + chmod +x sling + mv sling /bin/sling + popd # And we need cgov-util - curl -L -O https://github.com/GSA-TTS/fac-backup-utility/releases/download/v0.1.8/gov.gsa.fac.cgov-util-v0.1.8-linux-amd64.tar.gz - tar xvzf gov.gsa.fac.cgov-util-v0.1.8-linux-amd64.tar.gz gov.gsa.fac.cgov-util - chmod 755 gov.gsa.fac.cgov-util - mv gov.gsa.fac.cgov-util /bin/cgov-util - + pushd /tmp + local CGOV_VERSION=v0.1.8 + curl -L -O https://github.com/GSA-TTS/fac-backup-utility/releases/download/${CGOV_VERSION}/gov.gsa.fac.cgov-util-${CGOV_VERSION}-linux-amd64.tar.gz + tar xvzf gov.gsa.fac.cgov-util-${CGOV_VERSION}-linux-amd64.tar.gz gov.gsa.fac.cgov-util + chmod 755 gov.gsa.fac.cgov-util + mv gov.gsa.fac.cgov-util /bin/cgov-util + popd + export SLING_EXE='/bin/sling' export CGOV_UTIL_EXE='/bin/cgov-util' show_env_var "AWS_S3_PRIVATE_ENDPOINT" - + $SLING_EXE conns set \ BULK_DATA_EXPORT \ type=s3 \ From 9ea4a299290ca8eaab3fe644bdb7cd295007a9e6 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Fri, 25 Oct 2024 12:35:31 -0400 Subject: [PATCH 41/89] Move curation tracking init This is now "just SQL," and does not need to be managed by the management command. It should run in the `pre` sequence. This way, the triggers are configured early in the process. We then run the `--disable` promptly afterwards. --- .../sql/fac-db/pre/020_init_curation_auditing.sql} | 0 backend/run.sh | 13 +++++-------- ...g_init.sh => curation_audit_tracking_disable.sh} | 1 - 3 files changed, 5 insertions(+), 9 deletions(-) rename backend/{curation/sql/init_curation_auditing.sql => dissemination/sql/fac-db/pre/020_init_curation_auditing.sql} (100%) rename backend/tools/{curation_audit_tracking_init.sh => curation_audit_tracking_disable.sh} (83%) diff --git a/backend/curation/sql/init_curation_auditing.sql b/backend/dissemination/sql/fac-db/pre/020_init_curation_auditing.sql similarity index 100% rename from backend/curation/sql/init_curation_auditing.sql rename to backend/dissemination/sql/fac-db/pre/020_init_curation_auditing.sql diff --git a/backend/run.sh b/backend/run.sh index 9b2918effa..57de75725e 100755 --- a/backend/run.sh +++ b/backend/run.sh @@ -9,7 +9,7 @@ source tools/util_startup.sh # for local envs (LOCAL or TESTING) and cloud.gov source tools/setup_env.sh source tools/cgov_util_local_only.sh -source tools/curation_audit_tracking_init.sh +source tools/curation_audit_tracking_disable.sh source tools/sling_bulk_export.sh source tools/migrate_app_tables.sh source tools/seed_cog_baseline.sh @@ -36,6 +36,8 @@ gonogo "cgov_util_local_only" # This tears down things that would conflict with migrations, etc. sql_pre gonogo "sql_pre" +curation_audit_tracking_disable +gonogo "curation_audit_tracking_disable" ##### # MIGRATE APP TABLES @@ -51,16 +53,11 @@ gonogo "sql_post" ##### # BULK EXPORT -# Creates CSV and XLSX exports of all of the -# public data in the fac-private-s3 bucket. +# Creates CSV exports of all of the public data, +# placing it in the fac-private-s3 bucket. sling_bulk_export gonogo "sling_bulk_export" -##### -# CURATION AUDIT TRACKING -curation_audit_tracking_init -gonogo "curation_audit_tracking_init" - ##### # SEED COG/OVER TABLES # Setup tables for cog/over assignments diff --git a/backend/tools/curation_audit_tracking_init.sh b/backend/tools/curation_audit_tracking_disable.sh similarity index 83% rename from backend/tools/curation_audit_tracking_init.sh rename to backend/tools/curation_audit_tracking_disable.sh index d30e66499a..2a26eaa3cc 100644 --- a/backend/tools/curation_audit_tracking_init.sh +++ b/backend/tools/curation_audit_tracking_disable.sh @@ -2,7 +2,6 @@ source tools/util_startup.sh function curation_audit_tracking_init { startup_log "CURATION_AUDIT_TRACKING" "BEGIN" - python manage.py curation_audit_tracking --init python manage.py curation_audit_tracking --disable local result=$? startup_log "CURATION_AUDIT_TRACKING" "END" From cb0fa77658f911a16ddfa19edfd0aedb66c3791c Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Fri, 25 Oct 2024 12:38:51 -0400 Subject: [PATCH 42/89] Removing unnecessary CREATE SCHEMA These are redundant; they're issued earlier in the sequence. --- .../sql/fac-snapshot-db/post/121_api_v1_1_0_create_schema.sql | 3 --- .../sql/fac-snapshot-db/post/141_api_v2_0_0_create_schema.sql | 2 -- 2 files changed, 5 deletions(-) delete mode 100644 backend/dissemination/sql/fac-snapshot-db/post/121_api_v1_1_0_create_schema.sql delete mode 100644 backend/dissemination/sql/fac-snapshot-db/post/141_api_v2_0_0_create_schema.sql diff --git a/backend/dissemination/sql/fac-snapshot-db/post/121_api_v1_1_0_create_schema.sql b/backend/dissemination/sql/fac-snapshot-db/post/121_api_v1_1_0_create_schema.sql deleted file mode 100644 index c2d48c5db8..0000000000 --- a/backend/dissemination/sql/fac-snapshot-db/post/121_api_v1_1_0_create_schema.sql +++ /dev/null @@ -1,3 +0,0 @@ -CREATE SCHEMA IF NOT EXISTS api_v1_1_0; -CREATE SCHEMA IF NOT EXISTS api_v1_1_0_functions; - \ No newline at end of file diff --git a/backend/dissemination/sql/fac-snapshot-db/post/141_api_v2_0_0_create_schema.sql b/backend/dissemination/sql/fac-snapshot-db/post/141_api_v2_0_0_create_schema.sql deleted file mode 100644 index 9cfe24e77f..0000000000 --- a/backend/dissemination/sql/fac-snapshot-db/post/141_api_v2_0_0_create_schema.sql +++ /dev/null @@ -1,2 +0,0 @@ -CREATE SCHEMA IF NOT EXISTS api_v2_0_0; -CREATE SCHEMA IF NOT EXISTS api_v2_0_0_functions; From 03a4e6d1019677b085fd9995b9e73ef83faffdbc Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Fri, 25 Oct 2024 14:04:27 -0400 Subject: [PATCH 43/89] Fixed my partial rename --- backend/tools/curation_audit_tracking_disable.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/tools/curation_audit_tracking_disable.sh b/backend/tools/curation_audit_tracking_disable.sh index 2a26eaa3cc..e2597a5d05 100644 --- a/backend/tools/curation_audit_tracking_disable.sh +++ b/backend/tools/curation_audit_tracking_disable.sh @@ -1,6 +1,6 @@ source tools/util_startup.sh -function curation_audit_tracking_init { +function curation_audit_tracking_disable() { startup_log "CURATION_AUDIT_TRACKING" "BEGIN" python manage.py curation_audit_tracking --disable local result=$? From 1747897e68e936440d80fd622c7ae56e30dce13e Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Fri, 25 Oct 2024 14:57:47 -0400 Subject: [PATCH 44/89] This points advanced search at pd.combined Correctly uses DB2 for advanced search. May want to implement it differently. Changes for this in one commit, to make finding it easy. --- backend/config/db_url.py | 6 ++---- backend/config/settings.py | 23 +++++++++++++++++++---- backend/dissemination/models.py | 5 ++++- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/backend/config/db_url.py b/backend/config/db_url.py index d84419065d..39ac0cbd6b 100644 --- a/backend/config/db_url.py +++ b/backend/config/db_url.py @@ -1,12 +1,10 @@ from django.core.exceptions import ImproperlyConfigured -def get_db_url_from_vcap_services( - vcap, -): +def get_db_url_from_vcap_services(vcap, db_instance_name="fac-db"): database_url = None for db_service in vcap.get("aws-rds", []): - if db_service.get("instance_name") == "fac-db": + if db_service.get("instance_name") == db_instance_name: database_url = db_service["credentials"]["uri"] break diff --git a/backend/config/settings.py b/backend/config/settings.py index 0e42c17c96..3acc298eb0 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -229,9 +229,18 @@ DEBUG = False if ENVIRONMENT not in ["DEVELOPMENT", "PREVIEW", "STAGING", "PRODUCTION"]: + + # FIXME: This is now identical between local and cloud.gov, because we have + # a "fake" VCAP_SERVICES environment variable. Local DBs and S3 buckets + # can be configured the same way as their cloud equivalents. This can be + # refactored for simpler config loading in the app. + + vcap = json.loads(env.str("VCAP_SERVICES")) DATABASES = { - "default": env.dj_db_url( - "DATABASE_URL", default="postgres://postgres:password@0.0.0.0/backend" + "default": dj_database_url.parse(get_db_url_from_vcap_services(vcap, "fac-db")), + "fac-db": dj_database_url.parse(get_db_url_from_vcap_services(vcap, "fac-db")), + "fac-snapshot-db": dj_database_url.parse( + get_db_url_from_vcap_services(vcap, "fac-snapshot-db") ), } STORAGES = { @@ -301,8 +310,14 @@ vcap = json.loads(env.str("VCAP_SERVICES")) - DB_URL = get_db_url_from_vcap_services(vcap) - DATABASES = {"default": dj_database_url.parse(DB_URL)} + # DB_URL = get_db_url_from_vcap_services(vcap) + DATABASES = { + "default": dj_database_url.parse(get_db_url_from_vcap_services(vcap, "fac-db")), + "fac-db": dj_database_url.parse(get_db_url_from_vcap_services(vcap, "fac-db")), + "fac-snapshot-db": dj_database_url.parse( + get_db_url_from_vcap_services(vcap, "fac-snapshot-db") + ), + } for service in vcap["s3"]: if service["instance_name"] == "fac-public-s3": diff --git a/backend/dissemination/models.py b/backend/dissemination/models.py index 0abf7bee30..43a8391b17 100644 --- a/backend/dissemination/models.py +++ b/backend/dissemination/models.py @@ -627,7 +627,7 @@ class DisseminationCombined(models.Model): # Meta options class Meta: managed = False - db_table = "dissemination_combined" + db_table = 'public_data_v1_0_0"."combined' # General Information report_id = models.TextField( @@ -970,6 +970,9 @@ class Meta: ) +DisseminationCombined.objects = DisseminationCombined.objects.using("fac-snapshot-db") + + class InvalidAuditRecord(models.Model): """Model holds records that have been migrated as is, without validation or changes.""" From 7315cf93b4fae48ad72bd6cf34c2214d5f049382 Mon Sep 17 00:00:00 2001 From: Bobby Novak <176936850+rnovak338@users.noreply.github.com> Date: Fri, 25 Oct 2024 15:21:05 -0400 Subject: [PATCH 45/89] Admin Panel and touch-ups Admin Panel - Introducing 2 new groups - `helpdesk` and `readonly`. These groups will be used with new permissions for performing Admin API operations directly through the UI instead. - Introducing 2 new tables, which were previously accessible through the Admin API. Helpdesk will have CRUD operations on these tables. StaffUser management - Introduced a `staffusers.json` file, which will now manage the permissions for the staff in respect to the Admin UI. - New mgmt command which is handled through `run.sh` to create the staff users according to the above JSON list. - Updated onboarding document to reflect a new user joining the Admin Panel. OTHER - linting. --- .github/ISSUE_TEMPLATE/onboarding.md | 5 +- backend/Makefile | 7 +- backend/README.md | 270 ++++++++++++++++++ backend/config/staffusers.json | 20 ++ backend/dissemination/admin.py | 27 +- .../create_partition_statements.py | 4 +- backend/dissemination/sql/test_api.py | 7 +- backend/run.sh | 5 + backend/tools/setup_cgov_env.py | 1 - backend/users/admin.py | 29 +- .../management/commands/create_staffusers.py | 115 ++++++++ backend/users/models.py | 1 + 12 files changed, 476 insertions(+), 15 deletions(-) create mode 100644 backend/README.md create mode 100644 backend/config/staffusers.json create mode 100644 backend/users/management/commands/create_staffusers.py diff --git a/.github/ISSUE_TEMPLATE/onboarding.md b/.github/ISSUE_TEMPLATE/onboarding.md index 86b23a05d6..47b0a7d024 100644 --- a/.github/ISSUE_TEMPLATE/onboarding.md +++ b/.github/ISSUE_TEMPLATE/onboarding.md @@ -80,6 +80,8 @@ Note: If you're not able to do any of these yourself, you're still responsible f - [ ] [Add as a form manager to the touchpoints recruitment intercept](https://touchpoints.app.cloud.gov/admin/forms/9412c559/permissions) **For engineers, also...** +- [ ] Make sure you have a `login.gov` account and have logged into the FAC application at least once. + - [ ] Then, add your email to the `readonly` list in [staffusers.json](../../backend/config/staffusers.json). - [ ] [Add as a member of the FAC group in New Relic](https://one.newrelic.com/admin-portal/organizations/users-list) (@GSA-TTS/fac-admins can do this) **For product leads/owners, also...** @@ -88,5 +90,6 @@ Note: If you're not able to do any of these yourself, you're still responsible f - [ ] Also give them the `Maintainer` role in [the FAC-team team in GitHub](https://github.com/orgs/GSA-TTS/teams/fac-team/members). **For helpdesk, also...** -- [ ] Add them to the list of staff users for [Django Admin](https://app.fac.gov/admin/users/staffuser/). +- [ ] Make sure you have a `login.gov` account and have logged into the FAC application at least once. + - [ ] Then, add your email to the `helpdesk` list in [staffusers.json](../../backend/config/staffusers.json). - [ ] Give them access to the [Help Desk](https://fac-gov.zendesk.com/admin/people/team/members) as a team member. diff --git a/backend/Makefile b/backend/Makefile index 97d62c621a..2d5f42a508 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -128,7 +128,12 @@ system-prune: -docker system prune -f volume-prune: - -docker volume prune -f + -docker volume prune -f + +docker-clean: + docker compose down + docker rm -f $(shell docker ps -a -q) + docker volume rm $(shell docker volume ls -q) # Proceed past errors. # We want all of these commands to run, even if one fails. diff --git a/backend/README.md b/backend/README.md new file mode 100644 index 0000000000..30832a2381 --- /dev/null +++ b/backend/README.md @@ -0,0 +1,270 @@ + +

drawing

+ +

Slings from a data source to a data target.

+

See docs.slingdata.io for more details.

+ + +

+ + GitHub + + + + + + Go Reference + + + Discord + + + GitHub tag (latest SemVer pre-release) + + + Pip Downloads + + + Pip Downloads + +

+ +Sling is a passion project turned into a free CLI Product which offers an easy solution to create and maintain small to medium volume data pipelines using the Extract & Load (EL) approach. It focuses on data movement between: + +* Database to Database +* File System to Database +* Database to File System + + +https://github.com/slingdata-io/sling-cli/assets/7671010/e10ee716-1de8-4d53-8eb2-95c6d9d7f9f0 + +Some key features: +- Single Binary deployment (built with Go). See [installation](https://docs.slingdata.io/sling-cli/getting-started) page. +- Use Custom SQL as a stream: `--src-stream='select * from my_table where col1 > 10'` +- Manage / View / Test / Discover your connections with the [`sling conns`](https://docs.slingdata.io/sling-cli/environment#managing-connections) sub-command +- Use Environment Variable as connections if you prefer (`export MY_PG='postgres//...`)' +- Provide YAML or JSON configurations (perfect for git version control). +- Powerful [Replication](https://docs.slingdata.io/sling-cli/run/configuration/replication) logic, to replication many tables with a wildcard (`my_schema.*`). +- Reads your existing [DBT connections](https://docs.slingdata.io/sling-cli/environment#dbt-profiles-dbt-profiles.yml) +- Use your environment variable in your YAML / JSON config (`select * from my_table where date = '{date}'`) +- Convenient [Transformations](https://docs.slingdata.io/sling-cli/run/configuration/transformations), such as the `flatten` option, which auto-creates columns from your nested fields. +- Run Pre & Post SQL commands. +- many more! + +--- + +Example [Replication](https://docs.slingdata.io/sling-cli/run/configuration/replication): + +

replication.yaml

+ +--- + +Available Connectors: +- **Databases**: [`bigquery`](https://docs.slingdata.io/connections/database-connections/bigquery) [`bigtable`](https://docs.slingdata.io/connections/database-connections/bigtable) [`clickhouse`](https://docs.slingdata.io/connections/database-connections/clickhouse) [`duckdb`](https://docs.slingdata.io/connections/database-connections/duckdb) [`mariadb`](https://docs.slingdata.io/connections/database-connections/mariadb) [`motherduck`](https://docs.slingdata.io/connections/database-connections/motherduck) [`mysql`](https://docs.slingdata.io/connections/database-connections/mysql) [`oracle`](https://docs.slingdata.io/connections/database-connections/oracle) [`postgres`](https://docs.slingdata.io/connections/database-connections/postgres) [`redshift`](https://docs.slingdata.io/connections/database-connections/redshift) [`snowflake`](https://docs.slingdata.io/connections/database-connections/snowflake) [`sqlite`](https://docs.slingdata.io/connections/database-connections/sqlite) [`sqlserver`](https://docs.slingdata.io/connections/database-connections/sqlserver) [`starrocks`](https://docs.slingdata.io/connections/database-connections/starrocks) [`prometheus`](https://docs.slingdata.io/connections/database-connections/prometheus) [`proton`](https://docs.slingdata.io/connections/database-connections/proton) +- **File Systems**: [`azure`](https://docs.slingdata.io/connections/file-connections/azure) [`b2`](https://docs.slingdata.io/connections/file-connections/b2) [`dospaces`](https://docs.slingdata.io/connections/file-connections/dospaces) [`gs`](https://docs.slingdata.io/connections/file-connections/gs) [`local`](https://docs.slingdata.io/connections/file-connections/local) [`minio`](https://docs.slingdata.io/connections/file-connections/minio) [`r2`](https://docs.slingdata.io/connections/file-connections/r2) [`s3`](https://docs.slingdata.io/connections/file-connections/s3) [`sftp`](https://docs.slingdata.io/connections/file-connections/sftp) [`wasabi`](https://docs.slingdata.io/connections/file-connections/wasabi) +- **File Formats**: `csv`, `parquet`, `xlsx`, `json`, `avro`, `xml`, `sas7bday` + +Here are some additional links: +- https://slingdata.io +- https://docs.slingdata.io +- https://blog.slingdata.io + +--- + +Ever wanted to quickly pipe in a CSV or JSON file into your database? Use sling to do so: + +```bash +cat my_file.csv | sling run --tgt-conn MYDB --tgt-object my_schema.my_table +``` + +Or want to copy data between two databases? Do it with sling: +```bash +sling run --src-conn PG_DB --src-stream public.transactions \ + --tgt-conn MYSQL_DB --tgt-object mysql.bank_transactions \ + --mode full-refresh +``` + +Sling can also easily manage our local connections with the `sling conns` command: + +```bash +$ sling conns set MY_PG url='postgresql://postgres:myPassword@pghost:5432/postgres' + +$ sling conns list ++--------------------------+-----------------+-------------------+ +| CONN NAME | CONN TYPE | SOURCE | ++--------------------------+-----------------+-------------------+ +| AWS_S3 | FileSys - S3 | sling env yaml | +| FINANCE_BQ | DB - BigQuery | sling env yaml | +| DO_SPACES | FileSys - S3 | sling env yaml | +| LOCALHOST_DEV | DB - PostgreSQL | dbt profiles yaml | +| MSSQL | DB - SQLServer | sling env yaml | +| MYSQL | DB - MySQL | sling env yaml | +| ORACLE_DB | DB - Oracle | env variable | +| MY_PG | DB - PostgreSQL | sling env yaml | ++--------------------------+-----------------+-------------------+ + +$ sling conns discover LOCALHOST_DEV +9:05AM INF Found 344 streams: + - "public"."accounts" + - "public"."bills" + - "public"."connections" + ... +``` + +## Installation + +#### Brew on Mac + +```shell +brew install slingdata-io/sling/sling + +# You're good to go! +sling -h +``` + +#### Scoop on Windows + +```powershell +scoop bucket add sling https://github.com/slingdata-io/scoop-sling.git +scoop install sling + +# You're good to go! +sling -h +``` + +#### Binary on Linux + +```powershell +curl -LO 'https://github.com/slingdata-io/sling-cli/releases/latest/download/sling_linux_amd64.tar.gz' \ + && tar xf sling_linux_amd64.tar.gz \ + && rm -f sling_linux_amd64.tar.gz \ + && chmod +x sling + +# You're good to go! +sling -h +``` + +### Compiling From Source + +Requirements: +- Install Go 1.22+ (https://go.dev/doc/install) +- Install a C compiler ([gcc](https://www.google.com/search?q=install+gcc&oq=install+gcc), [tdm-gcc](https://jmeubank.github.io/tdm-gcc/), [mingw](https://www.google.com/search?q=install+mingw), etc) + +#### Linux or Mac +```bash +git clone https://github.com/slingdata-io/sling-cli.git +cd sling-cli +bash scripts/build.sh + +./sling --help +``` + +#### Windows (PowerShell) +```bash +git clone https://github.com/slingdata-io/sling-cli.git +cd sling-cli + +.\scripts\build.ps1 + +.\sling --help +``` + +### Automated Dev Builds + +Here are the links of the official development builds, which are the latest builds of the upcoming release. + +- **Linux (x64)**: https://f.slingdata.io/dev/latest/sling_linux_amd64.tar.gz +- **Mac (arm64)**: https://f.slingdata.io/dev/latest/sling_darwin_arm64.tar.gz +- **Windows (x64)**: https://f.slingdata.io/dev/latest/sling_windows_amd64.tar.gz + +### Installing via Python Wrapper + +`pip install sling` + +Then you should be able to run `sling --help` from command line. + + +## Contributing + +We welcome contributions to improve Sling! Here are some guidelines to help you get started. + +### Branch Naming Convention + +When creating a new branch for your contribution, please use the following naming convention: + +- `feature/your-feature-name` for new features +- `bugfix/issue-description` for bug fixes +- `docs/update-description` for documentation updates + +### Testing Guidelines + +Sling has three main test suites: Database, File and CLI. When contributing, please ensure that your changes pass the relevant tests. + +#### Running Tests + +To run the full test suite, run below. However you'd need to define all the needed connections as shown [here](https://github.com/slingdata-io/sling-cli/blob/main/cmd/sling/sling_test.go#L52-L83), so it's recommended to target specific tests instead. + +```sh +./scripts/build.sh +./scripts/test.sh +``` + +#### Targeting Specific Tests + +You can target specific tests or suites using environment variables: + +1. Database Suite: + ```sh + cd cmd/sling + go test -v -run TestSuiteDatabasePostgres # run all Postgres tests + TESTS="1-3" go test -v -run TestSuiteDatabasePostgres # run Postgres tests 1, 2, 3 + ``` + +2. File Suite: + ```sh + cd cmd/sling + go test -v -run TestSuiteFileS3 # run all S3 tests + TESTS="1,2,3" go test -v -run TestSuiteFileS3 # run S3 tests 1, 2, 3 + ``` + +3. CLI Suite: + ```sh + cd cmd/sling + export SLING_BIN=../../sling + go test -v -run TestCLI # run all CLI tests + TESTS="31+" go test -v -run TestCLI # run CLI tests 31 and all subsequent tests + ``` + +You can specify individual test numbers, ranges, or use the '+' suffix to run all tests from a certain number: + +- `TESTS="1,2,3"`: Run tests 1, 2, and 3 +- `TESTS="1-5"`: Run tests 1 through 5 +- `TESTS="3+"`: Run test 3 and all subsequent tests + +#### Test Suites Overview + +1. **Database Suite**: Tests database-related functionality. + - Located in: `cmd/sling/sling_test.go` + - Configuration: `cmd/sling/tests/suite.db.template.tsv` + +2. **File Suite**: Tests file system operations. + - Located in: `cmd/sling/sling_test.go` + - Configuration: `cmd/sling/tests/suite.file.template.tsv` + +3. **CLI Suite**: Tests command-line interface functionality. + - Located in: `cmd/sling/sling_cli_test.go` + - Configuration: `cmd/sling/tests/suite.cli.tsv` + +### Adding New Tests + +When introducing new features or addressing bugs, it's essential to incorporate relevant tests, focusing mainly on the CLI suite file located at `cmd/sling/suite.cli.tsv`. The database and file suites serve as templates applicable across all connectors, making them more sensitive to modifications. Therefore, any changes to these suites will be managed internally. + + When adding new test entries in the CLI suite file, feel free to create a new replication file in folder `cmd/sling/tests/replications`, or a corresponding source file in the `cmd/sling/tests/files` directory. Also include the expected output or the number of expected rows/streams in the new test entry. + +### Pull Request Process + +1. Ensure your code adheres to the existing style and passes all tests. +2. Update the README.md with details of changes to the interface, if applicable. +3. Create a Pull Request with a clear title and description. + +Thank you for contributing to Sling! \ No newline at end of file diff --git a/backend/config/staffusers.json b/backend/config/staffusers.json new file mode 100644 index 0000000000..53f661a722 --- /dev/null +++ b/backend/config/staffusers.json @@ -0,0 +1,20 @@ +{ + "readonly": [ + "philip.dominguez@gsa.gov", + "robert.novak@gsa.gov" + ], + "helpdesk": [ + "analyn.delossantos@gsa.gov", + "hassandeme.mamasambo@gsa.gov", + "iman.ali@gsa.gov", + "james.p.mason@gsa.gov", + "james.person@gsa.gov", + "leigh.cox@gsa.gov", + "laura.herring@gsa.gov", + "rochelle.ribeiro@gsa.gov" + ], + "superuser": [ + "matthew.jadud@gsa.gov", + "dan.swick@gsa.gov" + ] +} diff --git a/backend/dissemination/admin.py b/backend/dissemination/admin.py index cc1e4115a2..82b94f2e3e 100644 --- a/backend/dissemination/admin.py +++ b/backend/dissemination/admin.py @@ -1,5 +1,4 @@ from django.contrib import admin - from dissemination.models import ( AdditionalEin, AdditionalUei, @@ -11,7 +10,9 @@ Note, Passthrough, SecondaryAuditor, + TribalApiAccessKeyIds, ) +import datetime class AdditionalEinAdmin(admin.ModelAdmin): @@ -257,6 +258,29 @@ def has_view_permission(self, request, obj=None): search_fields = ("report_id",) +class TribalApiAccessKeyIdsAdmin(admin.ModelAdmin): + + list_display = ( + "email", + "key_id", + "date_added", + ) + + search_fields = ( + "email", + "key_id", + ) + + fields = [ + "email", + "key_id", + ] + + def save_model(self, request, obj, form, change): + obj.date_added = datetime.date.today() + super().save_model(request, obj, form, change) + + admin.site.register(AdditionalEin, AdditionalEinAdmin) admin.site.register(AdditionalUei, AdditionalUeiAdmin) admin.site.register(CapText, CapTextAdmin) @@ -267,3 +291,4 @@ def has_view_permission(self, request, obj=None): admin.site.register(Note, NoteAdmin) admin.site.register(Passthrough, PassThroughAdmin) admin.site.register(SecondaryAuditor, SecondaryAuditorAdmin) +admin.site.register(TribalApiAccessKeyIds, TribalApiAccessKeyIdsAdmin) diff --git a/backend/dissemination/sql/fac-snapshot-db/create_partition_statements.py b/backend/dissemination/sql/fac-snapshot-db/create_partition_statements.py index d998a8838b..94f01a146e 100644 --- a/backend/dissemination/sql/fac-snapshot-db/create_partition_statements.py +++ b/backend/dissemination/sql/fac-snapshot-db/create_partition_statements.py @@ -44,7 +44,7 @@ def partition_by_sequence_hash(): f""" DROP TABLE IF EXISTS public_data_v1_0_0.part_{table_name}_{ndx:02}; CREATE TABLE public_data_v1_0_0.part_{table_name}_{ndx:02} - PARTITION OF public_data_v1_0_0.{table_name} + PARTITION OF public_data_v1_0_0.{table_name} FOR VALUES WITH (modulus {number_of_partitions}, remainder {ndx}); """ ) @@ -94,7 +94,7 @@ def partition_by_audit_year(): f""" DROP TABLE IF EXISTS public_data_v1_0_0.part_{table_name}_20{ndx:02}; CREATE TABLE public_data_v1_0_0.part_{table_name}_20{ndx:02} - PARTITION OF public_data_v1_0_0.{table_name} + PARTITION OF public_data_v1_0_0.{table_name} FOR VALUES IN ('20{ndx:02}'); """ ) diff --git a/backend/dissemination/sql/test_api.py b/backend/dissemination/sql/test_api.py index 1c29fa88d0..e43a713359 100644 --- a/backend/dissemination/sql/test_api.py +++ b/backend/dissemination/sql/test_api.py @@ -3,7 +3,6 @@ # pytest -s --env local test_api.py # -import click import os import requests import sys @@ -37,10 +36,10 @@ def url(env): def headers(env): if env in ["local"]: - if EnvVars.FAC_AUTH_BEARER == None: + if EnvVars.FAC_AUTH_BEARER is None: print("FAC_AUTH_BEARER not set.") sys.exit() - if EnvVars.FAC_API_KEY_ID == None: + if EnvVars.FAC_API_KEY_ID is None: print("FAC_API_KEY_ID not set.") sys.exit() return { @@ -48,7 +47,7 @@ def headers(env): "x-api-user-id": EnvVars.FAC_API_KEY_ID, } elif env in ["preview", "dev", "staging", "production"]: - if EnvVars.FAC_API_KEY == None: + if EnvVars.FAC_API_KEY is None: print("FAC_API_KEY not set.") sys.exit() return { diff --git a/backend/run.sh b/backend/run.sh index 57de75725e..148fa6f50d 100755 --- a/backend/run.sh +++ b/backend/run.sh @@ -64,6 +64,11 @@ gonogo "sling_bulk_export" seed_cog_baseline gonogo "seed_cog_baseline" +##### +# CREATE STAFF USERS +# Prepares staff users for Django admin +python manage.py create_staffusers + ##### # LAUNCH THE APP # We will have died long ago if things didn't work. diff --git a/backend/tools/setup_cgov_env.py b/backend/tools/setup_cgov_env.py index 4e26cb3597..846a622c9f 100644 --- a/backend/tools/setup_cgov_env.py +++ b/backend/tools/setup_cgov_env.py @@ -1,5 +1,4 @@ import json -import os from pprint import pprint diff --git a/backend/users/admin.py b/backend/users/admin.py index 64261d48eb..e126e43597 100644 --- a/backend/users/admin.py +++ b/backend/users/admin.py @@ -17,20 +17,34 @@ class PermissionAdmin(admin.ModelAdmin): @admin.register(User) class UserAdmin(admin.ModelAdmin): - list_display = ["email", "can_read_tribal", "last_login", "date_joined"] - list_filter = ["is_staff", "is_superuser"] - exclude = ["groups", "user_permissions", "password"] + list_display = [ + "email", + "can_read_tribal", + "last_login", + "date_joined", + "assigned_groups", + ] + list_filter = ["is_staff", "is_superuser", "groups"] + exclude = ["user_permissions", "password"] readonly_fields = ["date_joined", "last_login"] search_fields = ("email", "username") def can_read_tribal(self, obj): return _can_read_tribal(obj) + def assigned_groups(self, obj): + return ", ".join([g.name for g in obj.groups.all()]) + @admin.register(UserPermission) class UserPermissionAdmin(admin.ModelAdmin): list_display = ["user", "email", "permission"] search_fields = ("email", "permission", "user") + fields = ["email", "permission"] + + def save_model(self, request, obj, form, change): + obj.user = User.objects.get(email=obj.email) + super().save_model(request, obj, form, change) @admin.register(StaffUserLog) @@ -57,8 +71,7 @@ def has_delete_permission(self, request, obj=None): class StaffUserAdmin(admin.ModelAdmin): list_display = [ "staff_email", - "added_by_email", - "date_added", + "privilege", ] fields = [ "staff_email", @@ -91,3 +104,9 @@ def has_add_permission(self, request, obj=None): def has_delete_permission(self, request, obj=None): return request.user.is_superuser + + def privilege(self, obj): + user = User.objects.get(email=obj.staff_email, is_staff=True) + if user.is_superuser: + return "Superuser" + return ", ".join([g.name for g in user.groups.all()]) diff --git a/backend/users/management/commands/create_staffusers.py b/backend/users/management/commands/create_staffusers.py new file mode 100644 index 0000000000..6c44212385 --- /dev/null +++ b/backend/users/management/commands/create_staffusers.py @@ -0,0 +1,115 @@ +from django.conf import settings +from django.contrib.auth import get_user_model +from django.contrib.auth.models import Group, Permission +from django.core.management.base import BaseCommand +from django.db import transaction +from users.models import StaffUser +import json +import logging +import os + +logger = logging.getLogger(__name__) +User = get_user_model() + + +class Command(BaseCommand): + + def handle(self, *args, **kwargs): + """Create a group with readonly permissions.""" + group_readonly, created = Group.objects.get_or_create(name="Read-only") + readonly_codenames = [ + "view_access", + "view_deletedaccess", + "view_singleauditchecklist", + "view_sacvalidationwaiver", + "view_ueivalidationwaiver", + "view_additionalein", + "view_additionaluei", + "view_captext", + "view_federalaward", + "view_findingtext", + "view_finding", + "view_general", + "view_note", + "view_passthrough", + "view_secondaryauditor", + "view_cognizantassignment", + "view_cognizantbaseline", + "view_staffuser", + "view_userpermission", + "view_tribalapiaccesskeyids", + ] + group_readonly.permissions.clear() + for code in readonly_codenames: + group_readonly.permissions.add(Permission.objects.get(codename=code)) + group_readonly.save() + + """Create a group with helpdesk permissions.""" + group_helpdesk, created = Group.objects.get_or_create(name="Helpdesk") + helpdesk_codenames = readonly_codenames + [ + "add_userpermission", + "change_userpermission", + "delete_userpermission", + "add_tribalapiaccesskeyids", + "change_tribalapiaccesskeyids", + "delete_tribalapiaccesskeyids", + "add_sacvalidationwaiver", + "add_ueivalidationwaiver", + "add_cognizantassignment", + ] + group_helpdesk.permissions.clear() + for code in helpdesk_codenames: + group_helpdesk.permissions.add(Permission.objects.get(codename=code)) + group_helpdesk.save() + + # read in staffusers JSON. + user_list = None + with open( + os.path.join(settings.BASE_DIR, "config/staffusers.json"), "r" + ) as file: + user_list = json.load(file) + + if user_list: + + # clear superuser privileges. + superusers = User.objects.filter(is_superuser=True) + for superuser in superusers: + superuser.is_superuser = False + superuser.save() + + # clear existing staff users. + StaffUser.objects.all().delete() + + for role in user_list: + for email in user_list[role]: + + # create staff user for each role. + with transaction.atomic(): + + StaffUser( + staff_email=email, + ).save() + + # attempt to update the user. + try: + user = User.objects.get(email=email, is_staff=True) + + user.groups.clear() + match role: + case "readonly": + user.groups.add(group_readonly) + case "helpdesk": + user.groups.clear() + user.groups.add(group_helpdesk) + case "superuser": + user.is_superuser = True + + user.save() + logger.info(f"Synced {email} to a StaffUser role.") + + # for whatever reason, this failed. Revert staffuser creation. + except Exception as e: + transaction.set_rollback(True) + logger.warning( + f"Failed to create a StaffUser for {email} ({e})." + ) diff --git a/backend/users/models.py b/backend/users/models.py index de502a92db..e0c0534b1f 100644 --- a/backend/users/models.py +++ b/backend/users/models.py @@ -70,6 +70,7 @@ def delete(self, *args, **kwargs): try: user = User.objects.get(email=self.staff_email) user.is_staff = False + user.is_superuser = False user.save() except User.DoesNotExist: pass # silently ignore. Nothing to do. From 0ff0629fb6bc5438ccbe2c1d329da63bc1ffeb32 Mon Sep 17 00:00:00 2001 From: Bobby Novak <176936850+rnovak338@users.noreply.github.com> Date: Fri, 25 Oct 2024 15:27:43 -0400 Subject: [PATCH 46/89] Remove Sling README --- backend/README.md | 270 ---------------------------------------------- 1 file changed, 270 deletions(-) delete mode 100644 backend/README.md diff --git a/backend/README.md b/backend/README.md deleted file mode 100644 index 30832a2381..0000000000 --- a/backend/README.md +++ /dev/null @@ -1,270 +0,0 @@ - -

drawing

- -

Slings from a data source to a data target.

-

See docs.slingdata.io for more details.

- - -

- - GitHub - - - - - - Go Reference - - - Discord - - - GitHub tag (latest SemVer pre-release) - - - Pip Downloads - - - Pip Downloads - -

- -Sling is a passion project turned into a free CLI Product which offers an easy solution to create and maintain small to medium volume data pipelines using the Extract & Load (EL) approach. It focuses on data movement between: - -* Database to Database -* File System to Database -* Database to File System - - -https://github.com/slingdata-io/sling-cli/assets/7671010/e10ee716-1de8-4d53-8eb2-95c6d9d7f9f0 - -Some key features: -- Single Binary deployment (built with Go). See [installation](https://docs.slingdata.io/sling-cli/getting-started) page. -- Use Custom SQL as a stream: `--src-stream='select * from my_table where col1 > 10'` -- Manage / View / Test / Discover your connections with the [`sling conns`](https://docs.slingdata.io/sling-cli/environment#managing-connections) sub-command -- Use Environment Variable as connections if you prefer (`export MY_PG='postgres//...`)' -- Provide YAML or JSON configurations (perfect for git version control). -- Powerful [Replication](https://docs.slingdata.io/sling-cli/run/configuration/replication) logic, to replication many tables with a wildcard (`my_schema.*`). -- Reads your existing [DBT connections](https://docs.slingdata.io/sling-cli/environment#dbt-profiles-dbt-profiles.yml) -- Use your environment variable in your YAML / JSON config (`select * from my_table where date = '{date}'`) -- Convenient [Transformations](https://docs.slingdata.io/sling-cli/run/configuration/transformations), such as the `flatten` option, which auto-creates columns from your nested fields. -- Run Pre & Post SQL commands. -- many more! - ---- - -Example [Replication](https://docs.slingdata.io/sling-cli/run/configuration/replication): - -

replication.yaml

- ---- - -Available Connectors: -- **Databases**: [`bigquery`](https://docs.slingdata.io/connections/database-connections/bigquery) [`bigtable`](https://docs.slingdata.io/connections/database-connections/bigtable) [`clickhouse`](https://docs.slingdata.io/connections/database-connections/clickhouse) [`duckdb`](https://docs.slingdata.io/connections/database-connections/duckdb) [`mariadb`](https://docs.slingdata.io/connections/database-connections/mariadb) [`motherduck`](https://docs.slingdata.io/connections/database-connections/motherduck) [`mysql`](https://docs.slingdata.io/connections/database-connections/mysql) [`oracle`](https://docs.slingdata.io/connections/database-connections/oracle) [`postgres`](https://docs.slingdata.io/connections/database-connections/postgres) [`redshift`](https://docs.slingdata.io/connections/database-connections/redshift) [`snowflake`](https://docs.slingdata.io/connections/database-connections/snowflake) [`sqlite`](https://docs.slingdata.io/connections/database-connections/sqlite) [`sqlserver`](https://docs.slingdata.io/connections/database-connections/sqlserver) [`starrocks`](https://docs.slingdata.io/connections/database-connections/starrocks) [`prometheus`](https://docs.slingdata.io/connections/database-connections/prometheus) [`proton`](https://docs.slingdata.io/connections/database-connections/proton) -- **File Systems**: [`azure`](https://docs.slingdata.io/connections/file-connections/azure) [`b2`](https://docs.slingdata.io/connections/file-connections/b2) [`dospaces`](https://docs.slingdata.io/connections/file-connections/dospaces) [`gs`](https://docs.slingdata.io/connections/file-connections/gs) [`local`](https://docs.slingdata.io/connections/file-connections/local) [`minio`](https://docs.slingdata.io/connections/file-connections/minio) [`r2`](https://docs.slingdata.io/connections/file-connections/r2) [`s3`](https://docs.slingdata.io/connections/file-connections/s3) [`sftp`](https://docs.slingdata.io/connections/file-connections/sftp) [`wasabi`](https://docs.slingdata.io/connections/file-connections/wasabi) -- **File Formats**: `csv`, `parquet`, `xlsx`, `json`, `avro`, `xml`, `sas7bday` - -Here are some additional links: -- https://slingdata.io -- https://docs.slingdata.io -- https://blog.slingdata.io - ---- - -Ever wanted to quickly pipe in a CSV or JSON file into your database? Use sling to do so: - -```bash -cat my_file.csv | sling run --tgt-conn MYDB --tgt-object my_schema.my_table -``` - -Or want to copy data between two databases? Do it with sling: -```bash -sling run --src-conn PG_DB --src-stream public.transactions \ - --tgt-conn MYSQL_DB --tgt-object mysql.bank_transactions \ - --mode full-refresh -``` - -Sling can also easily manage our local connections with the `sling conns` command: - -```bash -$ sling conns set MY_PG url='postgresql://postgres:myPassword@pghost:5432/postgres' - -$ sling conns list -+--------------------------+-----------------+-------------------+ -| CONN NAME | CONN TYPE | SOURCE | -+--------------------------+-----------------+-------------------+ -| AWS_S3 | FileSys - S3 | sling env yaml | -| FINANCE_BQ | DB - BigQuery | sling env yaml | -| DO_SPACES | FileSys - S3 | sling env yaml | -| LOCALHOST_DEV | DB - PostgreSQL | dbt profiles yaml | -| MSSQL | DB - SQLServer | sling env yaml | -| MYSQL | DB - MySQL | sling env yaml | -| ORACLE_DB | DB - Oracle | env variable | -| MY_PG | DB - PostgreSQL | sling env yaml | -+--------------------------+-----------------+-------------------+ - -$ sling conns discover LOCALHOST_DEV -9:05AM INF Found 344 streams: - - "public"."accounts" - - "public"."bills" - - "public"."connections" - ... -``` - -## Installation - -#### Brew on Mac - -```shell -brew install slingdata-io/sling/sling - -# You're good to go! -sling -h -``` - -#### Scoop on Windows - -```powershell -scoop bucket add sling https://github.com/slingdata-io/scoop-sling.git -scoop install sling - -# You're good to go! -sling -h -``` - -#### Binary on Linux - -```powershell -curl -LO 'https://github.com/slingdata-io/sling-cli/releases/latest/download/sling_linux_amd64.tar.gz' \ - && tar xf sling_linux_amd64.tar.gz \ - && rm -f sling_linux_amd64.tar.gz \ - && chmod +x sling - -# You're good to go! -sling -h -``` - -### Compiling From Source - -Requirements: -- Install Go 1.22+ (https://go.dev/doc/install) -- Install a C compiler ([gcc](https://www.google.com/search?q=install+gcc&oq=install+gcc), [tdm-gcc](https://jmeubank.github.io/tdm-gcc/), [mingw](https://www.google.com/search?q=install+mingw), etc) - -#### Linux or Mac -```bash -git clone https://github.com/slingdata-io/sling-cli.git -cd sling-cli -bash scripts/build.sh - -./sling --help -``` - -#### Windows (PowerShell) -```bash -git clone https://github.com/slingdata-io/sling-cli.git -cd sling-cli - -.\scripts\build.ps1 - -.\sling --help -``` - -### Automated Dev Builds - -Here are the links of the official development builds, which are the latest builds of the upcoming release. - -- **Linux (x64)**: https://f.slingdata.io/dev/latest/sling_linux_amd64.tar.gz -- **Mac (arm64)**: https://f.slingdata.io/dev/latest/sling_darwin_arm64.tar.gz -- **Windows (x64)**: https://f.slingdata.io/dev/latest/sling_windows_amd64.tar.gz - -### Installing via Python Wrapper - -`pip install sling` - -Then you should be able to run `sling --help` from command line. - - -## Contributing - -We welcome contributions to improve Sling! Here are some guidelines to help you get started. - -### Branch Naming Convention - -When creating a new branch for your contribution, please use the following naming convention: - -- `feature/your-feature-name` for new features -- `bugfix/issue-description` for bug fixes -- `docs/update-description` for documentation updates - -### Testing Guidelines - -Sling has three main test suites: Database, File and CLI. When contributing, please ensure that your changes pass the relevant tests. - -#### Running Tests - -To run the full test suite, run below. However you'd need to define all the needed connections as shown [here](https://github.com/slingdata-io/sling-cli/blob/main/cmd/sling/sling_test.go#L52-L83), so it's recommended to target specific tests instead. - -```sh -./scripts/build.sh -./scripts/test.sh -``` - -#### Targeting Specific Tests - -You can target specific tests or suites using environment variables: - -1. Database Suite: - ```sh - cd cmd/sling - go test -v -run TestSuiteDatabasePostgres # run all Postgres tests - TESTS="1-3" go test -v -run TestSuiteDatabasePostgres # run Postgres tests 1, 2, 3 - ``` - -2. File Suite: - ```sh - cd cmd/sling - go test -v -run TestSuiteFileS3 # run all S3 tests - TESTS="1,2,3" go test -v -run TestSuiteFileS3 # run S3 tests 1, 2, 3 - ``` - -3. CLI Suite: - ```sh - cd cmd/sling - export SLING_BIN=../../sling - go test -v -run TestCLI # run all CLI tests - TESTS="31+" go test -v -run TestCLI # run CLI tests 31 and all subsequent tests - ``` - -You can specify individual test numbers, ranges, or use the '+' suffix to run all tests from a certain number: - -- `TESTS="1,2,3"`: Run tests 1, 2, and 3 -- `TESTS="1-5"`: Run tests 1 through 5 -- `TESTS="3+"`: Run test 3 and all subsequent tests - -#### Test Suites Overview - -1. **Database Suite**: Tests database-related functionality. - - Located in: `cmd/sling/sling_test.go` - - Configuration: `cmd/sling/tests/suite.db.template.tsv` - -2. **File Suite**: Tests file system operations. - - Located in: `cmd/sling/sling_test.go` - - Configuration: `cmd/sling/tests/suite.file.template.tsv` - -3. **CLI Suite**: Tests command-line interface functionality. - - Located in: `cmd/sling/sling_cli_test.go` - - Configuration: `cmd/sling/tests/suite.cli.tsv` - -### Adding New Tests - -When introducing new features or addressing bugs, it's essential to incorporate relevant tests, focusing mainly on the CLI suite file located at `cmd/sling/suite.cli.tsv`. The database and file suites serve as templates applicable across all connectors, making them more sensitive to modifications. Therefore, any changes to these suites will be managed internally. - - When adding new test entries in the CLI suite file, feel free to create a new replication file in folder `cmd/sling/tests/replications`, or a corresponding source file in the `cmd/sling/tests/files` directory. Also include the expected output or the number of expected rows/streams in the new test entry. - -### Pull Request Process - -1. Ensure your code adheres to the existing style and passes all tests. -2. Update the README.md with details of changes to the interface, if applicable. -3. Create a Pull Request with a clear title and description. - -Thank you for contributing to Sling! \ No newline at end of file From 551f85e070085806fc4e1aa5405d3850e6b816d6 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Fri, 25 Oct 2024 16:07:25 -0400 Subject: [PATCH 47/89] Moves general back into the public_100 tables. The move out was part of debugging, and unnecessary. This reduces the number of files, and keeps like-with-like. Permissions simplification/removal; picked up elsewhere/later in the sequence. (And, the alter on sequences may not have been necessary at all.) --- ....sql => 020_public_data_v1_0_0_tables.sql} | 116 ++++++++++++++++ .../post/020a_public_data_v1_0_0_general.sql | 129 ------------------ .../post/120_api_v1_1_0_gate.sql | 2 +- ...80_permissions.sql => 800_permissions.sql} | 8 ++ .../fac-snapshot-db/pre/080_permissions.sql | 15 -- 5 files changed, 125 insertions(+), 145 deletions(-) rename backend/dissemination/sql/fac-snapshot-db/post/{020b_public_data_v1_0_0_tables.sql => 020_public_data_v1_0_0_tables.sql} (85%) delete mode 100644 backend/dissemination/sql/fac-snapshot-db/post/020a_public_data_v1_0_0_general.sql rename backend/dissemination/sql/fac-snapshot-db/post/{080_permissions.sql => 800_permissions.sql} (57%) delete mode 100644 backend/dissemination/sql/fac-snapshot-db/pre/080_permissions.sql diff --git a/backend/dissemination/sql/fac-snapshot-db/post/020b_public_data_v1_0_0_tables.sql b/backend/dissemination/sql/fac-snapshot-db/post/020_public_data_v1_0_0_tables.sql similarity index 85% rename from backend/dissemination/sql/fac-snapshot-db/post/020b_public_data_v1_0_0_tables.sql rename to backend/dissemination/sql/fac-snapshot-db/post/020_public_data_v1_0_0_tables.sql index a4eafb3bc3..93d9915936 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/020b_public_data_v1_0_0_tables.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/020_public_data_v1_0_0_tables.sql @@ -1,3 +1,112 @@ +----------------------------------------------------------- +-- general +----------------------------------------------------------- +-- We do general first because all other tables are built off of it. +CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_general() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE public_data_v1_0_0.general AS + SELECT + gen.id as id, + NEXTVAL('public_data_v1_0_0.seq_general') AS seq, + gen.report_id, + gen.auditee_uei, + gen.audit_year, + gen.agencies_with_prior_findings, + gen.audit_period_covered, + gen.audit_type, + gen.auditee_address_line_1, + gen.auditee_certified_date, + gen.auditee_certify_name, + gen.auditee_certify_title, + gen.auditee_city, + gen.auditee_contact_name, + gen.auditee_contact_title, + gen.auditee_ein, + gen.auditee_email, + gen.auditee_name, + gen.auditee_phone, + gen.auditee_state, + gen.auditee_zip, + gen.auditor_address_line_1, + gen.auditor_certified_date, + gen.auditor_certify_name, + gen.auditor_certify_title, + gen.auditor_city, + gen.auditor_contact_name, + gen.auditor_contact_title, + gen.auditor_country, + gen.auditor_ein, + gen.auditor_email, + gen.auditor_firm_name, + gen.auditor_foreign_address, + gen.auditor_phone, + gen.auditor_state, + gen.auditor_zip, + gen.cognizant_agency, + gen.data_source, + gen.date_created, + gen.dollar_threshold, + gen.entity_type, + gen.fac_accepted_date, + gen.fy_end_date, + gen.fy_start_date, + gen.gaap_results, + gen.is_additional_ueis, + gen.is_aicpa_audit_guide_included, + gen.is_going_concern_included, + gen.is_internal_control_deficiency_disclosed, + gen.is_internal_control_material_weakness_disclosed, + gen.is_low_risk_auditee, + gen.is_material_noncompliance_disclosed, + CASE EXISTS + ( + SELECT + ein.report_id + FROM + dissemination_additionalein ein + WHERE + ein.report_id = gen.report_id + ) + WHEN FALSE THEN 'No' + ELSE 'Yes' + END AS is_multiple_eins, + gen.is_public, + CASE EXISTS + ( + SELECT + aud.report_id + FROM + dissemination_secondaryauditor aud + WHERE + aud.report_id = gen.report_id + ) + WHEN FALSE THEN 'No' + ELSE 'Yes' + END AS is_secondary_auditors, + gen.is_sp_framework_required, + gen.number_months, + gen.oversight_agency, + gen.ready_for_certification_date, + gen.sp_framework_basis, + gen.sp_framework_opinions, + gen.submitted_date, + gen.total_amount_expended, + gen.type_audit_code + FROM + dissem_copy.dissemination_general gen + ORDER BY gen.id; + + -- Add a clean batch number after the table is created. + ALTER TABLE public_data_v1_0_0.general + ADD COLUMN batch_number INTEGER; + UPDATE public_data_v1_0_0.general SET batch_number=DIV(seq, public.batch_size()); + END + $ct$ + LANGUAGE plpgsql; + ----------------------------------------------------------- -- addition_eins @@ -487,6 +596,11 @@ CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_combined() ORDER BY seq ; + -- For advanced search, Django wants an `id` column. + ALTER TABLE public_data_v1_0_0.combined + ADD COLUMN id INTEGER; + UPDATE public_data_v1_0_0.combined SET id=seq; + -- Add a clean batch number after the table is created. ALTER TABLE public_data_v1_0_0.combined ADD COLUMN batch_number INTEGER; @@ -605,6 +719,8 @@ $GATE$ RAISE info '% Gate condition met. Skipping table creation.', api_ver; ELSE RAISE info '% %.% not found. Creating tables', api_ver, the_schema, the_table; + RAISE info 'Creating general'; + PERFORM public_data_v1_0_0_functions.create_general(); RAISE info 'Creating additional_eins'; PERFORM public_data_v1_0_0_functions.create_additional_eins(); RAISE info 'Creating additional_ueis'; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/020a_public_data_v1_0_0_general.sql b/backend/dissemination/sql/fac-snapshot-db/post/020a_public_data_v1_0_0_general.sql deleted file mode 100644 index 1a9a5ebb23..0000000000 --- a/backend/dissemination/sql/fac-snapshot-db/post/020a_public_data_v1_0_0_general.sql +++ /dev/null @@ -1,129 +0,0 @@ ------------------------------------------------------------ --- general ------------------------------------------------------------ -CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_general() - RETURNS VOID - AS - $ct$ - BEGIN - CREATE TABLE public_data_v1_0_0.general AS - SELECT - gen.id as id, - NEXTVAL('public_data_v1_0_0.seq_general') AS seq, - gen.report_id, - gen.auditee_uei, - gen.audit_year, - gen.agencies_with_prior_findings, - gen.audit_period_covered, - gen.audit_type, - gen.auditee_address_line_1, - gen.auditee_certified_date, - gen.auditee_certify_name, - gen.auditee_certify_title, - gen.auditee_city, - gen.auditee_contact_name, - gen.auditee_contact_title, - gen.auditee_ein, - gen.auditee_email, - gen.auditee_name, - gen.auditee_phone, - gen.auditee_state, - gen.auditee_zip, - gen.auditor_address_line_1, - gen.auditor_certified_date, - gen.auditor_certify_name, - gen.auditor_certify_title, - gen.auditor_city, - gen.auditor_contact_name, - gen.auditor_contact_title, - gen.auditor_country, - gen.auditor_ein, - gen.auditor_email, - gen.auditor_firm_name, - gen.auditor_foreign_address, - gen.auditor_phone, - gen.auditor_state, - gen.auditor_zip, - gen.cognizant_agency, - gen.data_source, - gen.date_created, - gen.dollar_threshold, - gen.entity_type, - gen.fac_accepted_date, - gen.fy_end_date, - gen.fy_start_date, - gen.gaap_results, - gen.is_additional_ueis, - gen.is_aicpa_audit_guide_included, - gen.is_going_concern_included, - gen.is_internal_control_deficiency_disclosed, - gen.is_internal_control_material_weakness_disclosed, - gen.is_low_risk_auditee, - gen.is_material_noncompliance_disclosed, - CASE EXISTS - ( - SELECT - ein.report_id - FROM - dissemination_additionalein ein - WHERE - ein.report_id = gen.report_id - ) - WHEN FALSE THEN 'No' - ELSE 'Yes' - END AS is_multiple_eins, - gen.is_public, - CASE EXISTS - ( - SELECT - aud.report_id - FROM - dissemination_secondaryauditor aud - WHERE - aud.report_id = gen.report_id - ) - WHEN FALSE THEN 'No' - ELSE 'Yes' - END AS is_secondary_auditors, - gen.is_sp_framework_required, - gen.number_months, - gen.oversight_agency, - gen.ready_for_certification_date, - gen.sp_framework_basis, - gen.sp_framework_opinions, - gen.submitted_date, - gen.total_amount_expended, - gen.type_audit_code - FROM - dissem_copy.dissemination_general gen - ORDER BY gen.id; - - -- Add a clean batch number after the table is created. - ALTER TABLE public_data_v1_0_0.general - ADD COLUMN batch_number INTEGER; - UPDATE public_data_v1_0_0.general SET batch_number=DIV(seq, public.batch_size()); - END - $ct$ - LANGUAGE plpgsql; - -DO LANGUAGE plpgsql -$GATE$ - DECLARE - the_schema varchar := 'public_data_v1_0_0'; - the_table varchar := 'metadata'; - api_ver varchar := 'api_v2_0_0'; - BEGIN - IF EXISTS ( - SELECT FROM pg_tables - WHERE schemaname = the_schema - AND tablename = the_table - ) - THEN - RAISE info '% Gate condition met. Skipping table creation.', api_ver; - ELSE - RAISE info '% %.% not found. Creating tables', api_ver, the_schema, the_table; - RAISE info 'Creating general'; - PERFORM public_data_v1_0_0_functions.create_general(); - END IF; - END -$GATE$; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/120_api_v1_1_0_gate.sql b/backend/dissemination/sql/fac-snapshot-db/post/120_api_v1_1_0_gate.sql index 8cc1878ce8..1ed472fb34 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/120_api_v1_1_0_gate.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/120_api_v1_1_0_gate.sql @@ -10,7 +10,7 @@ DO LANGUAGE plpgsql $GATE$ DECLARE - the_schema varchar := 'public'; + the_schema varchar := 'dissem_copy'; the_table varchar := 'dissemination_general'; api_ver varchar := 'api_v1_1_0'; BEGIN diff --git a/backend/dissemination/sql/fac-snapshot-db/post/080_permissions.sql b/backend/dissemination/sql/fac-snapshot-db/post/800_permissions.sql similarity index 57% rename from backend/dissemination/sql/fac-snapshot-db/post/080_permissions.sql rename to backend/dissemination/sql/fac-snapshot-db/post/800_permissions.sql index 1da5de0b8e..0a8fad276d 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/080_permissions.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/800_permissions.sql @@ -5,3 +5,11 @@ -- GRANT SELECT ON ALL TABLES IN SCHEMA api_v1_0_3 TO api_fac_gov; GRANT SELECT ON ALL TABLES IN SCHEMA api_v1_1_0 TO api_fac_gov; GRANT SELECT ON ALL TABLES IN SCHEMA api_v2_0_0 TO api_fac_gov; + +-- GRANT SELECT ON SEQUENCES IN SCHEMA public_data_v1_0_0; +-- -- SEQ: USAGE | SELECT | UPDATE +-- ALTER DEFAULT PRIVILEGES +-- IN SCHEMA public_data_v1_0_0 +-- GRANT SELECT, USAGE +-- ON SEQUENCES +-- TO api_fac_gov; diff --git a/backend/dissemination/sql/fac-snapshot-db/pre/080_permissions.sql b/backend/dissemination/sql/fac-snapshot-db/pre/080_permissions.sql deleted file mode 100644 index c3bd1249a8..0000000000 --- a/backend/dissemination/sql/fac-snapshot-db/pre/080_permissions.sql +++ /dev/null @@ -1,15 +0,0 @@ - - --- TABLES: SELECT | INSERT | UPDATE | DELETE | TRUNCATE | REFERENCES | TRIGGER | MAINTAIN -ALTER DEFAULT PRIVILEGES - IN SCHEMA public_data_v1_0_0 - GRANT SELECT - ON TABLES - TO api_fac_gov; - --- SEQ: USAGE | SELECT | UPDATE -ALTER DEFAULT PRIVILEGES - IN SCHEMA public_data_v1_0_0 - GRANT SELECT, USAGE - ON SEQUENCES - TO api_fac_gov; From affe2b7b7f191e63a35075109a001128611e9c36 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Fri, 25 Oct 2024 16:12:10 -0400 Subject: [PATCH 48/89] Simplified run.sh Removes an unnecessary step. --- backend/run.sh | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/backend/run.sh b/backend/run.sh index 148fa6f50d..483df37c89 100755 --- a/backend/run.sh +++ b/backend/run.sh @@ -51,13 +51,6 @@ gonogo "migrate_app_tables" sql_post gonogo "sql_post" -##### -# BULK EXPORT -# Creates CSV exports of all of the public data, -# placing it in the fac-private-s3 bucket. -sling_bulk_export -gonogo "sling_bulk_export" - ##### # SEED COG/OVER TABLES # Setup tables for cog/over assignments @@ -73,3 +66,4 @@ python manage.py create_staffusers # LAUNCH THE APP # We will have died long ago if things didn't work. npm run dev & python manage.py runserver 0.0.0.0:8000 + From 3aa6266d83d122468434db5dccf5c9ca4abd7380 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Fri, 25 Oct 2024 16:12:43 -0400 Subject: [PATCH 49/89] Updated the sling script for bulk data This is a proof-of-concept included with the work. It's in the tree, and I've taken it out of active use/an active part of startup. However, we might consider working it into the nightly, to start testing it. --- .../bulk_data_export/create_stream_yaml.py | 15 +- .../public_data_v1_0_0_single_csv.yaml | 664 +++++++++++++----- 2 files changed, 513 insertions(+), 166 deletions(-) diff --git a/backend/dissemination/sql/sling/bulk_data_export/create_stream_yaml.py b/backend/dissemination/sql/sling/bulk_data_export/create_stream_yaml.py index 7e2c6d0a2b..a8dfe0215c 100644 --- a/backend/dissemination/sql/sling/bulk_data_export/create_stream_yaml.py +++ b/backend/dissemination/sql/sling/bulk_data_export/create_stream_yaml.py @@ -6,6 +6,7 @@ import yaml from pprint import pprint +from datetime import datetime obj = {"streams": dict()} @@ -35,6 +36,10 @@ "secondary_auditors", ] +# Although this attempts to generate files all the way out to 2030, +# it will not generate anything where data does not exist. +# This future-proofs us for a year or two, so we don't have to worry +# about updating the `sling` script that is created. years = range(2016, 2031) @@ -42,10 +47,18 @@ ndx = 0 for y in years: obj["streams"][f"{schema}.{t}.{ndx}"] = { - "object": f"bulk_export/{{MM}}/{t}_{y}.csv", + "object": f"bulk_export/{{MM}}/{y}_{t}.csv", "sql": f"SELECT * FROM {schema}.{t} WHERE audit_year = '{y}'", "mode": "full-refresh", + "target_options": { + "format": "csv", + }, } ndx += 1 +today = datetime.today().strftime("%Y-%m-%d") +print("# DO NOT EDIT; THIS IS A GENERATED FILE") +print("# python create_stream_yaml.py > public_data_v1_0_0_single_csv.yaml") +print(f"# Last generated {today}") +print() print(yaml.dump(obj)) diff --git a/backend/dissemination/sql/sling/bulk_data_export/public_data_v1_0_0_single_csv.yaml b/backend/dissemination/sql/sling/bulk_data_export/public_data_v1_0_0_single_csv.yaml index 2e85bfc856..0f4986797a 100644 --- a/backend/dissemination/sql/sling/bulk_data_export/public_data_v1_0_0_single_csv.yaml +++ b/backend/dissemination/sql/sling/bulk_data_export/public_data_v1_0_0_single_csv.yaml @@ -1,3 +1,7 @@ +# DO NOT EDIT; THIS IS A GENERATED FILE +# python create_stream_yaml.py > public_data_v1_0_0_single_csv.yaml +# Last generated 2024-10-25 + defaults: target_options: compression: gzip @@ -7,678 +11,1008 @@ source: FAC_SNAPSHOT_URI streams: public_data_v1_0_0.additional_eins.0: mode: full-refresh - object: bulk_export/{MM}/additional_eins_2016.csv + object: bulk_export/{MM}/2016_additional_eins.csv sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2016' + target_options: + format: csv public_data_v1_0_0.additional_eins.1: mode: full-refresh - object: bulk_export/{MM}/additional_eins_2017.csv + object: bulk_export/{MM}/2017_additional_eins.csv sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2017' + target_options: + format: csv public_data_v1_0_0.additional_eins.10: mode: full-refresh - object: bulk_export/{MM}/additional_eins_2026.csv + object: bulk_export/{MM}/2026_additional_eins.csv sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2026' + target_options: + format: csv public_data_v1_0_0.additional_eins.11: mode: full-refresh - object: bulk_export/{MM}/additional_eins_2027.csv + object: bulk_export/{MM}/2027_additional_eins.csv sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2027' + target_options: + format: csv public_data_v1_0_0.additional_eins.12: mode: full-refresh - object: bulk_export/{MM}/additional_eins_2028.csv + object: bulk_export/{MM}/2028_additional_eins.csv sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2028' + target_options: + format: csv public_data_v1_0_0.additional_eins.13: mode: full-refresh - object: bulk_export/{MM}/additional_eins_2029.csv + object: bulk_export/{MM}/2029_additional_eins.csv sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2029' + target_options: + format: csv public_data_v1_0_0.additional_eins.14: mode: full-refresh - object: bulk_export/{MM}/additional_eins_2030.csv + object: bulk_export/{MM}/2030_additional_eins.csv sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2030' + target_options: + format: csv public_data_v1_0_0.additional_eins.2: mode: full-refresh - object: bulk_export/{MM}/additional_eins_2018.csv + object: bulk_export/{MM}/2018_additional_eins.csv sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2018' + target_options: + format: csv public_data_v1_0_0.additional_eins.3: mode: full-refresh - object: bulk_export/{MM}/additional_eins_2019.csv + object: bulk_export/{MM}/2019_additional_eins.csv sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2019' + target_options: + format: csv public_data_v1_0_0.additional_eins.4: mode: full-refresh - object: bulk_export/{MM}/additional_eins_2020.csv + object: bulk_export/{MM}/2020_additional_eins.csv sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2020' + target_options: + format: csv public_data_v1_0_0.additional_eins.5: mode: full-refresh - object: bulk_export/{MM}/additional_eins_2021.csv + object: bulk_export/{MM}/2021_additional_eins.csv sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2021' + target_options: + format: csv public_data_v1_0_0.additional_eins.6: mode: full-refresh - object: bulk_export/{MM}/additional_eins_2022.csv + object: bulk_export/{MM}/2022_additional_eins.csv sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2022' + target_options: + format: csv public_data_v1_0_0.additional_eins.7: mode: full-refresh - object: bulk_export/{MM}/additional_eins_2023.csv + object: bulk_export/{MM}/2023_additional_eins.csv sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2023' + target_options: + format: csv public_data_v1_0_0.additional_eins.8: mode: full-refresh - object: bulk_export/{MM}/additional_eins_2024.csv + object: bulk_export/{MM}/2024_additional_eins.csv sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2024' + target_options: + format: csv public_data_v1_0_0.additional_eins.9: mode: full-refresh - object: bulk_export/{MM}/additional_eins_2025.csv + object: bulk_export/{MM}/2025_additional_eins.csv sql: SELECT * FROM public_data_v1_0_0.additional_eins WHERE audit_year = '2025' + target_options: + format: csv public_data_v1_0_0.additional_ueis.0: mode: full-refresh - object: bulk_export/{MM}/additional_ueis_2016.csv + object: bulk_export/{MM}/2016_additional_ueis.csv sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2016' + target_options: + format: csv public_data_v1_0_0.additional_ueis.1: mode: full-refresh - object: bulk_export/{MM}/additional_ueis_2017.csv + object: bulk_export/{MM}/2017_additional_ueis.csv sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2017' + target_options: + format: csv public_data_v1_0_0.additional_ueis.10: mode: full-refresh - object: bulk_export/{MM}/additional_ueis_2026.csv + object: bulk_export/{MM}/2026_additional_ueis.csv sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2026' + target_options: + format: csv public_data_v1_0_0.additional_ueis.11: mode: full-refresh - object: bulk_export/{MM}/additional_ueis_2027.csv + object: bulk_export/{MM}/2027_additional_ueis.csv sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2027' + target_options: + format: csv public_data_v1_0_0.additional_ueis.12: mode: full-refresh - object: bulk_export/{MM}/additional_ueis_2028.csv + object: bulk_export/{MM}/2028_additional_ueis.csv sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2028' + target_options: + format: csv public_data_v1_0_0.additional_ueis.13: mode: full-refresh - object: bulk_export/{MM}/additional_ueis_2029.csv + object: bulk_export/{MM}/2029_additional_ueis.csv sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2029' + target_options: + format: csv public_data_v1_0_0.additional_ueis.14: mode: full-refresh - object: bulk_export/{MM}/additional_ueis_2030.csv + object: bulk_export/{MM}/2030_additional_ueis.csv sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2030' + target_options: + format: csv public_data_v1_0_0.additional_ueis.2: mode: full-refresh - object: bulk_export/{MM}/additional_ueis_2018.csv + object: bulk_export/{MM}/2018_additional_ueis.csv sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2018' + target_options: + format: csv public_data_v1_0_0.additional_ueis.3: mode: full-refresh - object: bulk_export/{MM}/additional_ueis_2019.csv + object: bulk_export/{MM}/2019_additional_ueis.csv sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2019' + target_options: + format: csv public_data_v1_0_0.additional_ueis.4: mode: full-refresh - object: bulk_export/{MM}/additional_ueis_2020.csv + object: bulk_export/{MM}/2020_additional_ueis.csv sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2020' + target_options: + format: csv public_data_v1_0_0.additional_ueis.5: mode: full-refresh - object: bulk_export/{MM}/additional_ueis_2021.csv + object: bulk_export/{MM}/2021_additional_ueis.csv sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2021' + target_options: + format: csv public_data_v1_0_0.additional_ueis.6: mode: full-refresh - object: bulk_export/{MM}/additional_ueis_2022.csv + object: bulk_export/{MM}/2022_additional_ueis.csv sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2022' + target_options: + format: csv public_data_v1_0_0.additional_ueis.7: mode: full-refresh - object: bulk_export/{MM}/additional_ueis_2023.csv + object: bulk_export/{MM}/2023_additional_ueis.csv sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2023' + target_options: + format: csv public_data_v1_0_0.additional_ueis.8: mode: full-refresh - object: bulk_export/{MM}/additional_ueis_2024.csv + object: bulk_export/{MM}/2024_additional_ueis.csv sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2024' + target_options: + format: csv public_data_v1_0_0.additional_ueis.9: mode: full-refresh - object: bulk_export/{MM}/additional_ueis_2025.csv + object: bulk_export/{MM}/2025_additional_ueis.csv sql: SELECT * FROM public_data_v1_0_0.additional_ueis WHERE audit_year = '2025' + target_options: + format: csv public_data_v1_0_0.combined.0: mode: full-refresh - object: bulk_export/{MM}/combined_2016.csv + object: bulk_export/{MM}/2016_combined.csv sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2016' + target_options: + format: csv public_data_v1_0_0.combined.1: mode: full-refresh - object: bulk_export/{MM}/combined_2017.csv + object: bulk_export/{MM}/2017_combined.csv sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2017' + target_options: + format: csv public_data_v1_0_0.combined.10: mode: full-refresh - object: bulk_export/{MM}/combined_2026.csv + object: bulk_export/{MM}/2026_combined.csv sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2026' + target_options: + format: csv public_data_v1_0_0.combined.11: mode: full-refresh - object: bulk_export/{MM}/combined_2027.csv + object: bulk_export/{MM}/2027_combined.csv sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2027' + target_options: + format: csv public_data_v1_0_0.combined.12: mode: full-refresh - object: bulk_export/{MM}/combined_2028.csv + object: bulk_export/{MM}/2028_combined.csv sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2028' + target_options: + format: csv public_data_v1_0_0.combined.13: mode: full-refresh - object: bulk_export/{MM}/combined_2029.csv + object: bulk_export/{MM}/2029_combined.csv sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2029' + target_options: + format: csv public_data_v1_0_0.combined.14: mode: full-refresh - object: bulk_export/{MM}/combined_2030.csv + object: bulk_export/{MM}/2030_combined.csv sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2030' + target_options: + format: csv public_data_v1_0_0.combined.2: mode: full-refresh - object: bulk_export/{MM}/combined_2018.csv + object: bulk_export/{MM}/2018_combined.csv sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2018' + target_options: + format: csv public_data_v1_0_0.combined.3: mode: full-refresh - object: bulk_export/{MM}/combined_2019.csv + object: bulk_export/{MM}/2019_combined.csv sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2019' + target_options: + format: csv public_data_v1_0_0.combined.4: mode: full-refresh - object: bulk_export/{MM}/combined_2020.csv + object: bulk_export/{MM}/2020_combined.csv sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2020' + target_options: + format: csv public_data_v1_0_0.combined.5: mode: full-refresh - object: bulk_export/{MM}/combined_2021.csv + object: bulk_export/{MM}/2021_combined.csv sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2021' + target_options: + format: csv public_data_v1_0_0.combined.6: mode: full-refresh - object: bulk_export/{MM}/combined_2022.csv + object: bulk_export/{MM}/2022_combined.csv sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2022' + target_options: + format: csv public_data_v1_0_0.combined.7: mode: full-refresh - object: bulk_export/{MM}/combined_2023.csv + object: bulk_export/{MM}/2023_combined.csv sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2023' + target_options: + format: csv public_data_v1_0_0.combined.8: mode: full-refresh - object: bulk_export/{MM}/combined_2024.csv + object: bulk_export/{MM}/2024_combined.csv sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2024' + target_options: + format: csv public_data_v1_0_0.combined.9: mode: full-refresh - object: bulk_export/{MM}/combined_2025.csv + object: bulk_export/{MM}/2025_combined.csv sql: SELECT * FROM public_data_v1_0_0.combined WHERE audit_year = '2025' + target_options: + format: csv public_data_v1_0_0.corrective_action_plans.0: mode: full-refresh - object: bulk_export/{MM}/corrective_action_plans_2016.csv + object: bulk_export/{MM}/2016_corrective_action_plans.csv sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year = '2016' + target_options: + format: csv public_data_v1_0_0.corrective_action_plans.1: mode: full-refresh - object: bulk_export/{MM}/corrective_action_plans_2017.csv + object: bulk_export/{MM}/2017_corrective_action_plans.csv sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year = '2017' + target_options: + format: csv public_data_v1_0_0.corrective_action_plans.10: mode: full-refresh - object: bulk_export/{MM}/corrective_action_plans_2026.csv + object: bulk_export/{MM}/2026_corrective_action_plans.csv sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year = '2026' + target_options: + format: csv public_data_v1_0_0.corrective_action_plans.11: mode: full-refresh - object: bulk_export/{MM}/corrective_action_plans_2027.csv + object: bulk_export/{MM}/2027_corrective_action_plans.csv sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year = '2027' + target_options: + format: csv public_data_v1_0_0.corrective_action_plans.12: mode: full-refresh - object: bulk_export/{MM}/corrective_action_plans_2028.csv + object: bulk_export/{MM}/2028_corrective_action_plans.csv sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year = '2028' + target_options: + format: csv public_data_v1_0_0.corrective_action_plans.13: mode: full-refresh - object: bulk_export/{MM}/corrective_action_plans_2029.csv + object: bulk_export/{MM}/2029_corrective_action_plans.csv sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year = '2029' + target_options: + format: csv public_data_v1_0_0.corrective_action_plans.14: mode: full-refresh - object: bulk_export/{MM}/corrective_action_plans_2030.csv + object: bulk_export/{MM}/2030_corrective_action_plans.csv sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year = '2030' + target_options: + format: csv public_data_v1_0_0.corrective_action_plans.2: mode: full-refresh - object: bulk_export/{MM}/corrective_action_plans_2018.csv + object: bulk_export/{MM}/2018_corrective_action_plans.csv sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year = '2018' + target_options: + format: csv public_data_v1_0_0.corrective_action_plans.3: mode: full-refresh - object: bulk_export/{MM}/corrective_action_plans_2019.csv + object: bulk_export/{MM}/2019_corrective_action_plans.csv sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year = '2019' + target_options: + format: csv public_data_v1_0_0.corrective_action_plans.4: mode: full-refresh - object: bulk_export/{MM}/corrective_action_plans_2020.csv + object: bulk_export/{MM}/2020_corrective_action_plans.csv sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year = '2020' + target_options: + format: csv public_data_v1_0_0.corrective_action_plans.5: mode: full-refresh - object: bulk_export/{MM}/corrective_action_plans_2021.csv + object: bulk_export/{MM}/2021_corrective_action_plans.csv sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year = '2021' + target_options: + format: csv public_data_v1_0_0.corrective_action_plans.6: mode: full-refresh - object: bulk_export/{MM}/corrective_action_plans_2022.csv + object: bulk_export/{MM}/2022_corrective_action_plans.csv sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year = '2022' + target_options: + format: csv public_data_v1_0_0.corrective_action_plans.7: mode: full-refresh - object: bulk_export/{MM}/corrective_action_plans_2023.csv + object: bulk_export/{MM}/2023_corrective_action_plans.csv sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year = '2023' + target_options: + format: csv public_data_v1_0_0.corrective_action_plans.8: mode: full-refresh - object: bulk_export/{MM}/corrective_action_plans_2024.csv + object: bulk_export/{MM}/2024_corrective_action_plans.csv sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year = '2024' + target_options: + format: csv public_data_v1_0_0.corrective_action_plans.9: mode: full-refresh - object: bulk_export/{MM}/corrective_action_plans_2025.csv + object: bulk_export/{MM}/2025_corrective_action_plans.csv sql: SELECT * FROM public_data_v1_0_0.corrective_action_plans WHERE audit_year = '2025' + target_options: + format: csv public_data_v1_0_0.federal_awards.0: mode: full-refresh - object: bulk_export/{MM}/federal_awards_2016.csv + object: bulk_export/{MM}/2016_federal_awards.csv sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2016' + target_options: + format: csv public_data_v1_0_0.federal_awards.1: mode: full-refresh - object: bulk_export/{MM}/federal_awards_2017.csv + object: bulk_export/{MM}/2017_federal_awards.csv sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2017' + target_options: + format: csv public_data_v1_0_0.federal_awards.10: mode: full-refresh - object: bulk_export/{MM}/federal_awards_2026.csv + object: bulk_export/{MM}/2026_federal_awards.csv sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2026' + target_options: + format: csv public_data_v1_0_0.federal_awards.11: mode: full-refresh - object: bulk_export/{MM}/federal_awards_2027.csv + object: bulk_export/{MM}/2027_federal_awards.csv sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2027' + target_options: + format: csv public_data_v1_0_0.federal_awards.12: mode: full-refresh - object: bulk_export/{MM}/federal_awards_2028.csv + object: bulk_export/{MM}/2028_federal_awards.csv sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2028' + target_options: + format: csv public_data_v1_0_0.federal_awards.13: mode: full-refresh - object: bulk_export/{MM}/federal_awards_2029.csv + object: bulk_export/{MM}/2029_federal_awards.csv sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2029' + target_options: + format: csv public_data_v1_0_0.federal_awards.14: mode: full-refresh - object: bulk_export/{MM}/federal_awards_2030.csv + object: bulk_export/{MM}/2030_federal_awards.csv sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2030' + target_options: + format: csv public_data_v1_0_0.federal_awards.2: mode: full-refresh - object: bulk_export/{MM}/federal_awards_2018.csv + object: bulk_export/{MM}/2018_federal_awards.csv sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2018' + target_options: + format: csv public_data_v1_0_0.federal_awards.3: mode: full-refresh - object: bulk_export/{MM}/federal_awards_2019.csv + object: bulk_export/{MM}/2019_federal_awards.csv sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2019' + target_options: + format: csv public_data_v1_0_0.federal_awards.4: mode: full-refresh - object: bulk_export/{MM}/federal_awards_2020.csv + object: bulk_export/{MM}/2020_federal_awards.csv sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2020' + target_options: + format: csv public_data_v1_0_0.federal_awards.5: mode: full-refresh - object: bulk_export/{MM}/federal_awards_2021.csv + object: bulk_export/{MM}/2021_federal_awards.csv sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2021' + target_options: + format: csv public_data_v1_0_0.federal_awards.6: mode: full-refresh - object: bulk_export/{MM}/federal_awards_2022.csv + object: bulk_export/{MM}/2022_federal_awards.csv sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2022' + target_options: + format: csv public_data_v1_0_0.federal_awards.7: mode: full-refresh - object: bulk_export/{MM}/federal_awards_2023.csv + object: bulk_export/{MM}/2023_federal_awards.csv sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2023' + target_options: + format: csv public_data_v1_0_0.federal_awards.8: mode: full-refresh - object: bulk_export/{MM}/federal_awards_2024.csv + object: bulk_export/{MM}/2024_federal_awards.csv sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2024' + target_options: + format: csv public_data_v1_0_0.federal_awards.9: mode: full-refresh - object: bulk_export/{MM}/federal_awards_2025.csv + object: bulk_export/{MM}/2025_federal_awards.csv sql: SELECT * FROM public_data_v1_0_0.federal_awards WHERE audit_year = '2025' + target_options: + format: csv public_data_v1_0_0.findings.0: mode: full-refresh - object: bulk_export/{MM}/findings_2016.csv + object: bulk_export/{MM}/2016_findings.csv sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2016' + target_options: + format: csv public_data_v1_0_0.findings.1: mode: full-refresh - object: bulk_export/{MM}/findings_2017.csv + object: bulk_export/{MM}/2017_findings.csv sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2017' + target_options: + format: csv public_data_v1_0_0.findings.10: mode: full-refresh - object: bulk_export/{MM}/findings_2026.csv + object: bulk_export/{MM}/2026_findings.csv sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2026' + target_options: + format: csv public_data_v1_0_0.findings.11: mode: full-refresh - object: bulk_export/{MM}/findings_2027.csv + object: bulk_export/{MM}/2027_findings.csv sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2027' + target_options: + format: csv public_data_v1_0_0.findings.12: mode: full-refresh - object: bulk_export/{MM}/findings_2028.csv + object: bulk_export/{MM}/2028_findings.csv sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2028' + target_options: + format: csv public_data_v1_0_0.findings.13: mode: full-refresh - object: bulk_export/{MM}/findings_2029.csv + object: bulk_export/{MM}/2029_findings.csv sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2029' + target_options: + format: csv public_data_v1_0_0.findings.14: mode: full-refresh - object: bulk_export/{MM}/findings_2030.csv + object: bulk_export/{MM}/2030_findings.csv sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2030' + target_options: + format: csv public_data_v1_0_0.findings.2: mode: full-refresh - object: bulk_export/{MM}/findings_2018.csv + object: bulk_export/{MM}/2018_findings.csv sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2018' + target_options: + format: csv public_data_v1_0_0.findings.3: mode: full-refresh - object: bulk_export/{MM}/findings_2019.csv + object: bulk_export/{MM}/2019_findings.csv sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2019' + target_options: + format: csv public_data_v1_0_0.findings.4: mode: full-refresh - object: bulk_export/{MM}/findings_2020.csv + object: bulk_export/{MM}/2020_findings.csv sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2020' + target_options: + format: csv public_data_v1_0_0.findings.5: mode: full-refresh - object: bulk_export/{MM}/findings_2021.csv + object: bulk_export/{MM}/2021_findings.csv sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2021' + target_options: + format: csv public_data_v1_0_0.findings.6: mode: full-refresh - object: bulk_export/{MM}/findings_2022.csv + object: bulk_export/{MM}/2022_findings.csv sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2022' + target_options: + format: csv public_data_v1_0_0.findings.7: mode: full-refresh - object: bulk_export/{MM}/findings_2023.csv + object: bulk_export/{MM}/2023_findings.csv sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2023' + target_options: + format: csv public_data_v1_0_0.findings.8: mode: full-refresh - object: bulk_export/{MM}/findings_2024.csv + object: bulk_export/{MM}/2024_findings.csv sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2024' + target_options: + format: csv public_data_v1_0_0.findings.9: mode: full-refresh - object: bulk_export/{MM}/findings_2025.csv + object: bulk_export/{MM}/2025_findings.csv sql: SELECT * FROM public_data_v1_0_0.findings WHERE audit_year = '2025' + target_options: + format: csv public_data_v1_0_0.findings_text.0: mode: full-refresh - object: bulk_export/{MM}/findings_text_2016.csv + object: bulk_export/{MM}/2016_findings_text.csv sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2016' + target_options: + format: csv public_data_v1_0_0.findings_text.1: mode: full-refresh - object: bulk_export/{MM}/findings_text_2017.csv + object: bulk_export/{MM}/2017_findings_text.csv sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2017' + target_options: + format: csv public_data_v1_0_0.findings_text.10: mode: full-refresh - object: bulk_export/{MM}/findings_text_2026.csv + object: bulk_export/{MM}/2026_findings_text.csv sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2026' + target_options: + format: csv public_data_v1_0_0.findings_text.11: mode: full-refresh - object: bulk_export/{MM}/findings_text_2027.csv + object: bulk_export/{MM}/2027_findings_text.csv sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2027' + target_options: + format: csv public_data_v1_0_0.findings_text.12: mode: full-refresh - object: bulk_export/{MM}/findings_text_2028.csv + object: bulk_export/{MM}/2028_findings_text.csv sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2028' + target_options: + format: csv public_data_v1_0_0.findings_text.13: mode: full-refresh - object: bulk_export/{MM}/findings_text_2029.csv + object: bulk_export/{MM}/2029_findings_text.csv sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2029' + target_options: + format: csv public_data_v1_0_0.findings_text.14: mode: full-refresh - object: bulk_export/{MM}/findings_text_2030.csv + object: bulk_export/{MM}/2030_findings_text.csv sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2030' + target_options: + format: csv public_data_v1_0_0.findings_text.2: mode: full-refresh - object: bulk_export/{MM}/findings_text_2018.csv + object: bulk_export/{MM}/2018_findings_text.csv sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2018' + target_options: + format: csv public_data_v1_0_0.findings_text.3: mode: full-refresh - object: bulk_export/{MM}/findings_text_2019.csv + object: bulk_export/{MM}/2019_findings_text.csv sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2019' + target_options: + format: csv public_data_v1_0_0.findings_text.4: mode: full-refresh - object: bulk_export/{MM}/findings_text_2020.csv + object: bulk_export/{MM}/2020_findings_text.csv sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2020' + target_options: + format: csv public_data_v1_0_0.findings_text.5: mode: full-refresh - object: bulk_export/{MM}/findings_text_2021.csv + object: bulk_export/{MM}/2021_findings_text.csv sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2021' + target_options: + format: csv public_data_v1_0_0.findings_text.6: mode: full-refresh - object: bulk_export/{MM}/findings_text_2022.csv + object: bulk_export/{MM}/2022_findings_text.csv sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2022' + target_options: + format: csv public_data_v1_0_0.findings_text.7: mode: full-refresh - object: bulk_export/{MM}/findings_text_2023.csv + object: bulk_export/{MM}/2023_findings_text.csv sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2023' + target_options: + format: csv public_data_v1_0_0.findings_text.8: mode: full-refresh - object: bulk_export/{MM}/findings_text_2024.csv + object: bulk_export/{MM}/2024_findings_text.csv sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2024' + target_options: + format: csv public_data_v1_0_0.findings_text.9: mode: full-refresh - object: bulk_export/{MM}/findings_text_2025.csv + object: bulk_export/{MM}/2025_findings_text.csv sql: SELECT * FROM public_data_v1_0_0.findings_text WHERE audit_year = '2025' + target_options: + format: csv public_data_v1_0_0.general.0: mode: full-refresh - object: bulk_export/{MM}/general_2016.csv + object: bulk_export/{MM}/2016_general.csv sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2016' + target_options: + format: csv public_data_v1_0_0.general.1: mode: full-refresh - object: bulk_export/{MM}/general_2017.csv + object: bulk_export/{MM}/2017_general.csv sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2017' + target_options: + format: csv public_data_v1_0_0.general.10: mode: full-refresh - object: bulk_export/{MM}/general_2026.csv + object: bulk_export/{MM}/2026_general.csv sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2026' + target_options: + format: csv public_data_v1_0_0.general.11: mode: full-refresh - object: bulk_export/{MM}/general_2027.csv + object: bulk_export/{MM}/2027_general.csv sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2027' + target_options: + format: csv public_data_v1_0_0.general.12: mode: full-refresh - object: bulk_export/{MM}/general_2028.csv + object: bulk_export/{MM}/2028_general.csv sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2028' + target_options: + format: csv public_data_v1_0_0.general.13: mode: full-refresh - object: bulk_export/{MM}/general_2029.csv + object: bulk_export/{MM}/2029_general.csv sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2029' + target_options: + format: csv public_data_v1_0_0.general.14: mode: full-refresh - object: bulk_export/{MM}/general_2030.csv + object: bulk_export/{MM}/2030_general.csv sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2030' + target_options: + format: csv public_data_v1_0_0.general.2: mode: full-refresh - object: bulk_export/{MM}/general_2018.csv + object: bulk_export/{MM}/2018_general.csv sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2018' + target_options: + format: csv public_data_v1_0_0.general.3: mode: full-refresh - object: bulk_export/{MM}/general_2019.csv + object: bulk_export/{MM}/2019_general.csv sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2019' + target_options: + format: csv public_data_v1_0_0.general.4: mode: full-refresh - object: bulk_export/{MM}/general_2020.csv + object: bulk_export/{MM}/2020_general.csv sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2020' + target_options: + format: csv public_data_v1_0_0.general.5: mode: full-refresh - object: bulk_export/{MM}/general_2021.csv + object: bulk_export/{MM}/2021_general.csv sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2021' + target_options: + format: csv public_data_v1_0_0.general.6: mode: full-refresh - object: bulk_export/{MM}/general_2022.csv + object: bulk_export/{MM}/2022_general.csv sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2022' + target_options: + format: csv public_data_v1_0_0.general.7: mode: full-refresh - object: bulk_export/{MM}/general_2023.csv + object: bulk_export/{MM}/2023_general.csv sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2023' + target_options: + format: csv public_data_v1_0_0.general.8: mode: full-refresh - object: bulk_export/{MM}/general_2024.csv + object: bulk_export/{MM}/2024_general.csv sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2024' + target_options: + format: csv public_data_v1_0_0.general.9: mode: full-refresh - object: bulk_export/{MM}/general_2025.csv + object: bulk_export/{MM}/2025_general.csv sql: SELECT * FROM public_data_v1_0_0.general WHERE audit_year = '2025' + target_options: + format: csv public_data_v1_0_0.notes_to_sefa.0: mode: full-refresh - object: bulk_export/{MM}/notes_to_sefa_2016.csv + object: bulk_export/{MM}/2016_notes_to_sefa.csv sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2016' + target_options: + format: csv public_data_v1_0_0.notes_to_sefa.1: mode: full-refresh - object: bulk_export/{MM}/notes_to_sefa_2017.csv + object: bulk_export/{MM}/2017_notes_to_sefa.csv sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2017' + target_options: + format: csv public_data_v1_0_0.notes_to_sefa.10: mode: full-refresh - object: bulk_export/{MM}/notes_to_sefa_2026.csv + object: bulk_export/{MM}/2026_notes_to_sefa.csv sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2026' + target_options: + format: csv public_data_v1_0_0.notes_to_sefa.11: mode: full-refresh - object: bulk_export/{MM}/notes_to_sefa_2027.csv + object: bulk_export/{MM}/2027_notes_to_sefa.csv sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2027' + target_options: + format: csv public_data_v1_0_0.notes_to_sefa.12: mode: full-refresh - object: bulk_export/{MM}/notes_to_sefa_2028.csv + object: bulk_export/{MM}/2028_notes_to_sefa.csv sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2028' + target_options: + format: csv public_data_v1_0_0.notes_to_sefa.13: mode: full-refresh - object: bulk_export/{MM}/notes_to_sefa_2029.csv + object: bulk_export/{MM}/2029_notes_to_sefa.csv sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2029' + target_options: + format: csv public_data_v1_0_0.notes_to_sefa.14: mode: full-refresh - object: bulk_export/{MM}/notes_to_sefa_2030.csv + object: bulk_export/{MM}/2030_notes_to_sefa.csv sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2030' + target_options: + format: csv public_data_v1_0_0.notes_to_sefa.2: mode: full-refresh - object: bulk_export/{MM}/notes_to_sefa_2018.csv + object: bulk_export/{MM}/2018_notes_to_sefa.csv sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2018' + target_options: + format: csv public_data_v1_0_0.notes_to_sefa.3: mode: full-refresh - object: bulk_export/{MM}/notes_to_sefa_2019.csv + object: bulk_export/{MM}/2019_notes_to_sefa.csv sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2019' + target_options: + format: csv public_data_v1_0_0.notes_to_sefa.4: mode: full-refresh - object: bulk_export/{MM}/notes_to_sefa_2020.csv + object: bulk_export/{MM}/2020_notes_to_sefa.csv sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2020' + target_options: + format: csv public_data_v1_0_0.notes_to_sefa.5: mode: full-refresh - object: bulk_export/{MM}/notes_to_sefa_2021.csv + object: bulk_export/{MM}/2021_notes_to_sefa.csv sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2021' + target_options: + format: csv public_data_v1_0_0.notes_to_sefa.6: mode: full-refresh - object: bulk_export/{MM}/notes_to_sefa_2022.csv + object: bulk_export/{MM}/2022_notes_to_sefa.csv sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2022' + target_options: + format: csv public_data_v1_0_0.notes_to_sefa.7: mode: full-refresh - object: bulk_export/{MM}/notes_to_sefa_2023.csv + object: bulk_export/{MM}/2023_notes_to_sefa.csv sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2023' + target_options: + format: csv public_data_v1_0_0.notes_to_sefa.8: mode: full-refresh - object: bulk_export/{MM}/notes_to_sefa_2024.csv + object: bulk_export/{MM}/2024_notes_to_sefa.csv sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2024' + target_options: + format: csv public_data_v1_0_0.notes_to_sefa.9: mode: full-refresh - object: bulk_export/{MM}/notes_to_sefa_2025.csv + object: bulk_export/{MM}/2025_notes_to_sefa.csv sql: SELECT * FROM public_data_v1_0_0.notes_to_sefa WHERE audit_year = '2025' + target_options: + format: csv public_data_v1_0_0.passthrough.0: mode: full-refresh - object: bulk_export/{MM}/passthrough_2016.csv + object: bulk_export/{MM}/2016_passthrough.csv sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2016' + target_options: + format: csv public_data_v1_0_0.passthrough.1: mode: full-refresh - object: bulk_export/{MM}/passthrough_2017.csv + object: bulk_export/{MM}/2017_passthrough.csv sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2017' + target_options: + format: csv public_data_v1_0_0.passthrough.10: mode: full-refresh - object: bulk_export/{MM}/passthrough_2026.csv + object: bulk_export/{MM}/2026_passthrough.csv sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2026' + target_options: + format: csv public_data_v1_0_0.passthrough.11: mode: full-refresh - object: bulk_export/{MM}/passthrough_2027.csv + object: bulk_export/{MM}/2027_passthrough.csv sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2027' + target_options: + format: csv public_data_v1_0_0.passthrough.12: mode: full-refresh - object: bulk_export/{MM}/passthrough_2028.csv + object: bulk_export/{MM}/2028_passthrough.csv sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2028' + target_options: + format: csv public_data_v1_0_0.passthrough.13: mode: full-refresh - object: bulk_export/{MM}/passthrough_2029.csv + object: bulk_export/{MM}/2029_passthrough.csv sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2029' + target_options: + format: csv public_data_v1_0_0.passthrough.14: mode: full-refresh - object: bulk_export/{MM}/passthrough_2030.csv + object: bulk_export/{MM}/2030_passthrough.csv sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2030' + target_options: + format: csv public_data_v1_0_0.passthrough.2: mode: full-refresh - object: bulk_export/{MM}/passthrough_2018.csv + object: bulk_export/{MM}/2018_passthrough.csv sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2018' + target_options: + format: csv public_data_v1_0_0.passthrough.3: mode: full-refresh - object: bulk_export/{MM}/passthrough_2019.csv + object: bulk_export/{MM}/2019_passthrough.csv sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2019' + target_options: + format: csv public_data_v1_0_0.passthrough.4: mode: full-refresh - object: bulk_export/{MM}/passthrough_2020.csv + object: bulk_export/{MM}/2020_passthrough.csv sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2020' + target_options: + format: csv public_data_v1_0_0.passthrough.5: mode: full-refresh - object: bulk_export/{MM}/passthrough_2021.csv + object: bulk_export/{MM}/2021_passthrough.csv sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2021' + target_options: + format: csv public_data_v1_0_0.passthrough.6: mode: full-refresh - object: bulk_export/{MM}/passthrough_2022.csv + object: bulk_export/{MM}/2022_passthrough.csv sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2022' + target_options: + format: csv public_data_v1_0_0.passthrough.7: mode: full-refresh - object: bulk_export/{MM}/passthrough_2023.csv + object: bulk_export/{MM}/2023_passthrough.csv sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2023' + target_options: + format: csv public_data_v1_0_0.passthrough.8: mode: full-refresh - object: bulk_export/{MM}/passthrough_2024.csv + object: bulk_export/{MM}/2024_passthrough.csv sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2024' + target_options: + format: csv public_data_v1_0_0.passthrough.9: mode: full-refresh - object: bulk_export/{MM}/passthrough_2025.csv + object: bulk_export/{MM}/2025_passthrough.csv sql: SELECT * FROM public_data_v1_0_0.passthrough WHERE audit_year = '2025' + target_options: + format: csv public_data_v1_0_0.secondary_auditors.0: mode: full-refresh - object: bulk_export/{MM}/secondary_auditors_2016.csv + object: bulk_export/{MM}/2016_secondary_auditors.csv sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2016' + target_options: + format: csv public_data_v1_0_0.secondary_auditors.1: mode: full-refresh - object: bulk_export/{MM}/secondary_auditors_2017.csv + object: bulk_export/{MM}/2017_secondary_auditors.csv sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2017' + target_options: + format: csv public_data_v1_0_0.secondary_auditors.10: mode: full-refresh - object: bulk_export/{MM}/secondary_auditors_2026.csv + object: bulk_export/{MM}/2026_secondary_auditors.csv sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2026' + target_options: + format: csv public_data_v1_0_0.secondary_auditors.11: mode: full-refresh - object: bulk_export/{MM}/secondary_auditors_2027.csv + object: bulk_export/{MM}/2027_secondary_auditors.csv sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2027' + target_options: + format: csv public_data_v1_0_0.secondary_auditors.12: mode: full-refresh - object: bulk_export/{MM}/secondary_auditors_2028.csv + object: bulk_export/{MM}/2028_secondary_auditors.csv sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2028' + target_options: + format: csv public_data_v1_0_0.secondary_auditors.13: mode: full-refresh - object: bulk_export/{MM}/secondary_auditors_2029.csv + object: bulk_export/{MM}/2029_secondary_auditors.csv sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2029' + target_options: + format: csv public_data_v1_0_0.secondary_auditors.14: mode: full-refresh - object: bulk_export/{MM}/secondary_auditors_2030.csv + object: bulk_export/{MM}/2030_secondary_auditors.csv sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2030' + target_options: + format: csv public_data_v1_0_0.secondary_auditors.2: mode: full-refresh - object: bulk_export/{MM}/secondary_auditors_2018.csv + object: bulk_export/{MM}/2018_secondary_auditors.csv sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2018' + target_options: + format: csv public_data_v1_0_0.secondary_auditors.3: mode: full-refresh - object: bulk_export/{MM}/secondary_auditors_2019.csv + object: bulk_export/{MM}/2019_secondary_auditors.csv sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2019' + target_options: + format: csv public_data_v1_0_0.secondary_auditors.4: mode: full-refresh - object: bulk_export/{MM}/secondary_auditors_2020.csv + object: bulk_export/{MM}/2020_secondary_auditors.csv sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2020' + target_options: + format: csv public_data_v1_0_0.secondary_auditors.5: mode: full-refresh - object: bulk_export/{MM}/secondary_auditors_2021.csv + object: bulk_export/{MM}/2021_secondary_auditors.csv sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2021' + target_options: + format: csv public_data_v1_0_0.secondary_auditors.6: mode: full-refresh - object: bulk_export/{MM}/secondary_auditors_2022.csv + object: bulk_export/{MM}/2022_secondary_auditors.csv sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2022' + target_options: + format: csv public_data_v1_0_0.secondary_auditors.7: mode: full-refresh - object: bulk_export/{MM}/secondary_auditors_2023.csv + object: bulk_export/{MM}/2023_secondary_auditors.csv sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2023' + target_options: + format: csv public_data_v1_0_0.secondary_auditors.8: mode: full-refresh - object: bulk_export/{MM}/secondary_auditors_2024.csv + object: bulk_export/{MM}/2024_secondary_auditors.csv sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2024' + target_options: + format: csv public_data_v1_0_0.secondary_auditors.9: mode: full-refresh - object: bulk_export/{MM}/secondary_auditors_2025.csv + object: bulk_export/{MM}/2025_secondary_auditors.csv sql: SELECT * FROM public_data_v1_0_0.secondary_auditors WHERE audit_year = '2025' + target_options: + format: csv target: BULK_DATA_EXPORT From d329f273f67b1bfb7ff4a5c829c0fbf3c17f3b4a Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Fri, 25 Oct 2024 16:14:04 -0400 Subject: [PATCH 50/89] Simplified local data load This used to do a cgov-util copy. Now, it doesn't. The startup sequence should take care of it. --- .../load_public_data_locally.sh | 36 ------------------- 1 file changed, 36 deletions(-) diff --git a/backend/util/load_public_dissem_data/load_public_data_locally.sh b/backend/util/load_public_dissem_data/load_public_data_locally.sh index 8919253330..63d85effb4 100755 --- a/backend/util/load_public_dissem_data/load_public_data_locally.sh +++ b/backend/util/load_public_dissem_data/load_public_data_locally.sh @@ -58,39 +58,3 @@ if [ $result -ne 0 ]; then else echo "Loaded lots of data without error, apparently." fi - -# We'll stop here. Now, on first boot, the FAC should see if things are -# in place in DB2. If not, the app startup process will take -# care of creating new tables, etc. - -# # Download cgov-util. -# # This will let us do a backup from the main database (fac-db) -# # to the secondary database (fac-snapshot-db). This mirrors what -# # happens in production. -# pushd /app/data - -# Run the backup of the dissemination_ tables from -# fac-db to fac-snapshot-db. -check_table_exists $FAC_SNAPSHOT_URI "public" "dissemination_general" -result=$? -# cgov-util wants to know the environment it is in. -export ENV="LOCAL" -if [ $result -ne 0 ]; then - # First run if it does not exist. - /layered/cgov-util db_to_db \ - --src_db fac-db \ - --dest_db fac-snapshot-db \ - --operation initial -else - /layered/cgov-util db_to_db \ - --src_db fac-db \ - --dest_db fac-snapshot-db \ - --operation backup -fi - -# # Now, we're going to run sling. -# # This will create the API tables. It essentially does a copy of -# # data from fac-snapshot-db (in the dissem_* tables) to a set of -# # tables that the API will point at. -# sling run -r /layered/sling.yaml -# sling run -r /layered/tribal_sling.yaml From c2149f19f168d1f1f9846b5f15a3e09ed0cde327 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Mon, 28 Oct 2024 11:26:22 -0400 Subject: [PATCH 51/89] Updating to reflect new startup sequence This was out of date. Retrying preview deploy. --- backend/.profile | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/backend/.profile b/backend/.profile index fd881aaf09..46fe29a7d5 100644 --- a/backend/.profile +++ b/backend/.profile @@ -29,19 +29,14 @@ if [[ "$CF_INSTANCE_INDEX" == 0 ]]; then # This tears down things that would conflict with migrations, etc. sql_pre gonogo "sql_pre" + curation_audit_tracking_disable + gonogo "curation_audit_tracking_disable" ##### # MIGRATE APP TABLES migrate_app_tables gonogo "migrate_app_tables" - ##### - # PREP API TABLES - # This runs sling and preps tables in the snapshot DB. - # Only runs if the tables are not present (e.g. first deploy) - sling_first_run - gonogo "sling_first_run" - ##### # SQL POST # Rebuild the API and prepare the system for execution. @@ -54,6 +49,17 @@ if [[ "$CF_INSTANCE_INDEX" == 0 ]]; then # Setup tables for cog/over assignments seed_cog_baseline gonogo "seed_cog_baseline" + + ##### + # CREATE STAFF USERS + # Prepares staff users for Django admin + python manage.py create_staffusers + + ##### + # LAUNCH THE APP + # We will have died long ago if things didn't work. + npm run dev & python manage.py runserver 0.0.0.0:8000 + fi # Make psql usable by scripts, for debugging, etc. From e1a3bb850fd1b9cba982eb537ea6d908b2d2969e Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Mon, 28 Oct 2024 11:38:15 -0400 Subject: [PATCH 52/89] And, forgot the sourcing... --- backend/.profile | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/backend/.profile b/backend/.profile index 46fe29a7d5..86a44bde14 100644 --- a/backend/.profile +++ b/backend/.profile @@ -2,19 +2,13 @@ set +e -# Source everything; everything is now a function. -# Remember: bash has no idea if a function exists, -# so a typo in a function name will fail silently. Similarly, -# bash has horrible scoping, so use of `local` in functions is -# critical for cleanliness in the startup script. -# This will choose the correct environment -# for local envs (LOCAL or TESTING) and cloud.gov +source tools/util_startup.sh +source tools/setup_env.sh +source tools/curation_audit_tracking_disable.sh +source tools/sling_bulk_export.sh source tools/migrate_app_tables.sh source tools/seed_cog_baseline.sh -source tools/setup_env.sh -source tools/sling_first_run.sh source tools/sql_pre_post.sh -source tools/util_startup.sh ##### # SETUP THE CGOV ENVIRONMENT From edf2c1fbcef682c2d6c1eb354aeaf7a4a0772182 Mon Sep 17 00:00:00 2001 From: Alex Steel <130377221+asteel-gsa@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:21:33 -0400 Subject: [PATCH 53/89] Fake audit, test if migrations will go through --- backend/tools/migrate_app_tables.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/tools/migrate_app_tables.sh b/backend/tools/migrate_app_tables.sh index aef200d092..5e751ed25d 100644 --- a/backend/tools/migrate_app_tables.sh +++ b/backend/tools/migrate_app_tables.sh @@ -2,7 +2,8 @@ source tools/util_startup.sh function migrate_app_tables { startup_log "MIGRATE_APP_TABLES" "BEGIN" - python manage.py migrate + # python manage.py migrate + python manage.py migrate --fake audit local result=$? startup_log "MIGRATE_APP_TABLES" "END" return $result From 5dd7e92bc93bce5876199f5583d6e7e4bbf4c0cf Mon Sep 17 00:00:00 2001 From: Alex Steel <130377221+asteel-gsa@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:36:54 -0400 Subject: [PATCH 54/89] Add replaces to Django Ref: https://github.com/GSA-TTS/FAC/commit/8d59859ec6ef011fc12f31947d74b4fbd9699964 --- ...hed_0013_singleauditchecklistflow_and_more.py | 16 ++++++++++++++++ backend/tools/migrate_app_tables.sh | 3 +-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/backend/audit/migrations/0001_squashed_0013_singleauditchecklistflow_and_more.py b/backend/audit/migrations/0001_squashed_0013_singleauditchecklistflow_and_more.py index 81345720c6..ccbe6a85d5 100644 --- a/backend/audit/migrations/0001_squashed_0013_singleauditchecklistflow_and_more.py +++ b/backend/audit/migrations/0001_squashed_0013_singleauditchecklistflow_and_more.py @@ -11,6 +11,22 @@ class Migration(migrations.Migration): + replaces = [ + ("audit", "0001_initial"), + ("audit", "0002_alter_singleauditchecklist_report_id"), + ("audit", "0003_alter_singleauditchecklist_data_source_and_more"), + ("audit", "0004_alter_singleauditchecklist_cognizant_agency_and_more"), + ("audit", "0005_alter_submissionevent_event"), + ("audit", "0006_deletedaccess"), + ("audit", "0007_alter_deletedaccess_options_and_more"), + ("audit", "0008_alter_submissionevent_event"), + ("audit", "0009_ueivalidationwaiver_sacvalidationwaiver"), + ("audit", "0010_alter_ueivalidationwaiver_uei"), + ("audit", "0011_ueivalidationwaiver_expiration_and_more"), + ("audit", "0012_alter_sacvalidationwaiver_waiver_types"), + ("audit", "0013_singleauditchecklistflow_and_more"), + ] + initial = True dependencies = [ diff --git a/backend/tools/migrate_app_tables.sh b/backend/tools/migrate_app_tables.sh index 5e751ed25d..aef200d092 100644 --- a/backend/tools/migrate_app_tables.sh +++ b/backend/tools/migrate_app_tables.sh @@ -2,8 +2,7 @@ source tools/util_startup.sh function migrate_app_tables { startup_log "MIGRATE_APP_TABLES" "BEGIN" - # python manage.py migrate - python manage.py migrate --fake audit + python manage.py migrate local result=$? startup_log "MIGRATE_APP_TABLES" "END" return $result From db1927b80e9390ff5086b92580cc052570c9c0e7 Mon Sep 17 00:00:00 2001 From: Alex Steel <130377221+asteel-gsa@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:41:25 -0400 Subject: [PATCH 55/89] Undo --- ...hed_0013_singleauditchecklistflow_and_more.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/backend/audit/migrations/0001_squashed_0013_singleauditchecklistflow_and_more.py b/backend/audit/migrations/0001_squashed_0013_singleauditchecklistflow_and_more.py index ccbe6a85d5..81345720c6 100644 --- a/backend/audit/migrations/0001_squashed_0013_singleauditchecklistflow_and_more.py +++ b/backend/audit/migrations/0001_squashed_0013_singleauditchecklistflow_and_more.py @@ -11,22 +11,6 @@ class Migration(migrations.Migration): - replaces = [ - ("audit", "0001_initial"), - ("audit", "0002_alter_singleauditchecklist_report_id"), - ("audit", "0003_alter_singleauditchecklist_data_source_and_more"), - ("audit", "0004_alter_singleauditchecklist_cognizant_agency_and_more"), - ("audit", "0005_alter_submissionevent_event"), - ("audit", "0006_deletedaccess"), - ("audit", "0007_alter_deletedaccess_options_and_more"), - ("audit", "0008_alter_submissionevent_event"), - ("audit", "0009_ueivalidationwaiver_sacvalidationwaiver"), - ("audit", "0010_alter_ueivalidationwaiver_uei"), - ("audit", "0011_ueivalidationwaiver_expiration_and_more"), - ("audit", "0012_alter_sacvalidationwaiver_waiver_types"), - ("audit", "0013_singleauditchecklistflow_and_more"), - ] - initial = True dependencies = [ From 3ec3e817a0fd4d271ec0ee08b7025d0c996c75e9 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Mon, 28 Oct 2024 17:30:04 -0400 Subject: [PATCH 56/89] Should fix local standup/build We need to always load the fake VCAP_SERVICES from the file when running locally. --- backend/config/settings.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/config/settings.py b/backend/config/settings.py index 3acc298eb0..de549b9e68 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -234,8 +234,11 @@ # a "fake" VCAP_SERVICES environment variable. Local DBs and S3 buckets # can be configured the same way as their cloud equivalents. This can be # refactored for simpler config loading in the app. + # + # During a build, there won't be an environment variable. Load the + # fake VCAP from the filesystem. + vcap = json.load(open("config/vcap_services_for_containers.json")) - vcap = json.loads(env.str("VCAP_SERVICES")) DATABASES = { "default": dj_database_url.parse(get_db_url_from_vcap_services(vcap, "fac-db")), "fac-db": dj_database_url.parse(get_db_url_from_vcap_services(vcap, "fac-db")), From ffad20f2faadb7b018ec9e2f747a8fd2cb87c624 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Tue, 29 Oct 2024 09:46:00 -0400 Subject: [PATCH 57/89] Removing orderby on combined There's no reason to order the results from the mega-table. --- .../sql/fac-snapshot-db/post/143_api_v2_0_0_views.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/dissemination/sql/fac-snapshot-db/post/143_api_v2_0_0_views.sql b/backend/dissemination/sql/fac-snapshot-db/post/143_api_v2_0_0_views.sql index 88aa11a736..aac2b3c5c0 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/143_api_v2_0_0_views.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/143_api_v2_0_0_views.sql @@ -73,7 +73,7 @@ CREATE VIEW api_v2_0_0.secondary_auditors AS --------------------------------------- CREATE VIEW api_v2_0_0.combined AS SELECT * FROM public_data_v1_0_0.combined comb - ORDER BY comb.seq; + ; --------------------------------------- -- metadata From 409792ba616cb7ce460983d9060dad0d313e1bb7 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Tue, 29 Oct 2024 12:14:26 -0400 Subject: [PATCH 58/89] Splits things away from startup Now, pre-post is per-db. That means we must run the pre/post on SNAPSHOT separately in an action before deploy. --- backend/.profile | 9 ++- backend/config/staffusers.json | 3 +- .../post/010_copy_dissem_tables.sql | 4 +- .../post/020_public_data_v1_0_0_tables.sql | 74 ++++++++----------- .../021_suppressed_data_v1_0_0_tables.sql | 41 ++++------ .../post/030_public_data_v1_0_0_metadata.sql | 24 ++---- .../fac-snapshot-db/post/800_permissions.sql | 7 -- .../sql/fac-snapshot-db/pre/060_schemas.sql | 28 +------ backend/tools/sql_pre_post.sh | 20 +++++ backend/util/nightly_api_refresh.sh | 17 +++++ 10 files changed, 97 insertions(+), 130 deletions(-) create mode 100644 backend/util/nightly_api_refresh.sh diff --git a/backend/.profile b/backend/.profile index 86a44bde14..bf157d9c57 100644 --- a/backend/.profile +++ b/backend/.profile @@ -21,8 +21,8 @@ if [[ "$CF_INSTANCE_INDEX" == 0 ]]; then # SQL PRE # We have SQL that we want to run before the migrations and sling are run. # This tears down things that would conflict with migrations, etc. - sql_pre - gonogo "sql_pre" + sql_pre_fac_db + gonogo "sql_pre_fac_db" curation_audit_tracking_disable gonogo "curation_audit_tracking_disable" @@ -35,8 +35,8 @@ if [[ "$CF_INSTANCE_INDEX" == 0 ]]; then # SQL POST # Rebuild the API and prepare the system for execution. # Runs after migrations. - sql_post - gonogo "sql_post" + sql_post_fac_db + gonogo "sql_post_fac_db" ##### # SEED COG/OVER TABLES @@ -48,6 +48,7 @@ if [[ "$CF_INSTANCE_INDEX" == 0 ]]; then # CREATE STAFF USERS # Prepares staff users for Django admin python manage.py create_staffusers + gonogo "create_staffusers" ##### # LAUNCH THE APP diff --git a/backend/config/staffusers.json b/backend/config/staffusers.json index 53f661a722..d378df350b 100644 --- a/backend/config/staffusers.json +++ b/backend/config/staffusers.json @@ -1,7 +1,8 @@ { "readonly": [ "philip.dominguez@gsa.gov", - "robert.novak@gsa.gov" + "robert.novak@gsa.gov", + "alexander.steel@gsa.gov" ], "helpdesk": [ "analyn.delossantos@gsa.gov", diff --git a/backend/dissemination/sql/fac-snapshot-db/post/010_copy_dissem_tables.sql b/backend/dissemination/sql/fac-snapshot-db/post/010_copy_dissem_tables.sql index ae5e846050..c4df416ee1 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/010_copy_dissem_tables.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/010_copy_dissem_tables.sql @@ -150,7 +150,7 @@ CREATE OR REPLACE FUNCTION dissem_copy.create_dissemination_secondaryauditor() LANGUAGE plpgsql; DO LANGUAGE plpgsql -$GATE$ +$go$ BEGIN RAISE info 'create_dissemination_additionalein'; PERFORM dissem_copy.create_dissemination_additionalein(); @@ -177,5 +177,5 @@ $GATE$ RAISE info 'create_dissemination_secondaryauditor'; PERFORM dissem_copy.create_dissemination_secondaryauditor(); END -$GATE$; +$go$; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/020_public_data_v1_0_0_tables.sql b/backend/dissemination/sql/fac-snapshot-db/post/020_public_data_v1_0_0_tables.sql index 93d9915936..061873884f 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/020_public_data_v1_0_0_tables.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/020_public_data_v1_0_0_tables.sql @@ -704,47 +704,33 @@ CREATE OR REPLACE FUNCTION public_data_v1_0_0_functions.create_invalid_audit_rec -- need this to run. ----------------------------------------------------------- DO LANGUAGE plpgsql -$GATE$ - DECLARE - the_schema varchar := 'public_data_v1_0_0'; - the_table varchar := 'metadata'; - api_ver varchar := 'api_v2_0_0'; - BEGIN - IF EXISTS ( - SELECT FROM pg_tables - WHERE schemaname = the_schema - AND tablename = the_table - ) - THEN - RAISE info '% Gate condition met. Skipping table creation.', api_ver; - ELSE - RAISE info '% %.% not found. Creating tables', api_ver, the_schema, the_table; - RAISE info 'Creating general'; - PERFORM public_data_v1_0_0_functions.create_general(); - RAISE info 'Creating additional_eins'; - PERFORM public_data_v1_0_0_functions.create_additional_eins(); - RAISE info 'Creating additional_ueis'; - PERFORM public_data_v1_0_0_functions.create_additional_ueis(); - RAISE info 'Creating corrective_action_plans'; - PERFORM public_data_v1_0_0_functions.create_corrective_action_plans(); - RAISE info 'Creating federal_awards'; - PERFORM public_data_v1_0_0_functions.create_federal_awards(); - RAISE info 'Creating findings'; - PERFORM public_data_v1_0_0_functions.create_findings(); - RAISE info 'Creating findings_text'; - PERFORM public_data_v1_0_0_functions.create_findings_text(); - RAISE info 'Creating notes_to_sefa'; - PERFORM public_data_v1_0_0_functions.create_notes_to_sefa(); - RAISE info 'Creating passthrough'; - PERFORM public_data_v1_0_0_functions.create_passthrough(); - RAISE info 'Creating secondary_auditors'; - PERFORM public_data_v1_0_0_functions.create_secondary_auditors(); - RAISE info 'Creating combined'; - PERFORM public_data_v1_0_0_functions.create_combined(); - RAISE info 'Creating migration_inspection_record'; - PERFORM public_data_v1_0_0_functions.create_migration_inspection_record(); - RAISE info 'Create invalid_audit_record'; - PERFORM public_data_v1_0_0_functions.create_invalid_audit_record(); - END IF; - END -$GATE$; +$GO$ + BEGIN + RAISE info 'Creating general'; + PERFORM public_data_v1_0_0_functions.create_general(); + RAISE info 'Creating additional_eins'; + PERFORM public_data_v1_0_0_functions.create_additional_eins(); + RAISE info 'Creating additional_ueis'; + PERFORM public_data_v1_0_0_functions.create_additional_ueis(); + RAISE info 'Creating corrective_action_plans'; + PERFORM public_data_v1_0_0_functions.create_corrective_action_plans(); + RAISE info 'Creating federal_awards'; + PERFORM public_data_v1_0_0_functions.create_federal_awards(); + RAISE info 'Creating findings'; + PERFORM public_data_v1_0_0_functions.create_findings(); + RAISE info 'Creating findings_text'; + PERFORM public_data_v1_0_0_functions.create_findings_text(); + RAISE info 'Creating notes_to_sefa'; + PERFORM public_data_v1_0_0_functions.create_notes_to_sefa(); + RAISE info 'Creating passthrough'; + PERFORM public_data_v1_0_0_functions.create_passthrough(); + RAISE info 'Creating secondary_auditors'; + PERFORM public_data_v1_0_0_functions.create_secondary_auditors(); + RAISE info 'Creating combined'; + PERFORM public_data_v1_0_0_functions.create_combined(); + RAISE info 'Creating migration_inspection_record'; + PERFORM public_data_v1_0_0_functions.create_migration_inspection_record(); + RAISE info 'Create invalid_audit_record'; + PERFORM public_data_v1_0_0_functions.create_invalid_audit_record(); + END +$GO$; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/021_suppressed_data_v1_0_0_tables.sql b/backend/dissemination/sql/fac-snapshot-db/post/021_suppressed_data_v1_0_0_tables.sql index 39708881fd..dfea791ee1 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/021_suppressed_data_v1_0_0_tables.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/021_suppressed_data_v1_0_0_tables.sql @@ -189,30 +189,17 @@ CREATE OR REPLACE FUNCTION suppressed_data_v1_0_0.create_invalid_audit_record() -- CONDITIONAL TABLE CREATION ----------------------------------------------------------- DO LANGUAGE plpgsql -$GATE$ - DECLARE - the_schema varchar := 'public_data_v1_0_0'; - the_table varchar := 'metadata'; - BEGIN - IF EXISTS ( - SELECT FROM pg_tables - WHERE schemaname = the_schema - AND tablename = the_table - ) - THEN - RAISE info 'Found the metadata table; skipping suppressed data table creation.'; - ELSE - RAISE info 'Metadata table not found. Creating suppressed data tables.'; - RAISE info 'Creating suppressed corrective_action_plans'; - PERFORM suppressed_data_v1_0_0.create_corrective_action_plans(); - RAISE info 'Creating suppressed findings_text'; - PERFORM suppressed_data_v1_0_0.create_findings_text(); - RAISE info 'Creating suppressed notes_to_sefa'; - PERFORM suppressed_data_v1_0_0.create_notes_to_sefa(); - RAISE info 'Creating migration_inspection_record'; - PERFORM suppressed_data_v1_0_0.create_migration_inspection_record(); - RAISE info 'Create invalid_audit_record'; - PERFORM suppressed_data_v1_0_0.create_invalid_audit_record(); - END IF; - END -$GATE$; +$GO$ + BEGIN + RAISE info 'Creating suppressed corrective_action_plans'; + PERFORM suppressed_data_v1_0_0.create_corrective_action_plans(); + RAISE info 'Creating suppressed findings_text'; + PERFORM suppressed_data_v1_0_0.create_findings_text(); + RAISE info 'Creating suppressed notes_to_sefa'; + PERFORM suppressed_data_v1_0_0.create_notes_to_sefa(); + RAISE info 'Creating migration_inspection_record'; + PERFORM suppressed_data_v1_0_0.create_migration_inspection_record(); + RAISE info 'Create invalid_audit_record'; + PERFORM suppressed_data_v1_0_0.create_invalid_audit_record(); + END +$GO$; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/030_public_data_v1_0_0_metadata.sql b/backend/dissemination/sql/fac-snapshot-db/post/030_public_data_v1_0_0_metadata.sql index c0940c1d4f..d7b30d2c5a 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/030_public_data_v1_0_0_metadata.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/030_public_data_v1_0_0_metadata.sql @@ -71,21 +71,9 @@ $ct$ LANGUAGE plpgsql; DO LANGUAGE plpgsql -$GATE$ - DECLARE - the_schema varchar := 'public_data_v1_0_0'; - the_table varchar := 'metadata'; - BEGIN - IF EXISTS ( - SELECT FROM pg_tables - WHERE schemaname = the_schema - AND tablename = the_table - ) - THEN - RAISE info 'Gate condition met. Skipping metadata table creation.'; - ELSE - RAISE info 'Creating metadata table'; - PERFORM public_data_v1_0_0.create_metadata(); - END IF; - END -$GATE$; +$GO$ + BEGIN + RAISE info 'Creating metadata table'; + PERFORM public_data_v1_0_0.create_metadata(); + END +$GO$; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/800_permissions.sql b/backend/dissemination/sql/fac-snapshot-db/post/800_permissions.sql index 0a8fad276d..dca062baef 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/800_permissions.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/800_permissions.sql @@ -6,10 +6,3 @@ GRANT SELECT ON ALL TABLES IN SCHEMA api_v1_1_0 TO api_fac_gov; GRANT SELECT ON ALL TABLES IN SCHEMA api_v2_0_0 TO api_fac_gov; --- GRANT SELECT ON SEQUENCES IN SCHEMA public_data_v1_0_0; --- -- SEQ: USAGE | SELECT | UPDATE --- ALTER DEFAULT PRIVILEGES --- IN SCHEMA public_data_v1_0_0 --- GRANT SELECT, USAGE --- ON SEQUENCES --- TO api_fac_gov; diff --git a/backend/dissemination/sql/fac-snapshot-db/pre/060_schemas.sql b/backend/dissemination/sql/fac-snapshot-db/pre/060_schemas.sql index 89baa719dd..dac5ab3d96 100644 --- a/backend/dissemination/sql/fac-snapshot-db/pre/060_schemas.sql +++ b/backend/dissemination/sql/fac-snapshot-db/pre/060_schemas.sql @@ -21,33 +21,7 @@ DROP SCHEMA IF EXISTS api_v2_0_0_functions CASCADE; -- fac-snapshot-db. Always drop the schema and make a new copy. ------------- DROP SCHEMA IF EXISTS dissem_copy CASCADE; - ------------------------------------------------------------------- --- CONDITIONAL ------------------------------------------------------------------- --- On deploys, we don't want to tear down the public data. --- So, we check to see if the metadata table is present, and if so, --- leave the schemas in place. -DO LANGUAGE plpgsql -$GATE$ - DECLARE - the_schema varchar := 'public_data_v1_0_0'; - the_table varchar := 'metadata'; - BEGIN - IF EXISTS ( - SELECT FROM pg_tables - WHERE schemaname = the_schema - AND tablename = the_table - ) - THEN - RAISE info 'The metadata table exists; leaving public data schemas in place.'; - ELSE - RAISE info 'The metadata table does not exist; dropping schemas for a clean rebuild.'; - DROP SCHEMA IF EXISTS public_data_v1_0_0 CASCADE; - DROP SCHEMA IF EXISTS suppressed_data_v1_0_0 CASCADE; - END IF; - END -$GATE$; +DROP SCHEMA IF EXISTS public_data_v1_0_0 CASCADE; --------------------------------- -- CREATE diff --git a/backend/tools/sql_pre_post.sh b/backend/tools/sql_pre_post.sh index 642aa7a73a..84750af65c 100644 --- a/backend/tools/sql_pre_post.sh +++ b/backend/tools/sql_pre_post.sh @@ -30,11 +30,31 @@ function sql_pre { run_sql_files $FAC_SNAPSHOT_URI "SQL_PRE" "pre" } +function sql_pre_fac_db { + run_sql_files $FAC_DB_URI "SQL_PRE" "pre" +} + +function sql_pre_fac_snapshot_db { + run_sql_files $FAC_SNAPSHOT_URI "SQL_PRE" "pre" +} + + + function sql_post { run_sql_files $FAC_DB_URI "SQL_POST" "post" run_sql_files $FAC_SNAPSHOT_URI "SQL_POST" "post" } + +function sql_post_fac_db { + run_sql_files $FAC_DB_URI "SQL_POST" "post" +} + + +function sql_post_fac_snapshot_db { + run_sql_files $FAC_SNAPSHOT_URI "SQL_POST" "post" +} + function vacuum_snapshot_db { # Vacuum things when we're done. # Cannot run inside a transaction. diff --git a/backend/util/nightly_api_refresh.sh b/backend/util/nightly_api_refresh.sh new file mode 100644 index 0000000000..0df7a0e320 --- /dev/null +++ b/backend/util/nightly_api_refresh.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -e +source tools/util_startup.sh +source tools/setup_env.sh +source tools/sql_pre_post.sh +setup_env + +# Run an RDS backup, to refresh dissemination_* in FAC_SNAPSHOT from FAC_DB +./../fac-backup-util.sh "v0.1.8" "rds_backup" +gonogo "fac-backup-util" + +# Run the pre/post. +sql_pre_fac_snapshot_db +gonogo "sql_pre_fac_snapshot_db" + +sql_post_fac_snapshot_db +gonogo "sql_post_fac_snapshot_db" From 7abe12ae1586f04320e8bf162174198a3bc8d422 Mon Sep 17 00:00:00 2001 From: Alex Steel <130377221+asteel-gsa@users.noreply.github.com> Date: Tue, 29 Oct 2024 12:43:34 -0400 Subject: [PATCH 59/89] Prep API Standup Test --- ...uler.yml => fac-api-standup-scheduler.yml} | 9 ++++--- ...ling-scheduled.yml => fac-api-standup.yml} | 25 +++++++++++-------- backend/util/nightly_api_refresh.sh | 3 ++- 3 files changed, 22 insertions(+), 15 deletions(-) rename .github/workflows/{fac-backup-scheduler.yml => fac-api-standup-scheduler.yml} (65%) rename .github/workflows/{fac-backup-and-sling-scheduled.yml => fac-api-standup.yml} (50%) diff --git a/.github/workflows/fac-backup-scheduler.yml b/.github/workflows/fac-api-standup-scheduler.yml similarity index 65% rename from .github/workflows/fac-backup-scheduler.yml rename to .github/workflows/fac-api-standup-scheduler.yml index 3daea83415..d25a589eaa 100644 --- a/.github/workflows/fac-backup-scheduler.yml +++ b/.github/workflows/fac-api-standup-scheduler.yml @@ -3,11 +3,11 @@ name: Backup Environments on: schedule: # Invoke every 2 hours - - cron: '0 12,14,16,18,20,22/2 * * *' + - cron: '0 2 * * *' workflow_dispatch: null jobs: - backup-environment: + scheduled-api-standup: strategy: fail-fast: false matrix: @@ -15,10 +15,11 @@ jobs: - name: dev - name: staging - name: production - uses: ./.github/workflows/fac-backup-util-scheduled.yml + - name: preview + uses: ./.github/workflows/fac-api-standup.yml secrets: inherit with: environment: ${{ matrix.environment.name }} util_version: "v0.1.8" - backup_operation: "scheduled_backup" + backup_operation: "rds_backup" diff --git a/.github/workflows/fac-backup-and-sling-scheduled.yml b/.github/workflows/fac-api-standup.yml similarity index 50% rename from .github/workflows/fac-backup-and-sling-scheduled.yml rename to .github/workflows/fac-api-standup.yml index ac4fe09c59..40f6035314 100644 --- a/.github/workflows/fac-backup-and-sling-scheduled.yml +++ b/.github/workflows/fac-api-standup.yml @@ -1,8 +1,8 @@ --- -name: Backup the database with fac-backup-utility +name: Standup the API ### Common Commands: -# ./fac-backup-util.sh v0.1.5 scheduled_backup -# ./fac-backup-util.sh v0.1.5 daily_backup +# ./fac-backup-util.sh v0.1.8 scheduled_backup +# ./fac-backup-util.sh v0.1.8 daily_backup on: workflow_call: inputs: @@ -17,13 +17,17 @@ on: description: Operation for fac-backup-utility required: true type: string + # push: + # branches: [jadudm/api-perf] jobs: - fac-backup-scheduled: - name: Perform Database Backup + api-standup: + name: Standup API runs-on: ubuntu-latest - environment: ${{ inputs.environment }} + #environment: ${{ inputs.environment }} + environment: preview env: - space: ${{ inputs.environment }} + #space: ${{ inputs.environment }} + space: preview steps: - name: Backup FAC Database, Sync Media, and sling API tables uses: cloud-gov/cg-cli-tools@main @@ -31,6 +35,7 @@ jobs: cf_username: ${{ secrets.CF_USERNAME }} cf_password: ${{ secrets.CF_PASSWORD }} cf_org: gsa-tts-oros-fac - cf_space: ${{ env.space }} - command: cf run-task gsa-fac -k 7G -m 3G --name backup_and_sling_scheduled --command "./util/fac-backup-and-sling-scheduled.sh ${{ inputs.util_version }} ${{ inputs.backup_operation }}" - + #cf_space: ${{ env.space }} + cf_space: preview + #command: cf run-task gsa-fac -k 3G -m 3G --name standup_api --command "./util/nightly_api_refresh.sh ${{ inputs.util_version }} ${{ inputs.backup_operation }}" + command: cf run-task gsa-fac -k 2G -m 2G --name standup_api --command "./util/nightly_api_refresh.sh v0.1.8 rds_backup" diff --git a/backend/util/nightly_api_refresh.sh b/backend/util/nightly_api_refresh.sh index 0df7a0e320..4e4d379081 100644 --- a/backend/util/nightly_api_refresh.sh +++ b/backend/util/nightly_api_refresh.sh @@ -6,7 +6,8 @@ source tools/sql_pre_post.sh setup_env # Run an RDS backup, to refresh dissemination_* in FAC_SNAPSHOT from FAC_DB -./../fac-backup-util.sh "v0.1.8" "rds_backup" +#./../fac-backup-util.sh "v0.1.8" "rds_backup" +./../fac-backup-util.sh "$1" "$2" gonogo "fac-backup-util" # Run the pre/post. From decc89a80773eea90507b71066cba3498a6a44d8 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Tue, 29 Oct 2024 12:48:50 -0400 Subject: [PATCH 60/89] Fixes run.sh Trying to figure out a bug in staffusers... :w --- .../sql/fac-snapshot-db/pre/060_schemas.sql | 1 + backend/run.sh | 11 +++++++++-- .../users/management/commands/create_staffusers.py | 12 +++++++++--- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/backend/dissemination/sql/fac-snapshot-db/pre/060_schemas.sql b/backend/dissemination/sql/fac-snapshot-db/pre/060_schemas.sql index dac5ab3d96..5c1215e432 100644 --- a/backend/dissemination/sql/fac-snapshot-db/pre/060_schemas.sql +++ b/backend/dissemination/sql/fac-snapshot-db/pre/060_schemas.sql @@ -22,6 +22,7 @@ DROP SCHEMA IF EXISTS api_v2_0_0_functions CASCADE; ------------- DROP SCHEMA IF EXISTS dissem_copy CASCADE; DROP SCHEMA IF EXISTS public_data_v1_0_0 CASCADE; +DROP SCHEMA IF EXISTS suppressed_data_v1_0_0 CASCADE; --------------------------------- -- CREATE diff --git a/backend/run.sh b/backend/run.sh index 483df37c89..0407815c0a 100755 --- a/backend/run.sh +++ b/backend/run.sh @@ -30,11 +30,18 @@ gonogo "setup_env" cgov_util_local_only gonogo "cgov_util_local_only" +##### +# SIMULATE NIGHTLY DATA/API CREATION +sql_pre_fac_snapshot_db +gonogo "sql_pre_fac_snapshot_db" +sql_post_fac_snapshot_db +gonogo "sql_post_fac_snapshot_db" + ##### # SQL PRE # We have SQL that we want to run before the migrations and sling are run. # This tears down things that would conflict with migrations, etc. -sql_pre +sql_pre_fac_db gonogo "sql_pre" curation_audit_tracking_disable gonogo "curation_audit_tracking_disable" @@ -48,7 +55,7 @@ gonogo "migrate_app_tables" # SQL POST # Rebuild the API and prepare the system for execution. # Runs after migrations. -sql_post +sql_post_fac_db gonogo "sql_post" ##### diff --git a/backend/users/management/commands/create_staffusers.py b/backend/users/management/commands/create_staffusers.py index 6c44212385..591bd4aa98 100644 --- a/backend/users/management/commands/create_staffusers.py +++ b/backend/users/management/commands/create_staffusers.py @@ -86,9 +86,15 @@ def handle(self, *args, **kwargs): # create staff user for each role. with transaction.atomic(): - StaffUser( - staff_email=email, - ).save() + try: + StaffUser( + staff_email=email, + ).save() + except Exception as e: + transaction.set_rollback(True) + logger.warning( + f"Could not create new StaffUser for {email}" + ) # attempt to update the user. try: From 337ea0d613f79eef40c0c16f77d78baec0f6959e Mon Sep 17 00:00:00 2001 From: Bobby Novak <176936850+rnovak338@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:21:42 -0400 Subject: [PATCH 61/89] Update Cypress E2E API checks - Fixed an issue where Cypress was checking an undefined method type `HEAD` for the existence of the PDF report. - Now, we are just retrieving it with `GET` and checking for a valid status code. --- .../support/dissemination-table-via-pdf.js | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/backend/cypress/support/dissemination-table-via-pdf.js b/backend/cypress/support/dissemination-table-via-pdf.js index e331b61efa..d0df9bfbec 100644 --- a/backend/cypress/support/dissemination-table-via-pdf.js +++ b/backend/cypress/support/dissemination-table-via-pdf.js @@ -72,9 +72,12 @@ export function testSubmissionAccessViaPDF(reportId, isTribal, isPublic) { expect(isPublic).to.be.true // We should be able to grab the PDF by URL // https://app.fac.gov/dissemination/report/pdf/2023-04-GSAFAC-0000050825 - cy.request('HEAD', '/dissemination/report/pdf/' + reportId); - - + cy.request({ + method: 'GET', + url: '/dissemination/report/pdf/' + reportId + }).should((response) => { + expect(response.isOkStatusCode).to.equal(true); + }); } //////////////////////////////////////// // The audit IS tribal and IS NOT public @@ -82,7 +85,14 @@ export function testSubmissionAccessViaPDF(reportId, isTribal, isPublic) { else if (isTribal && !isPublic) { expect(isTribal).to.be.true expect(isPublic).to.be.false - cy.request('HEAD', '/dissemination/report/pdf/' + reportId); + cy.request({ + method: 'GET', + url: '/dissemination/report/pdf/' + reportId, + failOnStatusCode: false, + }).should((response) => { + expect(response.isOkStatusCode).to.equal(false); + }); + } //////////////////////////////////////// // The audit IS NOT tribal and IS public @@ -92,7 +102,12 @@ export function testSubmissionAccessViaPDF(reportId, isTribal, isPublic) { expect(isTribal).to.be.false expect(isPublic).to.be.true // We should always find it in all endpoints, priv or unpriv. - cy.request('HEAD', '/dissemination/report/pdf/' + reportId); + cy.request({ + method: 'GET', + url: '/dissemination/report/pdf/' + reportId + }).should((response) => { + expect(response.isOkStatusCode).to.equal(true); + }); } //////////////////////////////////////// // The audit IS NOT tribal and IS NOT public From f2ae9deef8028477e844bd3cfe5f04a8d983fb70 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Tue, 29 Oct 2024 13:53:13 -0400 Subject: [PATCH 62/89] Changing apparent error to warning --- backend/.profile | 6 ------ .../management/commands/create_staffusers.py | 16 +++++----------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/backend/.profile b/backend/.profile index bf157d9c57..c1952d2597 100644 --- a/backend/.profile +++ b/backend/.profile @@ -49,12 +49,6 @@ if [[ "$CF_INSTANCE_INDEX" == 0 ]]; then # Prepares staff users for Django admin python manage.py create_staffusers gonogo "create_staffusers" - - ##### - # LAUNCH THE APP - # We will have died long ago if things didn't work. - npm run dev & python manage.py runserver 0.0.0.0:8000 - fi # Make psql usable by scripts, for debugging, etc. diff --git a/backend/users/management/commands/create_staffusers.py b/backend/users/management/commands/create_staffusers.py index 591bd4aa98..0a77188d5a 100644 --- a/backend/users/management/commands/create_staffusers.py +++ b/backend/users/management/commands/create_staffusers.py @@ -86,15 +86,9 @@ def handle(self, *args, **kwargs): # create staff user for each role. with transaction.atomic(): - try: - StaffUser( - staff_email=email, - ).save() - except Exception as e: - transaction.set_rollback(True) - logger.warning( - f"Could not create new StaffUser for {email}" - ) + StaffUser( + staff_email=email, + ).save() # attempt to update the user. try: @@ -114,8 +108,8 @@ def handle(self, *args, **kwargs): logger.info(f"Synced {email} to a StaffUser role.") # for whatever reason, this failed. Revert staffuser creation. - except Exception as e: + except User.DoesNotExist: transaction.set_rollback(True) logger.warning( - f"Failed to create a StaffUser for {email} ({e})." + f"StaffUser not created for {email}, they have not logged in yet." ) From df8236dc598d7eb6edc103b1b35c448d5208586a Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Tue, 29 Oct 2024 14:09:11 -0400 Subject: [PATCH 63/89] Fixing email, whitespace --- backend/config/staffusers.json | 6 +++--- backend/tools/sql_pre_post.sh | 15 --------------- 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/backend/config/staffusers.json b/backend/config/staffusers.json index d378df350b..fd8f32c2c9 100644 --- a/backend/config/staffusers.json +++ b/backend/config/staffusers.json @@ -1,10 +1,10 @@ { "readonly": [ - "philip.dominguez@gsa.gov", - "robert.novak@gsa.gov", "alexander.steel@gsa.gov" ], "helpdesk": [ + "philip.dominguez@gsa.gov", + "robert.novak@gsa.gov", "analyn.delossantos@gsa.gov", "hassandeme.mamasambo@gsa.gov", "iman.ali@gsa.gov", @@ -16,6 +16,6 @@ ], "superuser": [ "matthew.jadud@gsa.gov", - "dan.swick@gsa.gov" + "daniel.swick@gsa.gov" ] } diff --git a/backend/tools/sql_pre_post.sh b/backend/tools/sql_pre_post.sh index 84750af65c..c4503468a8 100644 --- a/backend/tools/sql_pre_post.sh +++ b/backend/tools/sql_pre_post.sh @@ -24,12 +24,6 @@ function run_sql_files { return 0 } - -function sql_pre { - run_sql_files $FAC_DB_URI "SQL_PRE" "pre" - run_sql_files $FAC_SNAPSHOT_URI "SQL_PRE" "pre" -} - function sql_pre_fac_db { run_sql_files $FAC_DB_URI "SQL_PRE" "pre" } @@ -38,19 +32,10 @@ function sql_pre_fac_snapshot_db { run_sql_files $FAC_SNAPSHOT_URI "SQL_PRE" "pre" } - - -function sql_post { - run_sql_files $FAC_DB_URI "SQL_POST" "post" - run_sql_files $FAC_SNAPSHOT_URI "SQL_POST" "post" -} - - function sql_post_fac_db { run_sql_files $FAC_DB_URI "SQL_POST" "post" } - function sql_post_fac_snapshot_db { run_sql_files $FAC_SNAPSHOT_URI "SQL_POST" "post" } From f2055c5189c70303bdd257c24686b1278106ed65 Mon Sep 17 00:00:00 2001 From: Alex Steel <130377221+asteel-gsa@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:01:25 -0400 Subject: [PATCH 64/89] Test api refresh --- .github/workflows/fac-api-standup.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/fac-api-standup.yml b/.github/workflows/fac-api-standup.yml index 40f6035314..85d639cc88 100644 --- a/.github/workflows/fac-api-standup.yml +++ b/.github/workflows/fac-api-standup.yml @@ -17,8 +17,8 @@ on: description: Operation for fac-backup-utility required: true type: string - # push: - # branches: [jadudm/api-perf] + push: + branches: [jadudm/api-perf] jobs: api-standup: name: Standup API @@ -38,4 +38,4 @@ jobs: #cf_space: ${{ env.space }} cf_space: preview #command: cf run-task gsa-fac -k 3G -m 3G --name standup_api --command "./util/nightly_api_refresh.sh ${{ inputs.util_version }} ${{ inputs.backup_operation }}" - command: cf run-task gsa-fac -k 2G -m 2G --name standup_api --command "./util/nightly_api_refresh.sh v0.1.8 rds_backup" + command: cf run-task gsa-fac -k 2G -m 2G --name api_refresh --command "./util/nightly_api_refresh.sh v0.1.8 rds_backup" From 825a3e04ba777672540b730ad61b32dce4c2c8ea Mon Sep 17 00:00:00 2001 From: Alex Steel <130377221+asteel-gsa@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:03:29 -0400 Subject: [PATCH 65/89] Api chmod +x --- .github/workflows/fac-api-standup.yml | 2 +- backend/util/nightly_api_refresh.sh | 0 2 files changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 backend/util/nightly_api_refresh.sh diff --git a/.github/workflows/fac-api-standup.yml b/.github/workflows/fac-api-standup.yml index 85d639cc88..114a7e7775 100644 --- a/.github/workflows/fac-api-standup.yml +++ b/.github/workflows/fac-api-standup.yml @@ -29,7 +29,7 @@ jobs: #space: ${{ inputs.environment }} space: preview steps: - - name: Backup FAC Database, Sync Media, and sling API tables + - name: Backup FAC Database and refresh the api uses: cloud-gov/cg-cli-tools@main with: cf_username: ${{ secrets.CF_USERNAME }} diff --git a/backend/util/nightly_api_refresh.sh b/backend/util/nightly_api_refresh.sh old mode 100644 new mode 100755 From 814a5a743976bdb0b49905978eed7580d8152232 Mon Sep 17 00:00:00 2001 From: Alex Steel <130377221+asteel-gsa@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:05:58 -0400 Subject: [PATCH 66/89] Remove historical data load --- .github/workflows/deploy-application.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/deploy-application.yml b/.github/workflows/deploy-application.yml index 4f811e4a0c..67a9288b94 100644 --- a/.github/workflows/deploy-application.yml +++ b/.github/workflows/deploy-application.yml @@ -103,15 +103,6 @@ jobs: cf_vars_file: backend/manifests/vars/vars-${{ env.space }}.yml command: bin/ops/deploy.sh - - name: Load historical data - uses: cloud-gov/cg-cli-tools@main - with: - cf_username: ${{ secrets.CF_USERNAME }} - cf_password: ${{ secrets.CF_PASSWORD }} - cf_org: gsa-tts-oros-fac - cf_space: ${{ env.space }} - command: cf run-task gsa-fac -k 6G -m 1G --name load_data --command "./load_data.sh" - # This has to happen after an application deployment because the manifest (currently) is responsible # for binding the "logdrain service" to the "gsa-fac application". This also needs to be done # based on the suspicion that fluentbit cannot register the incoming logs when it is initially From 72a86a046762a9877960c2263dd559e19d832b02 Mon Sep 17 00:00:00 2001 From: Alex Steel <130377221+asteel-gsa@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:19:09 -0400 Subject: [PATCH 67/89] Push to run api_refresh --- .github/workflows/fac-api-standup.yml | 20 ++++++++++++++++++++ .github/workflows/fac-backup-util.yml | 1 + 2 files changed, 21 insertions(+) diff --git a/.github/workflows/fac-api-standup.yml b/.github/workflows/fac-api-standup.yml index 114a7e7775..7b77845b51 100644 --- a/.github/workflows/fac-api-standup.yml +++ b/.github/workflows/fac-api-standup.yml @@ -17,6 +17,26 @@ on: description: Operation for fac-backup-utility required: true type: string + workflow_dispatch: + inputs: + environment: + required: true + type: choice + options: + - 'dev' + - 'preview' + - 'staging' + - 'production' + util_version: + description: Version for fac backup utility to use (ex. vX.Y.Z) + required: true + type: string + backup_operation: + description: Operation for fac-backup-utility + required: true + type: choice + options: + - 'rds_backup' push: branches: [jadudm/api-perf] jobs: diff --git a/.github/workflows/fac-backup-util.yml b/.github/workflows/fac-backup-util.yml index 47eed32461..8e9bf804da 100644 --- a/.github/workflows/fac-backup-util.yml +++ b/.github/workflows/fac-backup-util.yml @@ -18,6 +18,7 @@ on: description: Version for fac backup utility to use (ex. vX.Y.Z) required: true type: string + default: "v0.1.8" backup_operation: description: Operation for fac-backup-utility required: true From 04970aaebaef9c83b0d2c0961c9e8f9a7180a1ed Mon Sep 17 00:00:00 2001 From: Alex Steel <130377221+asteel-gsa@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:33:46 -0400 Subject: [PATCH 68/89] Use pushd & popd to put us where we need to be for source & other utils --- backend/util/nightly_api_refresh.sh | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/backend/util/nightly_api_refresh.sh b/backend/util/nightly_api_refresh.sh index 4e4d379081..d6a1aec314 100755 --- a/backend/util/nightly_api_refresh.sh +++ b/backend/util/nightly_api_refresh.sh @@ -1,15 +1,17 @@ #!/bin/bash set -e -source tools/util_startup.sh -source tools/setup_env.sh -source tools/sql_pre_post.sh -setup_env +pushd .. + source tools/util_startup.sh + source tools/setup_env.sh + source tools/sql_pre_post.sh + setup_env -# Run an RDS backup, to refresh dissemination_* in FAC_SNAPSHOT from FAC_DB -#./../fac-backup-util.sh "v0.1.8" "rds_backup" -./../fac-backup-util.sh "$1" "$2" -gonogo "fac-backup-util" + # Run an RDS backup, to refresh dissemination_* in FAC_SNAPSHOT from FAC_DB + #./../fac-backup-util.sh "v0.1.8" "rds_backup" + ./fac-backup-util.sh "$1" "$2" + gonogo "fac-backup-util" +popd # Run the pre/post. sql_pre_fac_snapshot_db gonogo "sql_pre_fac_snapshot_db" From d2d910d956c4010631deab2bd421f85a6bcd474c Mon Sep 17 00:00:00 2001 From: Alex Steel <130377221+asteel-gsa@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:47:35 -0400 Subject: [PATCH 69/89] API Refresh test --- backend/util/nightly_api_refresh.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/util/nightly_api_refresh.sh b/backend/util/nightly_api_refresh.sh index d6a1aec314..a34a993129 100755 --- a/backend/util/nightly_api_refresh.sh +++ b/backend/util/nightly_api_refresh.sh @@ -8,6 +8,7 @@ pushd .. # Run an RDS backup, to refresh dissemination_* in FAC_SNAPSHOT from FAC_DB #./../fac-backup-util.sh "v0.1.8" "rds_backup" + # Deploy test ./fac-backup-util.sh "$1" "$2" gonogo "fac-backup-util" From 53ede5ae71bb52fbde0cbc56d6fc543a6ceb6034 Mon Sep 17 00:00:00 2001 From: Alex Steel <130377221+asteel-gsa@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:53:51 -0400 Subject: [PATCH 70/89] Overthinking bash pathing --- backend/util/nightly_api_refresh.sh | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/backend/util/nightly_api_refresh.sh b/backend/util/nightly_api_refresh.sh index a34a993129..bc2a36ba37 100755 --- a/backend/util/nightly_api_refresh.sh +++ b/backend/util/nightly_api_refresh.sh @@ -1,18 +1,19 @@ #!/bin/bash set -e -pushd .. - source tools/util_startup.sh - source tools/setup_env.sh - source tools/sql_pre_post.sh - setup_env - # Run an RDS backup, to refresh dissemination_* in FAC_SNAPSHOT from FAC_DB - #./../fac-backup-util.sh "v0.1.8" "rds_backup" - # Deploy test - ./fac-backup-util.sh "$1" "$2" - gonogo "fac-backup-util" +source tools/util_startup.sh +source tools/setup_env.sh +source tools/sql_pre_post.sh +setup_env + +# Run an RDS backup, to refresh dissemination_* in FAC_SNAPSHOT from FAC_DB +#./../fac-backup-util.sh "v0.1.8" "rds_backup" +# Deploy test +echo "$pwd" +./fac-backup-util.sh "$1" "$2" +gonogo "fac-backup-util" + -popd # Run the pre/post. sql_pre_fac_snapshot_db gonogo "sql_pre_fac_snapshot_db" From 5f8a7ec5a952503571d388240874df03d08ba4e7 Mon Sep 17 00:00:00 2001 From: Alex Steel <130377221+asteel-gsa@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:56:43 -0400 Subject: [PATCH 71/89] Debugging typo removal --- backend/util/nightly_api_refresh.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/util/nightly_api_refresh.sh b/backend/util/nightly_api_refresh.sh index bc2a36ba37..e3f32c26b9 100755 --- a/backend/util/nightly_api_refresh.sh +++ b/backend/util/nightly_api_refresh.sh @@ -9,7 +9,6 @@ setup_env # Run an RDS backup, to refresh dissemination_* in FAC_SNAPSHOT from FAC_DB #./../fac-backup-util.sh "v0.1.8" "rds_backup" # Deploy test -echo "$pwd" ./fac-backup-util.sh "$1" "$2" gonogo "fac-backup-util" From 9b361e900ffd71b80e580f5ed9a43364a47b19a8 Mon Sep 17 00:00:00 2001 From: Alex Steel <130377221+asteel-gsa@users.noreply.github.com> Date: Tue, 29 Oct 2024 16:00:35 -0400 Subject: [PATCH 72/89] Version bump backup-util to v0.1.9 --- .github/workflows/deploy-application.yml | 4 +-- .../workflows/fac-api-standup-scheduler.yml | 2 +- .github/workflows/fac-api-standup.yml | 7 +++-- .../workflows/fac-backup-util-scheduled.yml | 4 +-- .github/workflows/fac-backup-util.yml | 6 ++-- .../workflows/fac-check-tables-scheduler.yml | 2 +- .github/workflows/fac-check-tables.yml | 2 +- backend/tools/setup_local_env.sh | 28 +++++++++---------- .../util/load_public_dissem_data/Dockerfile | 6 ++-- .../util/load_public_dissem_data/README.md | 8 +++--- .../load_public_data_locally.sh | 4 +-- backend/util/nightly_api_refresh.sh | 2 +- docs/backups_and_restores.md | 10 +++---- 13 files changed, 43 insertions(+), 42 deletions(-) diff --git a/.github/workflows/deploy-application.yml b/.github/workflows/deploy-application.yml index 67a9288b94..dd19d7b79a 100644 --- a/.github/workflows/deploy-application.yml +++ b/.github/workflows/deploy-application.yml @@ -77,7 +77,7 @@ jobs: cf_password: ${{ secrets.CF_PASSWORD }} cf_org: gsa-tts-oros-fac cf_space: ${{ env.space }} - command: cf run-task gsa-fac -k 7G -m 3G --name deploy_backup --command "./fac-backup-util.sh v0.1.5 deploy_backup" + command: cf run-task gsa-fac -k 7G -m 3G --name deploy_backup --command "./fac-backup-util.sh v0.1.9 deploy_backup" - name: Deploy Preview to cloud.gov if: ${{ inputs.environment == 'preview' }} @@ -123,5 +123,5 @@ jobs: secrets: inherit with: environment: ${{ inputs.environment }} - util_version: "v0.1.8" + util_version: "v0.1.9" backup_operation: "check_tables" diff --git a/.github/workflows/fac-api-standup-scheduler.yml b/.github/workflows/fac-api-standup-scheduler.yml index d25a589eaa..85c04be326 100644 --- a/.github/workflows/fac-api-standup-scheduler.yml +++ b/.github/workflows/fac-api-standup-scheduler.yml @@ -20,6 +20,6 @@ jobs: secrets: inherit with: environment: ${{ matrix.environment.name }} - util_version: "v0.1.8" + util_version: "v0.1.9" backup_operation: "rds_backup" diff --git a/.github/workflows/fac-api-standup.yml b/.github/workflows/fac-api-standup.yml index 7b77845b51..580dbb6ac1 100644 --- a/.github/workflows/fac-api-standup.yml +++ b/.github/workflows/fac-api-standup.yml @@ -1,8 +1,8 @@ --- name: Standup the API ### Common Commands: -# ./fac-backup-util.sh v0.1.8 scheduled_backup -# ./fac-backup-util.sh v0.1.8 daily_backup +# ./fac-backup-util.sh v0.1.9 scheduled_backup +# ./fac-backup-util.sh v0.1.9 daily_backup on: workflow_call: inputs: @@ -31,6 +31,7 @@ on: description: Version for fac backup utility to use (ex. vX.Y.Z) required: true type: string + default: "v0.1.9" backup_operation: description: Operation for fac-backup-utility required: true @@ -58,4 +59,4 @@ jobs: #cf_space: ${{ env.space }} cf_space: preview #command: cf run-task gsa-fac -k 3G -m 3G --name standup_api --command "./util/nightly_api_refresh.sh ${{ inputs.util_version }} ${{ inputs.backup_operation }}" - command: cf run-task gsa-fac -k 2G -m 2G --name api_refresh --command "./util/nightly_api_refresh.sh v0.1.8 rds_backup" + command: cf run-task gsa-fac -k 2G -m 2G --name api_refresh --command "./util/nightly_api_refresh.sh v0.1.9 rds_backup" diff --git a/.github/workflows/fac-backup-util-scheduled.yml b/.github/workflows/fac-backup-util-scheduled.yml index d5992fac3c..4509e38fba 100644 --- a/.github/workflows/fac-backup-util-scheduled.yml +++ b/.github/workflows/fac-backup-util-scheduled.yml @@ -1,8 +1,8 @@ --- name: Backup the database with fac-backup-utility ### Common Commands: -# ./fac-backup-util.sh v0.1.8 scheduled_backup -# ./fac-backup-util.sh v0.1.8 daily_backup +# ./fac-backup-util.sh v0.1.9 scheduled_backup +# ./fac-backup-util.sh v0.1.9 daily_backup on: workflow_call: inputs: diff --git a/.github/workflows/fac-backup-util.yml b/.github/workflows/fac-backup-util.yml index 8e9bf804da..11a7caadc7 100644 --- a/.github/workflows/fac-backup-util.yml +++ b/.github/workflows/fac-backup-util.yml @@ -1,8 +1,8 @@ --- name: Backup the database with fac-backup-utility ### Common Commands: -# ./fac-backup-util.sh v0.1.8 initial_backup -# ./fac-backup-util.sh v0.1.8 deploy_backup +# ./fac-backup-util.sh v0.1.9 initial_backup +# ./fac-backup-util.sh v0.1.9 deploy_backup on: workflow_dispatch: inputs: @@ -18,7 +18,7 @@ on: description: Version for fac backup utility to use (ex. vX.Y.Z) required: true type: string - default: "v0.1.8" + default: "v0.1.9" backup_operation: description: Operation for fac-backup-utility required: true diff --git a/.github/workflows/fac-check-tables-scheduler.yml b/.github/workflows/fac-check-tables-scheduler.yml index 111ca0029e..a1b22b5593 100644 --- a/.github/workflows/fac-check-tables-scheduler.yml +++ b/.github/workflows/fac-check-tables-scheduler.yml @@ -20,5 +20,5 @@ jobs: secrets: inherit with: environment: ${{ matrix.environment.name }} - util_version: "v0.1.8" + util_version: "v0.1.9" backup_operation: "check_tables" diff --git a/.github/workflows/fac-check-tables.yml b/.github/workflows/fac-check-tables.yml index f45d0f6003..0ce9c2d146 100644 --- a/.github/workflows/fac-check-tables.yml +++ b/.github/workflows/fac-check-tables.yml @@ -1,7 +1,7 @@ --- name: Check existing tables in an environment ### Common Commands: -# ./fac-backup-util.sh v0.1.8 check_tables +# ./fac-backup-util.sh v0.1.9 check_tables on: workflow_dispatch: inputs: diff --git a/backend/tools/setup_local_env.sh b/backend/tools/setup_local_env.sh index d5f96d777a..2c9da76210 100644 --- a/backend/tools/setup_local_env.sh +++ b/backend/tools/setup_local_env.sh @@ -4,7 +4,7 @@ function setup_local_env { if [[ "${ENV}" == "LOCAL" || "${ENV}" == "TESTING" ]]; then startup_log "LOCAL_ENV" "We are in a local envirnoment." - + # Load a fake VCAP_SERVICES file into the environment variable, # so we can mimic the cloud.gov setup. export VCAP_SERVICES=$(cat config/vcap_services_for_containers.json) @@ -23,16 +23,16 @@ function setup_local_env { # export AWS_PRIVATE_BUCKET_NAME="fac-private-s3" export AWS_PRIVATE_BUCKET_NAME=$(echo $VCAP_SERVICES \ - | jq --raw-output '.s3 - | map(select(.instance_name - | contains("fac-private-s3"))) + | jq --raw-output '.s3 + | map(select(.instance_name + | contains("fac-private-s3"))) | .[] .credentials.bucket') check_env_var_not_empty "AWS_PRIVATE_BUCKET_NAME" - + export AWS_PUBLIC_BUCKET_NAME=$(echo $VCAP_SERVICES \ - | jq --raw-output '.s3 - | map(select(.instance_name - | contains("fac-public-s3"))) + | jq --raw-output '.s3 + | map(select(.instance_name + | contains("fac-public-s3"))) | .[] .credentials.bucket') @@ -51,7 +51,7 @@ function setup_local_env { get_aws_s3 "fac-private-s3" "endpoint" export AWS_S3_PRIVATE_ENDPOINT=$_GET_AWS_RESULT check_env_var_not_empty "AWS_S3_PRIVATE_ENDPOINT" - + get_aws_s3 "fac-public-s3" "access_key_id" export AWS_PUBLIC_ACCESS_KEY_ID=$_GET_AWS_RESULT check_env_var_not_empty "AWS_PUBLIC_ACCESS_KEY_ID" @@ -63,7 +63,7 @@ function setup_local_env { get_aws_s3 "fac-public-s3" "endpoint" export AWS_S3_PUBLIC_ENDPOINT=$_GET_AWS_RESULT check_env_var_not_empty "AWS_S3_PUBLIC_ENDPOINT" - + #export MC_HOST_=https://::@ export MC_HOST_myminio="http://${AWS_PRIVATE_ACCESS_KEY_ID}:${AWS_PRIVATE_SECRET_ACCESS_KEY}@minio:9000" # mc alias set myminio ${AWS_S3_PRIVATE_ENDPOINT} ${AWS_PRIVATE_ACCESS_KEY_ID} ${AWS_PRIVATE_ACCESS_KEY_ID} @@ -72,13 +72,13 @@ function setup_local_env { # https://min.io/docs/minio/linux/reference/minio-mc/mc-mb.html mc mb --ignore-existing myminio/${AWS_PUBLIC_BUCKET_NAME} mc mb --ignore-existing myminio/${AWS_PRIVATE_BUCKET_NAME} - + # MCJ 20241016 FIXME: Is this even needed locally? I don't think so. # mc admin user svcacct add \ # --access-key="${AWS_PRIVATE_ACCESS_KEY_ID}" \ # --secret-key="${AWS_PRIVATE_SECRET_ACCESS_KEY}" \ # myminio minioadmin - + # For database work export FAC_DB_URI=${DATABASE_URL} #?sslmode=disable export FAC_SNAPSHOT_URI=${SNAPSHOT_URL} @@ -98,13 +98,13 @@ function setup_local_env { # And we need cgov-util pushd /tmp - local CGOV_VERSION=v0.1.8 + local CGOV_VERSION=v0.1.9 curl -L -O https://github.com/GSA-TTS/fac-backup-utility/releases/download/${CGOV_VERSION}/gov.gsa.fac.cgov-util-${CGOV_VERSION}-linux-amd64.tar.gz tar xvzf gov.gsa.fac.cgov-util-${CGOV_VERSION}-linux-amd64.tar.gz gov.gsa.fac.cgov-util chmod 755 gov.gsa.fac.cgov-util mv gov.gsa.fac.cgov-util /bin/cgov-util popd - + export SLING_EXE='/bin/sling' export CGOV_UTIL_EXE='/bin/cgov-util' diff --git a/backend/util/load_public_dissem_data/Dockerfile b/backend/util/load_public_dissem_data/Dockerfile index 686278867d..68d76caca2 100644 --- a/backend/util/load_public_dissem_data/Dockerfile +++ b/backend/util/load_public_dissem_data/Dockerfile @@ -17,9 +17,9 @@ RUN curl -LO 'https://github.com/slingdata-io/sling-cli/releases/latest/download && chmod +x sling \ && mv sling /bin/sling -RUN curl -L -O https://github.com/GSA-TTS/fac-backup-utility/releases/download/v0.1.8/gov.gsa.fac.cgov-util-v0.1.8-linux-amd64.tar.gz \ - && tar xvzf gov.gsa.fac.cgov-util-v0.1.8-linux-amd64.tar.gz gov.gsa.fac.cgov-util \ - && chmod 755 gov.gsa.fac.cgov-util \ +RUN curl -L -O https://github.com/GSA-TTS/fac-backup-utility/releases/download/v0.1.9/gov.gsa.fac.cgov-util-v0.1.9-linux-amd64.tar.gz \ + && tar xvzf gov.gsa.fac.cgov-util-v0.1.9-linux-amd64.tar.gz gov.gsa.fac.cgov-util \ + && chmod 755 gov.gsa.fac.cgov-util \ && mv gov.gsa.fac.cgov-util /layered/cgov-util # Where do we need the config? Try home? diff --git a/backend/util/load_public_dissem_data/README.md b/backend/util/load_public_dissem_data/README.md index 3134d062a0..62a62f117b 100644 --- a/backend/util/load_public_dissem_data/README.md +++ b/backend/util/load_public_dissem_data/README.md @@ -50,7 +50,7 @@ This is containerized, so it should work on all platforms. To build the containe make build ``` -Then, to run the container, +Then, to run the container, ``` make run @@ -58,7 +58,7 @@ make run You need to run this from the current directory. -NOTE: The docker command in the Makefile uses the `--network` flag. The `--network` flag tells Docker to run this container on the same network as your currently running stack. It assumes that you did a `docker compose up` on the FAC stack, and that the web container has the default name of `backend-web-1`. If this does not work, you will need to... +NOTE: The docker command in the Makefile uses the `--network` flag. The `--network` flag tells Docker to run this container on the same network as your currently running stack. It assumes that you did a `docker compose up` on the FAC stack, and that the web container has the default name of `backend-web-1`. If this does not work, you will need to... ``` make NETWORK= run @@ -68,10 +68,10 @@ where `` is the name of your web container. This should allow th ## When to rebuild this container -Note this is pinned to v0.1.8 of the cgov-util. +Note this is pinned to v0.1.9 of the cgov-util. https://github.com/GSA-TTS/fac-backup-utility If that gets updated, you'll need to update the dockerfile. -It also copies in the YAML for sling from `dissemination/sql/sling`. If that changes, you'll want to +It also copies in the YAML for sling from `dissemination/sql/sling`. If that changes, you'll want to diff --git a/backend/util/load_public_dissem_data/load_public_data_locally.sh b/backend/util/load_public_dissem_data/load_public_data_locally.sh index 63d85effb4..b71ced9619 100755 --- a/backend/util/load_public_dissem_data/load_public_data_locally.sh +++ b/backend/util/load_public_dissem_data/load_public_data_locally.sh @@ -1,6 +1,6 @@ #!/bin/bash -export CGOV_UTIL_VERSION=v0.1.8 +export CGOV_UTIL_VERSION=v0.1.9 export FAC_DB_URI=postgresql://postgres@db:5432/postgres?sslmode=disable export FAC_SNAPSHOT_URI=postgresql://postgres@db2:5432/postgres?sslmode=disable @@ -42,7 +42,7 @@ echo "Unzipping data." unzip db_dissem_dump.zip popd -# Truncate the dissemination_* tables if they exist. +# Truncate the dissemination_* tables if they exist. # CASCADE as well. This makes sure we don't duplicate data, # which causes PK/FK problems. echo "select 'TRUNCATE TABLE '||tablename||' CASCADE;' FROM pg_tables WHERE tablename LIKE 'dissemination_%'" | \ diff --git a/backend/util/nightly_api_refresh.sh b/backend/util/nightly_api_refresh.sh index e3f32c26b9..e78609105c 100755 --- a/backend/util/nightly_api_refresh.sh +++ b/backend/util/nightly_api_refresh.sh @@ -7,7 +7,7 @@ source tools/sql_pre_post.sh setup_env # Run an RDS backup, to refresh dissemination_* in FAC_SNAPSHOT from FAC_DB -#./../fac-backup-util.sh "v0.1.8" "rds_backup" +#./../fac-backup-util.sh "v0.1.9" "rds_backup" # Deploy test ./fac-backup-util.sh "$1" "$2" gonogo "fac-backup-util" diff --git a/docs/backups_and_restores.md b/docs/backups_and_restores.md index 9019cae010..720c82a343 100644 --- a/docs/backups_and_restores.md +++ b/docs/backups_and_restores.md @@ -34,7 +34,7 @@ Information regarding the fac-backup-utility can be found [at the repository](ht Database backups occur in the following ways: 1. An initial backup, where a backup has not been run in the target environment. This input of `initial_backup` is important, as when it does a the `db_to_db` command, it will not truncate the target table, as the table does not exist in the destination database. ```bash -./fac-backup-util.sh v0.1.5 initial_backup +./fac-backup-util.sh v0.1.9 initial_backup # Curl the utility # Install AWS # DB to S3 table dump (backups) @@ -44,7 +44,7 @@ Database backups occur in the following ways: 2. A deploy backup, where the `db_to_db` function is not called. This is a standard backup strategy before the application deploys, to ensure the s3 contents of the primary s3 are sync'd to the backups bucket, and a table dump is stored in the backups bucket. ```bash -./fac-backup-util.sh v0.1.5 deploy_backup +./fac-backup-util.sh v0.1.9 deploy_backup # Curl the utility # Install AWS # DB to S3 table dump (backups) @@ -53,7 +53,7 @@ Database backups occur in the following ways: 3. A scheduled backup is run every two hours, across each environment, ensuring that we have a clean backup in s3, rds, and the bucket contents are in sync. ```bash -./fac-backup-util.sh v0.1.5 scheduled_backup +./fac-backup-util.sh v0.1.9 scheduled_backup # Curl the utility # Install AWS # DB to S3 table dump (fac-db -> backups) @@ -66,7 +66,7 @@ Restoring from backups can be run via workflow, from designated individuals. The 1. S3 Restore takes a `operation-mm-DD-HH` input (ex `scheduled-06-04-10`), and is required for the backups to be restored. The utility looks in `s3://${bucket}/backups/operation-mm-DD-HH/` for its table dumps, and without supplying the target backups, it will not restore. Once it does a `--data-only` restoration, it will then sync the files from the backups bucket to the application bucket. We do this to ensure the contents of the application bucket are up to date, relative to the data in the database. We know that if we use the latest folder in `/backups/` then the contents of the s3 are the latest available, from the prior backup. ```bash -./fac-restore-util.sh v0.1.5 s3_restore scheduled-06-04-10 +./fac-restore-util.sh v0.1.9 s3_restore scheduled-06-04-10 # Curl the utility # Install AWS # DB to S3 table dump (backups -> fac-db) [Truncate target table before --data-only pg_restore] @@ -81,7 +81,7 @@ daily-mm-dd 2. Database to database restoration also can occur as well, using `psql` to dump the tables from the cold store database to the live database. ```bash -./fac-restore-util.sh v0.1.5 db_restore +./fac-restore-util.sh v0.1.9 db_restore # Curl the utility # Install AWS # DB to DB table dump (fac-snapshot-db -> fac-db) [Truncate target table before dump] From bdbbe31503d4a11bd95cd502befe6eee3ad6dda1 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Tue, 29 Oct 2024 16:02:01 -0400 Subject: [PATCH 73/89] TF, sequencing 1. Updates terraform with correct APIs. 2. Removes more schemas 3. Aligns compose with our changes --- .../{999_finalize.sql => 810_bring_up_api.sql} | 0 .../sql/fac-snapshot-db/pre/060_schemas.sql | 3 +++ backend/docker-compose-web.yml | 13 ++----------- backend/docker-compose.yml | 15 +-------------- terraform/shared/modules/env/postgrest.tf | 2 +- 5 files changed, 7 insertions(+), 26 deletions(-) rename backend/dissemination/sql/fac-snapshot-db/post/{999_finalize.sql => 810_bring_up_api.sql} (100%) diff --git a/backend/dissemination/sql/fac-snapshot-db/post/999_finalize.sql b/backend/dissemination/sql/fac-snapshot-db/post/810_bring_up_api.sql similarity index 100% rename from backend/dissemination/sql/fac-snapshot-db/post/999_finalize.sql rename to backend/dissemination/sql/fac-snapshot-db/post/810_bring_up_api.sql diff --git a/backend/dissemination/sql/fac-snapshot-db/pre/060_schemas.sql b/backend/dissemination/sql/fac-snapshot-db/pre/060_schemas.sql index 5c1215e432..d1cf79a750 100644 --- a/backend/dissemination/sql/fac-snapshot-db/pre/060_schemas.sql +++ b/backend/dissemination/sql/fac-snapshot-db/pre/060_schemas.sql @@ -22,7 +22,9 @@ DROP SCHEMA IF EXISTS api_v2_0_0_functions CASCADE; ------------- DROP SCHEMA IF EXISTS dissem_copy CASCADE; DROP SCHEMA IF EXISTS public_data_v1_0_0 CASCADE; +DROP SCHEMA IF EXISTS public_data_v1_0_0_functions CASCADE; DROP SCHEMA IF EXISTS suppressed_data_v1_0_0 CASCADE; +DROP SCHEMA IF EXISTS suppressed_data_v1_0_0_functions CASCADE; --------------------------------- -- CREATE @@ -41,6 +43,7 @@ CREATE SCHEMA IF NOT EXISTS api_v1_1_0_functions; CREATE SCHEMA IF NOT EXISTS public_data_v1_0_0; CREATE SCHEMA IF NOT EXISTS public_data_v1_0_0_functions; CREATE SCHEMA IF NOT EXISTS suppressed_data_v1_0_0; +CREATE SCHEMA IF NOT EXISTS suppressed_data_v1_0_0_functions; CREATE SCHEMA IF NOT EXISTS dissem_copy; CREATE SCHEMA IF NOT EXISTS api_v2_0_0; diff --git a/backend/docker-compose-web.yml b/backend/docker-compose-web.yml index 98deabef2e..eb0502e7c2 100644 --- a/backend/docker-compose-web.yml +++ b/backend/docker-compose-web.yml @@ -28,14 +28,6 @@ services: interval: 10s timeout: 5s retries: 10 - - historic-data: - image: ghcr.io/gsa-tts/fac-historic-public-csvs/load-historic-public-data:20230912 - depends_on: - db: - condition: service_healthy - environment: - DATABASE_URL: postgres://postgres@db/postgres web: image: ghcr.io/gsa-tts/fac/web-container:latest @@ -94,12 +86,11 @@ services: expose: - "3000" environment: - PGRST_DB_URI: postgres://postgres@db:5432/postgres - PGRST2_DB_URI: postgres://postgres@db:5431/postgres + PGRST_DB_URI: postgres://postgres@db2:5432/postgres PGRST_OPENAPI_SERVER_PROXY_URI: http://127.0.0.1:3000 PGRST_DB_ANON_ROLE: anon # See https://postgrest.org/en/stable/references/api/schemas.html#multiple-schemas for multiple schemas - PGRST_DB_SCHEMAS: "api_v1_0_3, api_v1_1_0, admin_api_v1_1_0" + PGRST_DB_SCHEMAS: "api_v1_1_0,api_v2_0_0" PGRST_JWT_SECRET: ${PGRST_JWT_SECRET:-32_chars_fallback_secret_testing} # Fallback value for testing environments depends_on: db: diff --git a/backend/docker-compose.yml b/backend/docker-compose.yml index 3a8b65914e..e876017448 100644 --- a/backend/docker-compose.yml +++ b/backend/docker-compose.yml @@ -31,17 +31,6 @@ services: timeout: 5s retries: 10 - #--------------------------------------------- - # Historic data - #--------------------------------------------- - # historic-data: - # image: ghcr.io/gsa-tts/fac-historic-public-csvs/load-historic-public-data:20230912 - # depends_on: - # db: - # condition: service_healthy - # environment: - # DATABASE_URL: postgres://postgres@db/postgres - #--------------------------------------------- # Django app #--------------------------------------------- @@ -142,13 +131,11 @@ services: expose: - "3000" environment: - # This now requires us to `sling` data to db2 for local API testing. PGRST_DB_URI: postgres://postgres@db2:5432/postgres - # PGRST2_DB_URI: postgres://postgres@db:5431/postgres PGRST_OPENAPI_SERVER_PROXY_URI: http://127.0.0.1:3000 PGRST_DB_ANON_ROLE: anon # See https://postgrest.org/en/stable/references/api/schemas.html#multiple-schemas for multiple schemas - PGRST_DB_SCHEMAS: "api_v1_0_3,api_v1_1_0,admin_api_v1_1_0,api_v2_0_0" + PGRST_DB_SCHEMAS: "api_v1_1_0,api_v2_0_0" PGRST_JWT_SECRET: ${PGRST_JWT_SECRET:-32_chars_fallback_secret_testing} # Fallback value for testing environments # Enable this to inspect the DB plans for queries via EXPLAIN PGRST_DB_PLAN_ENABLED: 1 diff --git a/terraform/shared/modules/env/postgrest.tf b/terraform/shared/modules/env/postgrest.tf index d105846d00..78dffd6211 100644 --- a/terraform/shared/modules/env/postgrest.tf +++ b/terraform/shared/modules/env/postgrest.tf @@ -32,7 +32,7 @@ resource "cloudfoundry_app" "postgrest" { environment = { PGRST_DB_URI : cloudfoundry_service_key.postgrest.credentials.uri - PGRST_DB_SCHEMAS : "api_v1_0_3,api_v1_1_0,admin_api_v1_1_0,api_v2_0_0" + PGRST_DB_SCHEMAS : "api_v1_1_0,api_v2_0_0" PGRST_DB_ANON_ROLE : "anon" PGRST_JWT_SECRET : var.pgrst_jwt_secret PGRST_DB_MAX_ROWS : 20000 From cf769b9d0af6fe109f6866541d31f28a18c2391e Mon Sep 17 00:00:00 2001 From: Alex Steel <130377221+asteel-gsa@users.noreply.github.com> Date: Tue, 29 Oct 2024 16:17:06 -0400 Subject: [PATCH 74/89] API Refresh Test - Final? --- backend/util/nightly_api_refresh.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/util/nightly_api_refresh.sh b/backend/util/nightly_api_refresh.sh index e78609105c..0376b2cc8c 100755 --- a/backend/util/nightly_api_refresh.sh +++ b/backend/util/nightly_api_refresh.sh @@ -8,11 +8,9 @@ setup_env # Run an RDS backup, to refresh dissemination_* in FAC_SNAPSHOT from FAC_DB #./../fac-backup-util.sh "v0.1.9" "rds_backup" -# Deploy test ./fac-backup-util.sh "$1" "$2" gonogo "fac-backup-util" - # Run the pre/post. sql_pre_fac_snapshot_db gonogo "sql_pre_fac_snapshot_db" From 8977b6ddace1f3b99acf4cd34abd6acaf0fb1464 Mon Sep 17 00:00:00 2001 From: Alex Steel <130377221+asteel-gsa@users.noreply.github.com> Date: Tue, 29 Oct 2024 16:26:28 -0400 Subject: [PATCH 75/89] disable api refresh on push --- .github/workflows/fac-api-standup.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fac-api-standup.yml b/.github/workflows/fac-api-standup.yml index 580dbb6ac1..ddf843fade 100644 --- a/.github/workflows/fac-api-standup.yml +++ b/.github/workflows/fac-api-standup.yml @@ -38,8 +38,9 @@ on: type: choice options: - 'rds_backup' - push: - branches: [jadudm/api-perf] + # push: + # branches: [jadudm/api-perf] + # Uncomment above to do api refreshing jobs: api-standup: name: Standup API From 209c26a4b7ea15072f41b128f65133973e31fba3 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Wed, 30 Oct 2024 10:53:03 -0400 Subject: [PATCH 76/89] Testing with data --- .github/workflows/fac-api-standup.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fac-api-standup.yml b/.github/workflows/fac-api-standup.yml index ddf843fade..2c68f2bf04 100644 --- a/.github/workflows/fac-api-standup.yml +++ b/.github/workflows/fac-api-standup.yml @@ -38,8 +38,8 @@ on: type: choice options: - 'rds_backup' - # push: - # branches: [jadudm/api-perf] + push: + branches: [jadudm/api-perf] # Uncomment above to do api refreshing jobs: api-standup: From 38ac254ee4d92d011ae2f55ad90f007a87858e7a Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Thu, 31 Oct 2024 08:21:21 -0400 Subject: [PATCH 77/89] Allows for configuration of the DB This makes it possible to target a DB that is tunneled, as opposed to only using the containerized DB. Handy for testing in the preview environment. Uses external env vars to do this, so is safe. --- .../util/load_public_dissem_data/Dockerfile | 2 ++ backend/util/load_public_dissem_data/Makefile | 5 ++- .../load_public_data_locally.sh | 31 +++++++++++++++---- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/backend/util/load_public_dissem_data/Dockerfile b/backend/util/load_public_dissem_data/Dockerfile index 68d76caca2..c5d84055de 100644 --- a/backend/util/load_public_dissem_data/Dockerfile +++ b/backend/util/load_public_dissem_data/Dockerfile @@ -25,6 +25,8 @@ RUN curl -L -O https://github.com/GSA-TTS/fac-backup-utility/releases/download/v # Where do we need the config? Try home? WORKDIR /root/.fac COPY data/config.json . +ARG FAC_DB_URI="postgresql://postgres@db:5432/postgres?sslmode=disable" +ARG RUNNING_LOCAL=0 WORKDIR /app ENTRYPOINT [ "./load_public_data_locally.sh" ] diff --git a/backend/util/load_public_dissem_data/Makefile b/backend/util/load_public_dissem_data/Makefile index bd02fe9bc7..77733d2fc7 100644 --- a/backend/util/load_public_dissem_data/Makefile +++ b/backend/util/load_public_dissem_data/Makefile @@ -1,7 +1,10 @@ NETWORK?=backend-db2-1 build: - docker build -t facloaddata . + docker build -t facloaddata --build-arg FAC_DB_URI=${FAC_DB_URI} . run: docker run -i --rm -v ${PWD}:/app --network container:${NETWORK} -t facloaddata + +run_remote: + docker run -i --rm -v ${PWD}:/app facloaddata "${FAC_DB_URI}" diff --git a/backend/util/load_public_dissem_data/load_public_data_locally.sh b/backend/util/load_public_dissem_data/load_public_data_locally.sh index b71ced9619..562e1496f7 100755 --- a/backend/util/load_public_dissem_data/load_public_data_locally.sh +++ b/backend/util/load_public_dissem_data/load_public_data_locally.sh @@ -1,8 +1,20 @@ #!/bin/bash export CGOV_UTIL_VERSION=v0.1.9 -export FAC_DB_URI=postgresql://postgres@db:5432/postgres?sslmode=disable -export FAC_SNAPSHOT_URI=postgresql://postgres@db2:5432/postgres?sslmode=disable + +echo "FAC_DB_URI is [" $1 "]" + +if [ -z "$1" ] +then + export FAC_DB_URI="postgresql://postgres@db:5432/postgres?sslmode=disable" +else + export FAC_DB_URI="$1" + export RUNNING_LOCAL=1 +fi + +echo "FAC_DB_URI is now [" $FAC_DB_URI "] local [" $RUNNING_LOCAL "]" + +export FAC_SNAPSHOT_URI="postgresql://postgres@db2:5432/postgres?sslmode=disable" function check_table_exists() { @@ -22,8 +34,14 @@ sleep 4 # First, we cleanup the local filesystem. # This removes any temporary files from any # previous data loads -rm -f /app/data/db_dissem_dump -rm -rf /app/data/__MACOSX +if [ "$RUNNING_LOCAL" -eq "1" ]; then + export BASE_PATH="." +else + export BASE_PATH="/app" +fi + +rm -f $BASE_PATH/data/db_dissem_dump +rm -rf $BASE_PATH/data/__MACOSX # # Next, we drop the public_data schema. # # This is because we want to make sure it is @@ -37,7 +55,8 @@ rm -rf /app/data/__MACOSX # psql $FAC_SNAPSHOT_URI -c "CREATE SEQUENCE IF NOT EXISTS public_data_v1_0_0.seq_combined START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE" # Unzip the compressed historical data dump. -pushd /app/data + +pushd $BASE_PATH/data echo "Unzipping data." unzip db_dissem_dump.zip popd @@ -50,7 +69,7 @@ echo "select 'TRUNCATE TABLE '||tablename||' CASCADE;' FROM pg_tables WHERE tabl psql $FAC_DB_URI # Load the large historic dataset. -psql $FAC_DB_URI -v ON_ERROR_STOP=on < /app/data/db_dissem_dump +psql $FAC_DB_URI -v ON_ERROR_STOP=on < $BASE_PATH/data/db_dissem_dump result=$? if [ $result -ne 0 ]; then echo "Something went wrong." From ec602d513fb8f0be75f8b946a984524ee5e7a6a0 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Thu, 31 Oct 2024 09:57:02 -0400 Subject: [PATCH 78/89] Adding documentation --- .../0020_alter_disseminationcombined_table.py | 17 ++ backend/dissemination/sql/SQL_README.md | 168 +++++++++++++++--- .../sql/fac-db/post/999_finalize.sql | 5 +- .../fac-db/pre/030_disable_audit_curation.sql | 13 ++ .../post/129_api_v1_1_0_perms.sql | 5 - ..._v2_0_0_rpc.sql => 144_api_v2_0_0_rpc.sql} | 0 .../post/149_api_v2_0_0_perms.sql | 7 - .../fac-snapshot-db/post/800_permissions.sql | 16 +- 8 files changed, 192 insertions(+), 39 deletions(-) create mode 100644 backend/dissemination/migrations/0020_alter_disseminationcombined_table.py create mode 100644 backend/dissemination/sql/fac-db/pre/030_disable_audit_curation.sql delete mode 100644 backend/dissemination/sql/fac-snapshot-db/post/129_api_v1_1_0_perms.sql rename backend/dissemination/sql/fac-snapshot-db/post/{0144_api_v2_0_0_rpc.sql => 144_api_v2_0_0_rpc.sql} (100%) delete mode 100644 backend/dissemination/sql/fac-snapshot-db/post/149_api_v2_0_0_perms.sql diff --git a/backend/dissemination/migrations/0020_alter_disseminationcombined_table.py b/backend/dissemination/migrations/0020_alter_disseminationcombined_table.py new file mode 100644 index 0000000000..0502e03bc4 --- /dev/null +++ b/backend/dissemination/migrations/0020_alter_disseminationcombined_table.py @@ -0,0 +1,17 @@ +# Generated by Django 5.1.2 on 2024-10-28 21:27 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("dissemination", "0019_delete_sacvalidationwaiver"), + ] + + operations = [ + migrations.AlterModelTable( + name="disseminationcombined", + table='public_data_v1_0_0"."combined', + ), + ] diff --git a/backend/dissemination/sql/SQL_README.md b/backend/dissemination/sql/SQL_README.md index 816da9f82c..cdd6c67ce9 100644 --- a/backend/dissemination/sql/SQL_README.md +++ b/backend/dissemination/sql/SQL_README.md @@ -1,26 +1,48 @@ # sql folder -From the perspective of the application, there are both managed and unamanged tables. +If you are reading this, it is assumed you are FAC developer who is about to touch the SQL in this folder. -However, we have data and processes that are outside of the application, but inside the databases. +This document describes -The SQL folder provides a mechanism for: +1. The database layout of the FAC +2. What is in this folder +3. How and when it executes +4. Things you should do +5. Things to watch out for -1. Running SQL against the databases *before* we run migrations. -2. Running SQL against the databases *afetr* we run migrations. +## the database layout of the FAC -We have two databases. +The FAC has two databases. -1. **fac-db** is the production database. The app talks to this database for all live operations. -2. **fac-snapshot-db**, which began life (and continutes to be used for) snapshots of the production database right before a deploy. Because it wasn't doing anything else, we have co-opted this database to also serve as a "read replica" of the dissemination tables, and host the API. +**DB1** is the **production** database. The app talks to this database for all live operations. -## folder organization +1. When a user updates their submission, they are updating a `singleauditchecklist` record in DB1. +2. When a user does a *basic* search, they are searching `dissemination_general` in DB1. +3. When you update user roles in `/admin`, you are editing a table in DB1. -* `fac-db` holds code that operates **on** fac-db before/after migrations. -* `fac-snapshot-db` holds code that operates **on** `fac-snapshot-db` before/after migrations. -* `sling` holds files used by the `sling` application (for moving data between the databases). +**DB2** began life as a place to do a database snapshot before deploy. We are now using this as a *read replica* for DB1. It hosts a *data pipeline* that is implemented entirely as a sequence of actions in SQL. -# pre/post +**DB2 updates nightly.** It is *completely* destroyed every night, and completely rebuilt. No data is persisted. In this regard, DB2 is serves a *stateless data pipeline*. More on this later. + +1. The PostgREST API uses DB2 to resolve *all* API queries. +2. When a user does an *advanced search*, they are using DB2. + +## what is in this folder + +The SQL folder contains one folder for each database: `fac-db` and `fac-snapshot-db`. These names align with our Terraform configuration. + +Inside of each folder are two sub-folders: `pre` and `post`. + +1. `pre` contains SQL we run against the databases *before* migrations. +2. `post` contains SQL we run against the databases *after* migrations. + +In the case of `fac-db` (DB1), we run all of the scripts in the `pre` folder, we run migrations, and then we run everything in the `post` folder. + +In the case of `fac-snapshot-db` (DB2), it is slightly different. We tear things down, then run everything in the `pre` folder, and then we run everything in the `post` folder. There are no migrations in DB2, because it is a stateless copy of DB1. The structure is parallel/preserved/kept-the-same-as DB1 for consistency, but it is worth noting that DB2 does not have any migrations. + +There is one other folder, `sling`. More on this later. + +## pre/post The `pre` and `post` folders contain SQL files in execution order. That means that the way the files are named matters. @@ -32,18 +54,115 @@ If the following files are in the `pre` folder: then they will execute in the lexigraphical order as shown. *However*, only files ending in `.sql` will be executed. This means that `000_first.sql` will be executed, `010_nope.SKIP` will be skipped, and `020_second.sql` will be run second. (Although it encourages a dirty tree, we *might* want to keep a file in the tree, but not have it execute.) -## in practice +### the pre/post sequence, generally + +On each DB, in broad strokes (at time of this being written): + +#### DB1 (fac-db) + +On DB1, we do not do much. + +##### pre + +1. Drop the API schemas. +2. Initialize audit curation code + +The first step is because we will no longer serve the API from DB1. Therefore, all of the API schemas can go away. + +The second is because we now have SQL triggers to support *data curation*. These triggers are defined here. Finally, we *disable* audit curation as a "just-in-case" measure. Because it is a state in the DB, the app could crash, and we would be in a condition of recording all changes to the SAC table. This would be *bad*. So, we do a "disable" as part of startup. + +##### post + +We tear out the old, old, OLD, Census data (used for the cog/over work in early days). + +In the case of DB1, all of the actions could *probably* be `pre` actions. It does not particularly matter. + +#### DB2 (fac-snapshot-db) + +We do a lot on DB2. + +##### pre + +1. Set up roles (for PostgREST). Without these, PostgREST cannot authenticate/operate. +2. Tear down *all* schemas associated with the data pipeline. +3. Tear down and rebuild sequences used in constructing the new `public_data` tables. + +##### post + +1. Copy the `dissemination_*` tables to a `dissem_copy` schema. + +We do this because the API is going to attach to `dissem_copy.dissemination_*` tables. We do this instead of using `public.dissemination_*` for the simple reason that those tables are overwritten with each deploy. If we attached the API `VIEW`s to the `public` tables, it would interrupt/disrupt/break the pre-deploy backups. So, the first thing we do is make a copy. + +2. Create `public_data` tables. + +These tables are a copy of the `dissem_copy` tables, with some changes. + +1. We create a `combined` table that does a 4x `JOIN` across `general`, `federal_awards`, `passthrough`, and `findings`. This is all 100% public data. +2. We apply a `general.is_public=true` filter to all tables containing suppressed data, therefore guaranteeing that `notes_to_sefa`, `corrective_action_plans`, and `finding_text` (for example) contain *only* public data. +3. Sequences are inserted in all tables, and a `batch_number`. This is indexed for fast downloading of bulk data. -The API is torn down every time we deploy (`pre`). This is because the API has `VIEW`s on the database tables that interfere with migrations. Then, everything in `post` is run, which stands up the API and performs operations that optimize or otherwise improve upon the DB's health/performance. +This is the "data pipeline." It is copying and modifying data to put it in the "right" shape for our API. This way, our API becomes a simple `SELECT *` in a `VIEW`. -# in case it isn't confusing +1. Create `suppressed_data` tables. -* The basic search (in the application) talks to fac-db. -* The advanced search (in the application) talks to fac-snapshot-db. -* The public/read API talks to fac-snapshot-db. -* The *write* portion of tribal access (adding/removing keys) talks to fac-db. -* The *read* portion of tribal access (reading files) talks to fac-snapshot-db. -* The admin API only talks to fac-db. +These are "the same" as the above, but they are filtered to contain only suppressed/Tribal data. + +4. Create `metadata` table. + +A `metadata` table containing counts of rows in all tables is created. + +5. Create the `api_v1_1_0`. + +This is the same code as previously existed, flaws and all. It points at `dissem_copy` tables, because they are 1:1 with what used to be in DB1. Hence, it "just works" "as-was." + +A good refactoring would be to point these tables at `public_data` tables instead. The views would no longer require `JOIN` statements, and access control could be handled more gracefully. + + +6. Create `api_v2_0_0`. + +This points at the `public_data` and `suppressed_data` tables. + +7. Setup permissions + +All of the API access permissions are set in one place after the tables/views are created. + +8. Bring up the API + +We issue a `NOTIFY` to PostgREST which tells it to re-read the schemas and provide an API. + +9. Indexing + +Now, we index *everything*. If something is not performant, *add more indexes*. + + +## possible improvements + +These are largely ticketed. However, there is a Python script floating around to generate `PARTITION` tables. (It is very repetative code, hence having a script to spit out the SQL makes sense. It is assumed it would be run once, manually, and the output added to the sequence.) However, this will multiply the number of tables we have by, like, Nx, where `N` is the number of partitions on each table. Performance testing suggests there is some improvement, but we should see significant improvements with `api_v2_0_0` that are "good enough" for a start without adding the complexity of `PARTITION`s at this point. + +## `sling` + +[sling](https://slingdata.io/) is a very cool tool. It takes a YAML config, and can copy data: + +* From a DB to a different DB +* DB to CSV (locally or direct to an S3 bucket) +* DB to JSON (same) +* ... + +basically, it "slings data around." + +The `sling` folder contains an example that, if wired into the nightly sequence, will generate compressed CSVs of all of the public data. + +In rough pseudocode: + +``` +for each year in the range 2016 - 2030 + for each table in the `public_data` space + sling a CSV to S3 called `-.csv.zip` +``` + +The sling YAML config is auto-generated from a short (manual) Python script. Why? Because I wanted to loop over the years, and wanted to loop over the table names. Also, you can't have more than 1M rows in a CSV, or Excel will choke when you try and open it. Hence, our tables *must* be split, somehow, for it to be useful to our customers. + +This is 95% of the way to providing downloads of bulk data. If we wire it in to the nightly data pipeline (as a last step), it will copy the `public_data` tables out to `/bulk_export/...` in our `private` S3 bucket, and the Django app can provide a page/links for authenticated users to grab those files (single-use URLs). This will 1) slow users down from getting them *all the time*, and give us better logging/tracking of who is accessing the bulk data. # running tests @@ -53,4 +172,7 @@ pytest -s --env local test_api.py where the env can be `local`, `preview`, `dev`, `staging`, or `production` to run queries against the API in any of those environments. -`FAC_API_KEY` and `CYPRESS_API_GOV_JWT` need to be set in the environment for this script to work. +* When running against local, `CYPRESS_API_GOV_JWT` needs to be set in your environment for this to work. And, it needs to match the `Authorization: Bearer` value in the Docker stack. +* `FAC_API_KEY` needs to be set when testing against the production environments. + +These tests are not (yet) integrated with any repeatable framework. They were developed as a way to quickly be confident that access controls were being implemented correctly. It is a ticket to integrate API testing. diff --git a/backend/dissemination/sql/fac-db/post/999_finalize.sql b/backend/dissemination/sql/fac-db/post/999_finalize.sql index d1168e7a39..6068a8af5b 100644 --- a/backend/dissemination/sql/fac-db/post/999_finalize.sql +++ b/backend/dissemination/sql/fac-db/post/999_finalize.sql @@ -1,3 +1,2 @@ --- PostgREST likes to know when the schemas and things --- attached to them change. -NOTIFY pgrst, 'reload schema'; +-- Currently, there are no finalization actions for fac-db. +-- They would go here. diff --git a/backend/dissemination/sql/fac-db/pre/030_disable_audit_curation.sql b/backend/dissemination/sql/fac-db/pre/030_disable_audit_curation.sql new file mode 100644 index 0000000000..828555c22a --- /dev/null +++ b/backend/dissemination/sql/fac-db/pre/030_disable_audit_curation.sql @@ -0,0 +1,13 @@ +----------------------- +-- AUDIT CURATION +-- This disables curation tracking on the below tables. +-- We do this at startup *just in case* the app crashed while we were +-- doing data curation. If that were true, the DB would be recording +-- EVERY change to these two tables. Given that the `singleauditchecklist` +-- table is hit *constantly*, this would be bad. +-- +-- Therefore, one of the first things we do every time we startup is +-- make sure that this state is disabled in the database. + +select curation.disable_tracking('public.audit_singleauditchecklist'::regclass); +select curation.disable_tracking('public.support_cognizantassignment'::regclass); diff --git a/backend/dissemination/sql/fac-snapshot-db/post/129_api_v1_1_0_perms.sql b/backend/dissemination/sql/fac-snapshot-db/post/129_api_v1_1_0_perms.sql deleted file mode 100644 index 8dab5a0c9a..0000000000 --- a/backend/dissemination/sql/fac-snapshot-db/post/129_api_v1_1_0_perms.sql +++ /dev/null @@ -1,5 +0,0 @@ -GRANT USAGE ON SCHEMA api_v1_1_0_functions TO api_fac_gov; -GRANT USAGE ON SCHEMA api_v1_1_0 TO api_fac_gov; - -GRANT SELECT ON ALL TABLES IN SCHEMA api_v1_1_0 to api_fac_gov; -GRANT SELECT, USAGE ON ALL SEQUENCES IN SCHEMA api_v1_1_0 TO api_fac_gov; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/0144_api_v2_0_0_rpc.sql b/backend/dissemination/sql/fac-snapshot-db/post/144_api_v2_0_0_rpc.sql similarity index 100% rename from backend/dissemination/sql/fac-snapshot-db/post/0144_api_v2_0_0_rpc.sql rename to backend/dissemination/sql/fac-snapshot-db/post/144_api_v2_0_0_rpc.sql diff --git a/backend/dissemination/sql/fac-snapshot-db/post/149_api_v2_0_0_perms.sql b/backend/dissemination/sql/fac-snapshot-db/post/149_api_v2_0_0_perms.sql deleted file mode 100644 index 4dcacfcf64..0000000000 --- a/backend/dissemination/sql/fac-snapshot-db/post/149_api_v2_0_0_perms.sql +++ /dev/null @@ -1,7 +0,0 @@ ------------------------------------------------------ --- PERMISSIONS ------------------------------------------------------ -GRANT USAGE ON SCHEMA api_v2_0_0_functions TO api_fac_gov; -GRANT USAGE ON SCHEMA api_v2_0_0 TO api_fac_gov; - -GRANT SELECT ON ALL TABLES IN SCHEMA api_v2_0_0 TO api_fac_gov; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/800_permissions.sql b/backend/dissemination/sql/fac-snapshot-db/post/800_permissions.sql index dca062baef..3963a5c006 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/800_permissions.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/800_permissions.sql @@ -3,6 +3,20 @@ -- Decommissioned 20241024 -- GRANT SELECT ON ALL TABLES IN SCHEMA api_v1_0_3 TO api_fac_gov; + +----------------------------------------------------- +-- api_v1_1_0 PERMISSIONS +----------------------------------------------------- +GRANT USAGE ON SCHEMA api_v1_1_0_functions TO api_fac_gov; +GRANT USAGE ON SCHEMA api_v1_1_0 TO api_fac_gov; GRANT SELECT ON ALL TABLES IN SCHEMA api_v1_1_0 TO api_fac_gov; -GRANT SELECT ON ALL TABLES IN SCHEMA api_v2_0_0 TO api_fac_gov; +-- There are no sequences currently on api_v1_1_0 +-- GRANT SELECT, USAGE ON ALL SEQUENCES IN SCHEMA api_v1_1_0 TO api_fac_gov; +----------------------------------------------------- +-- api_v2_0_0 PERMISSIONS +----------------------------------------------------- +GRANT USAGE ON SCHEMA api_v2_0_0_functions TO api_fac_gov; +GRANT USAGE ON SCHEMA api_v2_0_0 TO api_fac_gov; +GRANT SELECT ON ALL TABLES IN SCHEMA api_v2_0_0 TO api_fac_gov; +GRANT SELECT, USAGE ON ALL SEQUENCES IN SCHEMA api_v2_0_0 TO api_fac_gov; From 2d009d6b00dacb6fe2d1146f0820646aa0054d48 Mon Sep 17 00:00:00 2001 From: Alex Steel <130377221+asteel-gsa@users.noreply.github.com> Date: Fri, 1 Nov 2024 10:43:33 -0400 Subject: [PATCH 79/89] Update api refresh workflow --- .github/workflows/fac-api-standup.yml | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/.github/workflows/fac-api-standup.yml b/.github/workflows/fac-api-standup.yml index 2c68f2bf04..96f3896a87 100644 --- a/.github/workflows/fac-api-standup.yml +++ b/.github/workflows/fac-api-standup.yml @@ -1,8 +1,5 @@ --- name: Standup the API -### Common Commands: -# ./fac-backup-util.sh v0.1.9 scheduled_backup -# ./fac-backup-util.sh v0.1.9 daily_backup on: workflow_call: inputs: @@ -38,18 +35,13 @@ on: type: choice options: - 'rds_backup' - push: - branches: [jadudm/api-perf] - # Uncomment above to do api refreshing jobs: api-standup: name: Standup API runs-on: ubuntu-latest - #environment: ${{ inputs.environment }} - environment: preview + environment: ${{ inputs.environment }} env: - #space: ${{ inputs.environment }} - space: preview + space: ${{ inputs.environment }} steps: - name: Backup FAC Database and refresh the api uses: cloud-gov/cg-cli-tools@main @@ -57,7 +49,6 @@ jobs: cf_username: ${{ secrets.CF_USERNAME }} cf_password: ${{ secrets.CF_PASSWORD }} cf_org: gsa-tts-oros-fac - #cf_space: ${{ env.space }} - cf_space: preview - #command: cf run-task gsa-fac -k 3G -m 3G --name standup_api --command "./util/nightly_api_refresh.sh ${{ inputs.util_version }} ${{ inputs.backup_operation }}" - command: cf run-task gsa-fac -k 2G -m 2G --name api_refresh --command "./util/nightly_api_refresh.sh v0.1.9 rds_backup" + cf_space: ${{ env.space }} + command: cf run-task gsa-fac -k 2G -m 2G --name standup_api --command "./util/nightly_api_refresh.sh ${{ inputs.util_version }} ${{ inputs.backup_operation }}" + # command: cf run-task gsa-fac -k 2G -m 2G --name api_refresh --command "./util/nightly_api_refresh.sh v0.1.9 rds_backup" From e569dc617ae933a982b59713d41edbd6fe48dd40 Mon Sep 17 00:00:00 2001 From: Alex Steel <130377221+asteel-gsa@users.noreply.github.com> Date: Fri, 1 Nov 2024 10:51:08 -0400 Subject: [PATCH 80/89] Fix workflows --- ...up-scheduler.yml => fac-api-scheduler.yml} | 7 +++--- .github/workflows/fac-backup-scheduler.yml | 23 +++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) rename .github/workflows/{fac-api-standup-scheduler.yml => fac-api-scheduler.yml} (81%) create mode 100644 .github/workflows/fac-backup-scheduler.yml diff --git a/.github/workflows/fac-api-standup-scheduler.yml b/.github/workflows/fac-api-scheduler.yml similarity index 81% rename from .github/workflows/fac-api-standup-scheduler.yml rename to .github/workflows/fac-api-scheduler.yml index 85c04be326..521dcae32f 100644 --- a/.github/workflows/fac-api-standup-scheduler.yml +++ b/.github/workflows/fac-api-scheduler.yml @@ -1,9 +1,9 @@ --- -name: Backup Environments +name: API Refresh Schedule on: schedule: - # Invoke every 2 hours - - cron: '0 2 * * *' + # Invoke at 12:30 EST + - cron: '30 4 * * *' workflow_dispatch: null jobs: @@ -22,4 +22,3 @@ jobs: environment: ${{ matrix.environment.name }} util_version: "v0.1.9" backup_operation: "rds_backup" - diff --git a/.github/workflows/fac-backup-scheduler.yml b/.github/workflows/fac-backup-scheduler.yml new file mode 100644 index 0000000000..98e42e30c0 --- /dev/null +++ b/.github/workflows/fac-backup-scheduler.yml @@ -0,0 +1,23 @@ +--- +name: Backup Environments +on: + schedule: + # Invoke every 2 hours + - cron: '0 12,14,16,18,20,22/2 * * *' + workflow_dispatch: null + +jobs: + backup-environment: + strategy: + fail-fast: false + matrix: + environment: + - name: dev + - name: staging + - name: production + uses: ./.github/workflows/fac-backup-util-scheduled.yml + secrets: inherit + with: + environment: ${{ matrix.environment.name }} + util_version: "v0.1.9" + backup_operation: "scheduled_backup" From fc614dd4b6555b33e2ca2991131fd4f8c413d462 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Fri, 1 Nov 2024 13:19:15 -0400 Subject: [PATCH 81/89] Fixing two missing tables... --- backend/dissemination/sql/SQL_README.md | 82 +++++++++---------- .../post/010_copy_dissem_tables.sql | 27 ++++++ .../post/122_api_v1_1_0_functions.sql | 2 +- .../post/124_api_v1_1_0_rpc.sql | 10 +-- .../post/142_api_v2_0_0_functions.sql | 2 +- .../post/144_api_v2_0_0_rpc.sql | 10 +-- .../fac-snapshot-db/post/800_permissions.sql | 2 + ...{810_bring_up_api.sql => 999_finalize.sql} | 0 backend/tools/cgov_util_local_only.sh | 17 ++-- backend/tools/setup_cgov_env.py | 79 ------------------ backend/util/nightly_api_refresh.sh | 6 ++ 11 files changed, 99 insertions(+), 138 deletions(-) rename backend/dissemination/sql/fac-snapshot-db/post/{810_bring_up_api.sql => 999_finalize.sql} (100%) delete mode 100644 backend/tools/setup_cgov_env.py diff --git a/backend/dissemination/sql/SQL_README.md b/backend/dissemination/sql/SQL_README.md index cdd6c67ce9..863d2d523b 100644 --- a/backend/dissemination/sql/SQL_README.md +++ b/backend/dissemination/sql/SQL_README.md @@ -12,57 +12,48 @@ This document describes ## the database layout of the FAC -The FAC has two databases. +*This is high-level background for reference.* -**DB1** is the **production** database. The app talks to this database for all live operations. +The FAC has two databases. -1. When a user updates their submission, they are updating a `singleauditchecklist` record in DB1. -2. When a user does a *basic* search, they are searching `dissemination_general` in DB1. -3. When you update user roles in `/admin`, you are editing a table in DB1. +**DB1** is `fac-db`. The app talks to this database for all live operations. -**DB2** began life as a place to do a database snapshot before deploy. We are now using this as a *read replica* for DB1. It hosts a *data pipeline* that is implemented entirely as a sequence of actions in SQL. +When a user updates their submission, they are updating a `singleauditchecklist` record in DB1. When a user does a *basic* search, they are searching `dissemination_general` in DB1. And, finally, when you update user roles in `/admin`, you are editing a table in DB1. -**DB2 updates nightly.** It is *completely* destroyed every night, and completely rebuilt. No data is persisted. In this regard, DB2 is serves a *stateless data pipeline*. More on this later. +**DB2** is `fac-snapshot-db`. began life as a place to do a database snapshot before deploy. It still serves this purpos. However, we are now using it as a place to build a *data pipeline* that is implemented entirely as a sequence of actions in SQL. In this regard, it becomes a *read replica* of sorts where we can serve both *advanced search* and the API. -1. The PostgREST API uses DB2 to resolve *all* API queries. -2. When a user does an *advanced search*, they are using DB2. +**DB2 updates nightly.** The tables described below are *completely* destroyed and rebuilt every night. No data is persisted: DB2 is serves as a *stateless data pipeline*. ## what is in this folder -The SQL folder contains one folder for each database: `fac-db` and `fac-snapshot-db`. These names align with our Terraform configuration. +The SQL folder contains one folder for each database: `fac-db` and `fac-snapshot-db`. These names align with the "friendly names" of our database services in our system configuration. Inside of each folder are two sub-folders: `pre` and `post`. 1. `pre` contains SQL we run against the databases *before* migrations. 2. `post` contains SQL we run against the databases *after* migrations. -In the case of `fac-db` (DB1), we run all of the scripts in the `pre` folder, we run migrations, and then we run everything in the `post` folder. - -In the case of `fac-snapshot-db` (DB2), it is slightly different. We tear things down, then run everything in the `pre` folder, and then we run everything in the `post` folder. There are no migrations in DB2, because it is a stateless copy of DB1. The structure is parallel/preserved/kept-the-same-as DB1 for consistency, but it is worth noting that DB2 does not have any migrations. +In the case of `fac-db` (DB1), we run all of the scripts in the `pre` folder when we deploy, we run migrations, and then we run everything in the `post` folder. This is consistent with what took place previously. -There is one other folder, `sling`. More on this later. +In the case of `fac-snapshot-db` (DB2), it is slightly different. We run everything in the `pre` folder, and then we run everything in the `post` folder. There are no migrations in DB2, because it is a stateless copy of DB1. ## pre/post -The `pre` and `post` folders contain SQL files in execution order. That means that the way the files are named matters. +The `pre` and `post` folders contain SQL files in execution order. That means that the ordering of the files matters. If the following files are in the `pre` folder: -`000_first.sql` -`010_nope.SKIP` -`020_second.sql` +1. `000_first.sql` +2. `010_nope.SKIP` +3. `020_second.sql` then they will execute in the lexigraphical order as shown. *However*, only files ending in `.sql` will be executed. This means that `000_first.sql` will be executed, `010_nope.SKIP` will be skipped, and `020_second.sql` will be run second. (Although it encourages a dirty tree, we *might* want to keep a file in the tree, but not have it execute.) -### the pre/post sequence, generally - -On each DB, in broad strokes (at time of this being written): - -#### DB1 (fac-db) +### what happens on DB1 (fac-db) -On DB1, we do not do much. +On DB1, we remove old schemas and tables (if they exist). If they don't exist, we essentially do nothing. -##### pre +#### pre 1. Drop the API schemas. 2. Initialize audit curation code @@ -71,66 +62,73 @@ The first step is because we will no longer serve the API from DB1. Therefore, a The second is because we now have SQL triggers to support *data curation*. These triggers are defined here. Finally, we *disable* audit curation as a "just-in-case" measure. Because it is a state in the DB, the app could crash, and we would be in a condition of recording all changes to the SAC table. This would be *bad*. So, we do a "disable" as part of startup. -##### post +#### post We tear out the old, old, OLD, Census data (used for the cog/over work in early days). In the case of DB1, all of the actions could *probably* be `pre` actions. It does not particularly matter. -#### DB2 (fac-snapshot-db) +### what happens on DB2 (fac-snapshot-db) -We do a lot on DB2. +Every night, on DB2, we first back up DB1. Then, we tear down our data pipeline and API, and rebuild it all from the backup we just made. This means that the data pipeline---including the backup---is essentially stateless. -##### pre +#### pre 1. Set up roles (for PostgREST). Without these, PostgREST cannot authenticate/operate. 2. Tear down *all* schemas associated with the data pipeline. 3. Tear down and rebuild sequences used in constructing the new `public_data` tables. -##### post +#### post -1. Copy the `dissemination_*` tables to a `dissem_copy` schema. +##### Copy the `dissemination_*` tables to a `dissem_copy` schema. We do this because the API is going to attach to `dissem_copy.dissemination_*` tables. We do this instead of using `public.dissemination_*` for the simple reason that those tables are overwritten with each deploy. If we attached the API `VIEW`s to the `public` tables, it would interrupt/disrupt/break the pre-deploy backups. So, the first thing we do is make a copy. -2. Create `public_data` tables. +##### Create `public_data` tables. These tables are a copy of the `dissem_copy` tables, with some changes. -1. We create a `combined` table that does a 4x `JOIN` across `general`, `federal_awards`, `passthrough`, and `findings`. This is all 100% public data. +1. We create a `combined` table that does a 4x `JOIN` across `general`, `federal_awards`, `passthrough`, and `findings`. This is all 100% public data. (It was previously our `MATERIALIZED VIEW`.) 2. We apply a `general.is_public=true` filter to all tables containing suppressed data, therefore guaranteeing that `notes_to_sefa`, `corrective_action_plans`, and `finding_text` (for example) contain *only* public data. 3. Sequences are inserted in all tables, and a `batch_number`. This is indexed for fast downloading of bulk data. This is the "data pipeline." It is copying and modifying data to put it in the "right" shape for our API. This way, our API becomes a simple `SELECT *` in a `VIEW`. -1. Create `suppressed_data` tables. +As new data needs are discovered, it is assumed that the `post` operations on DB2 will implement additional copies/table creations/etc. to extend our data pipeline in order to address customer/user needs. + +##### Create `suppressed_data` tables. These are "the same" as the above, but they are filtered to contain only suppressed/Tribal data. -4. Create `metadata` table. +These tables are only accessible via API if you have gone through our Tribal API attestation/access process. Only Federal agencies are able to obtain API access to this data in order to support their oversight operations. Non-privileged keys will find empty result sets (`[]`) if they attempt to query these tables. + +##### Create `metadata` table. + +A `metadata` table containing counts of rows in all tables create above. -A `metadata` table containing counts of rows in all tables is created. +This also is exposed to `api_v2_0_0`. This allows users to quickly find 1) which tables are present, and 2) how much data is in those tables. This meets customer needs in an important way: when they are downloading data, they want to know "did I get everything?" This lets them do a bulk download via API and then answer that question in a programmatic manner. -5. Create the `api_v1_1_0`. +It also serves as a demonstration for one kind of data manipulation that can be used to create new tables and, therefore, new functionality for users via the API. + +##### Create the `api_v1_1_0`. This is the same code as previously existed, flaws and all. It points at `dissem_copy` tables, because they are 1:1 with what used to be in DB1. Hence, it "just works" "as-was." A good refactoring would be to point these tables at `public_data` tables instead. The views would no longer require `JOIN` statements, and access control could be handled more gracefully. - -6. Create `api_v2_0_0`. +##### Create `api_v2_0_0`. This points at the `public_data` and `suppressed_data` tables. -7. Setup permissions +##### Setup permissions All of the API access permissions are set in one place after the tables/views are created. -8. Bring up the API +##### Bring up the API We issue a `NOTIFY` to PostgREST which tells it to re-read the schemas and provide an API. -9. Indexing +##### Indexing Now, we index *everything*. If something is not performant, *add more indexes*. diff --git a/backend/dissemination/sql/fac-snapshot-db/post/010_copy_dissem_tables.sql b/backend/dissemination/sql/fac-snapshot-db/post/010_copy_dissem_tables.sql index c4df416ee1..50d65999df 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/010_copy_dissem_tables.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/010_copy_dissem_tables.sql @@ -149,6 +149,29 @@ CREATE OR REPLACE FUNCTION dissem_copy.create_dissemination_secondaryauditor() $ct$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION dissem_copy.create_dissemination_tribalapiaccesskeyids() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE dissem_copy.dissemination_tribalapiaccesskeyids + AS SELECT * FROM public.dissemination_tribalapiaccesskeyids; + END + $ct$ + LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION dissem_copy.create_dissemination_onetimeaccess() + RETURNS VOID + AS + $ct$ + BEGIN + CREATE TABLE dissem_copy.dissemination_onetimeaccess + AS SELECT * FROM public.dissemination_onetimeaccess; + END + $ct$ + LANGUAGE plpgsql; + + DO LANGUAGE plpgsql $go$ BEGIN @@ -176,6 +199,10 @@ $go$ PERFORM dissem_copy.create_dissemination_passthrough(); RAISE info 'create_dissemination_secondaryauditor'; PERFORM dissem_copy.create_dissemination_secondaryauditor(); + RAISE INFO 'dissemination_tribalapiaccesskeyids'; + PERFORM dissem_copy.create_dissemination_tribalapiaccesskeyids(); + RAISE info 'create_dissemination_onetimeaccess'; + PERFORM dissem_copy.create_dissemination_onetimeaccess(); END $go$; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/122_api_v1_1_0_functions.sql b/backend/dissemination/sql/fac-snapshot-db/post/122_api_v1_1_0_functions.sql index 62e648d496..945edbb15e 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/122_api_v1_1_0_functions.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/122_api_v1_1_0_functions.sql @@ -41,7 +41,7 @@ BEGIN SELECT CASE WHEN EXISTS ( SELECT key_id - FROM copy.dissemination_tribalapiaccesskeyids taaki + FROM dissem_copy.dissemination_tribalapiaccesskeyids taaki WHERE taaki.key_id = uuid_header::TEXT) THEN 1::BOOLEAN ELSE 0::BOOLEAN diff --git a/backend/dissemination/sql/fac-snapshot-db/post/124_api_v1_1_0_rpc.sql b/backend/dissemination/sql/fac-snapshot-db/post/124_api_v1_1_0_rpc.sql index a5b749f6ce..9067b8ef05 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/124_api_v1_1_0_rpc.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/124_api_v1_1_0_rpc.sql @@ -11,19 +11,19 @@ BEGIN SELECT api_v1_1_0_functions.get_api_key_uuid() INTO v_uuid_header; - -- Check if the provided API key exists in copy.dissemination_TribalApiAccessKeyIds + -- Check if the provided API key exists in dissem_copy.dissemination_TribalApiAccessKeyIds SELECT EXISTS( SELECT 1 - FROM copy.dissemination_tribalapiaccesskeyids + FROM dissem_copy.dissemination_tribalapiaccesskeyids WHERE key_id = v_uuid_header ) INTO v_key_exists; - -- Get the added date of the key from copy.dissemination_TribalApiAccessKeyIds + -- Get the added date of the key from dissem_copy.dissemination_TribalApiAccessKeyIds SELECT date_added INTO v_key_added_date - FROM copy.dissemination_tribalapiaccesskeyids + FROM dissem_copy.dissemination_tribalapiaccesskeyids WHERE key_id = v_uuid_header; @@ -33,7 +33,7 @@ BEGIN SELECT gen_random_uuid() INTO v_access_uuid; -- Inserting data into the one_time_access table - INSERT INTO copy.dissemination_onetimeaccess (uuid, api_key_id, timestamp, report_id) + INSERT INTO dissem_copy.dissemination_onetimeaccess (uuid, api_key_id, timestamp, report_id) VALUES (v_access_uuid::UUID, v_uuid_header, CURRENT_TIMESTAMP, report_id); -- Return the UUID to the user diff --git a/backend/dissemination/sql/fac-snapshot-db/post/142_api_v2_0_0_functions.sql b/backend/dissemination/sql/fac-snapshot-db/post/142_api_v2_0_0_functions.sql index a8ce2c47b5..25445aa7c4 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/142_api_v2_0_0_functions.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/142_api_v2_0_0_functions.sql @@ -41,7 +41,7 @@ BEGIN SELECT CASE WHEN EXISTS ( SELECT key_id - FROM copy.dissemination_tribalapiaccesskeyids taaki + FROM dissem_copy.dissemination_tribalapiaccesskeyids taaki WHERE taaki.key_id = uuid_header::TEXT) THEN 1::BOOLEAN ELSE 0::BOOLEAN diff --git a/backend/dissemination/sql/fac-snapshot-db/post/144_api_v2_0_0_rpc.sql b/backend/dissemination/sql/fac-snapshot-db/post/144_api_v2_0_0_rpc.sql index 38e3a19621..afa8337fec 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/144_api_v2_0_0_rpc.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/144_api_v2_0_0_rpc.sql @@ -11,19 +11,19 @@ BEGIN SELECT api_v2_0_0_functions.get_api_key_uuid() INTO v_uuid_header; - -- Check if the provided API key exists in copy.dissemination_TribalApiAccessKeyIds + -- Check if the provided API key exists in dissem_copy.dissemination_TribalApiAccessKeyIds SELECT EXISTS( SELECT 1 - FROM copy.dissemination_tribalapiaccesskeyids + FROM dissem_copy.dissemination_tribalapiaccesskeyids WHERE key_id = v_uuid_header ) INTO v_key_exists; - -- Get the added date of the key from copy.dissemination_TribalApiAccessKeyIds + -- Get the added date of the key from dissem_copy.dissemination_TribalApiAccessKeyIds SELECT date_added INTO v_key_added_date - FROM copy.dissemination_tribalapiaccesskeyids + FROM dissem_copy.dissemination_tribalapiaccesskeyids WHERE key_id = v_uuid_header; @@ -33,7 +33,7 @@ BEGIN SELECT gen_random_uuid() INTO v_access_uuid; -- Inserting data into the one_time_access table - INSERT INTO copy.dissemination_onetimeaccess (uuid, api_key_id, timestamp, report_id) + INSERT INTO dissem_copy.dissemination_onetimeaccess (uuid, api_key_id, timestamp, report_id) VALUES (v_access_uuid::UUID, v_uuid_header, CURRENT_TIMESTAMP, report_id); -- Return the UUID to the user diff --git a/backend/dissemination/sql/fac-snapshot-db/post/800_permissions.sql b/backend/dissemination/sql/fac-snapshot-db/post/800_permissions.sql index 3963a5c006..0f9f127623 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/800_permissions.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/800_permissions.sql @@ -10,6 +10,8 @@ GRANT USAGE ON SCHEMA api_v1_1_0_functions TO api_fac_gov; GRANT USAGE ON SCHEMA api_v1_1_0 TO api_fac_gov; GRANT SELECT ON ALL TABLES IN SCHEMA api_v1_1_0 TO api_fac_gov; +-- GRANT SELECT ON ALL TABLES IN SCHEMA dissem_copy to api_fac_gov; + -- There are no sequences currently on api_v1_1_0 -- GRANT SELECT, USAGE ON ALL SEQUENCES IN SCHEMA api_v1_1_0 TO api_fac_gov; diff --git a/backend/dissemination/sql/fac-snapshot-db/post/810_bring_up_api.sql b/backend/dissemination/sql/fac-snapshot-db/post/999_finalize.sql similarity index 100% rename from backend/dissemination/sql/fac-snapshot-db/post/810_bring_up_api.sql rename to backend/dissemination/sql/fac-snapshot-db/post/999_finalize.sql diff --git a/backend/tools/cgov_util_local_only.sh b/backend/tools/cgov_util_local_only.sh index e340eb83ae..fecc5550ff 100644 --- a/backend/tools/cgov_util_local_only.sh +++ b/backend/tools/cgov_util_local_only.sh @@ -1,19 +1,26 @@ source tools/util_startup.sh function cgov_util_local_only() { - startup_log "CGOV_LOCAL_ONLY" "Making an initial 'backup'" + + # Really, really only run this locally. Or in a GH runner. + + if [[ "${ENV}" == "LOCAL" || "${ENV}" == "TESTING" ]]; then + startup_log "CGOV_LOCAL_ONLY" "Making an initial 'backup'" + + $PSQL_EXE $FAC_SNAPSHOT_URI -c "DROP SCHEMA IF EXISTS public CASCADE" + gonogo "DROP PUBLIC in fac-snapshot-db" + $PSQL_EXE $FAC_SNAPSHOT_URI -c "CREATE SCHEMA public" + gonogo "CREATE PUBLIC fac-snapshot-db" - check_table_exists $FAC_SNAPSHOT_URI 'public' 'dissemination_general' - local is_general_table=$FUNCTION_RESULT - if [ $is_general_table -ne 0 ]; then # This is the first run. startup_log "CGOV_LOCAL_ONLY" "Running cgov-util INITIAL." $CGOV_UTIL_EXE db_to_db \ --src_db fac-db \ --dest_db fac-snapshot-db \ --operation initial + + startup_log "CGOV_LOCAL_ONLY" "Done" fi - startup_log "CGOV_LOCAL_ONLY" "Done" return 0 } diff --git a/backend/tools/setup_cgov_env.py b/backend/tools/setup_cgov_env.py deleted file mode 100644 index 846a622c9f..0000000000 --- a/backend/tools/setup_cgov_env.py +++ /dev/null @@ -1,79 +0,0 @@ -import json -from pprint import pprint - - -def get_vcap(): - # return json.loads(os.getenv("VCAP_SERVICES")) - obj = json.load(open("example_vcap.json")) - return obj - - -# eg "user-provided", "name", ""https-proxy-creds"" -def from_array(arr, key, value): - for o in arr: - if o[key] == value: - return o - raise ValueError("Nope") - - -def at_path(vcap, keys): - if isinstance(keys, str): - keys = [keys] - struct = vcap - for k in keys: - struct = struct[k] - return struct - - -def vcap_lookup(key): - vcap = get_vcap() - - -def setup_cgov_env(): - env = {} - env["SSL_CERT_FILE"] = "/etc/ssl/certs/ca-certificates.crt" - env["REQUESTS_CA_BUNDLE"] = "/etc/ssl/certs/ca-certificates.crt" - env["https_proxy"] = at_path( - from_array(at_path(get_vcap(), ["user-provided"]), "name", "https-proxy-creds"), - ["credentials", "uri"], - ) - env["smtp_proxy_domain"] = at_path( - from_array(at_path(get_vcap(), ["user-provided"]), "name", "smtp-proxy-creds"), - ["credentials", "domain"], - ) - env["smtp_proxy_port"] = at_path( - from_array(at_path(get_vcap(), ["user-provided"]), "name", "smtp-proxy-creds"), - ["credentials", "port"], - ) - - # S3_ENDPOINT_FOR_NO_PROXY="$(echo $VCAP_SERVICES | jq --raw-output --arg service_name "fac-public-s3" ".[][] | select(.name == \$service_name) | .credentials.endpoint")" - # S3_FIPS_ENDPOINT_FOR_NO_PROXY="$(echo $VCAP_SERVICES | jq --raw-output --arg service_name "fac-public-s3" ".[][] | select(.name == \$service_name) | .credentials.fips_endpoint")" - # S3_PRIVATE_ENDPOINT_FOR_NO_PROXY="$(echo $VCAP_SERVICES | jq --raw-output --arg service_name "fac-private-s3" ".[][] | select(.name == \$service_name) | .credentials.endpoint")" - # S3_PRIVATE_FIPS_ENDPOINT_FOR_NO_PROXY="$(echo $VCAP_SERVICES | jq --raw-output --arg service_name "fac-private-s3" ".[][] | select(.name == \$service_name) | .credentials.fips_endpoint")" - # export no_proxy="${S3_ENDPOINT_FOR_NO_PROXY},${S3_FIPS_ENDPOINT_FOR_NO_PROXY},${S3_PRIVATE_ENDPOINT_FOR_NO_PROXY},${S3_PRIVATE_FIPS_ENDPOINT_FOR_NO_PROXY},apps.internal" - - S3_ENDPOINT_FOR_NO_PROXY = at_path( - from_array(at_path(get_vcap(), ["s3"]), "name", "fac-public-s3"), - ["credentials", "endpoint"], - ) - S3_FIPS_ENDPOINT_FOR_NO_PROXY = at_path( - from_array(at_path(get_vcap(), ["s3"]), "name", "fac-public-s3"), - ["credentials", "fips_endpoint"], - ) - S3_PRIVATE_ENDPOINT_FOR_NO_PROXY = at_path( - from_array(at_path(get_vcap(), ["s3"]), "name", "fac-private-s3"), - ["credentials", "endpoint"], - ) - S3_PRIVATE_FIPS_ENDPOINT_FOR_NO_PROXY = at_path( - from_array(at_path(get_vcap(), ["s3"]), "name", "fac-private-s3"), - ["credentials", "fips_endpoint"], - ) - env["no_proxy"] = ( - f"{S3_ENDPOINT_FOR_NO_PROXY},{S3_FIPS_ENDPOINT_FOR_NO_PROXY},{S3_PRIVATE_ENDPOINT_FOR_NO_PROXY},{S3_PRIVATE_FIPS_ENDPOINT_FOR_NO_PROXY},apps.internal" - ) - - pprint(env) - - -if __name__ == "__main__": - setup_cgov_env() diff --git a/backend/util/nightly_api_refresh.sh b/backend/util/nightly_api_refresh.sh index 0376b2cc8c..d63ed05557 100755 --- a/backend/util/nightly_api_refresh.sh +++ b/backend/util/nightly_api_refresh.sh @@ -17,3 +17,9 @@ gonogo "sql_pre_fac_snapshot_db" sql_post_fac_snapshot_db gonogo "sql_post_fac_snapshot_db" + +# We might, at some point, +# consider running a vacuum on DB1 +# as part of a nightly or weekly job. +# Below is *representative* code. +# run_sql $FAC_DB_URI -c "VACUUM(FULL, ANALYZE)" From ab68fa0df19d8700cfa1174d3e87e689a26bb6a7 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Fri, 1 Nov 2024 14:25:49 -0400 Subject: [PATCH 82/89] Updating tests. :wq --- .../fac-snapshot-db/post/800_permissions.sql | 4 +++- backend/dissemination/sql/test_api.py | 22 +++++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/backend/dissemination/sql/fac-snapshot-db/post/800_permissions.sql b/backend/dissemination/sql/fac-snapshot-db/post/800_permissions.sql index 0f9f127623..5778fc75bc 100644 --- a/backend/dissemination/sql/fac-snapshot-db/post/800_permissions.sql +++ b/backend/dissemination/sql/fac-snapshot-db/post/800_permissions.sql @@ -10,7 +10,9 @@ GRANT USAGE ON SCHEMA api_v1_1_0_functions TO api_fac_gov; GRANT USAGE ON SCHEMA api_v1_1_0 TO api_fac_gov; GRANT SELECT ON ALL TABLES IN SCHEMA api_v1_1_0 TO api_fac_gov; --- GRANT SELECT ON ALL TABLES IN SCHEMA dissem_copy to api_fac_gov; +-- Need these for tribal access checks -- functions and tables +GRANT SELECT ON ALL TABLES IN SCHEMA dissem_copy to api_fac_gov; +GRANT USAGE ON SCHEMA dissem_copy to api_fac_gov; -- There are no sequences currently on api_v1_1_0 -- GRANT SELECT, USAGE ON ALL SEQUENCES IN SCHEMA api_v1_1_0 TO api_fac_gov; diff --git a/backend/dissemination/sql/test_api.py b/backend/dissemination/sql/test_api.py index e43a713359..7ee0575fce 100644 --- a/backend/dissemination/sql/test_api.py +++ b/backend/dissemination/sql/test_api.py @@ -2,6 +2,17 @@ # Execute as a pytest. # pytest -s --env local test_api.py # +# set environment variable +# +# CAN_READ_SUPPRESSED=0 +# +# if you are testing with a key that *should* be rejected for +# suppressed audits, and +# +# CAN_READ_SUPPRESSED=1 +# +# if you have a key that should be able to read suppressed audits. +# The default is 0. import os import requests @@ -14,6 +25,11 @@ class EnvVars: FAC_API_KEY_ID = os.getenv("CYPRESS_API_GOV_USER_ID") FAC_AUTH_BEARER = os.getenv("CYPRESS_API_GOV_JWT") RECORDS_REQUESTED = 5 + CAN_READ_SUPPRESSED = ( + str(os.getenv("CAN_READ_SUPPRESSED")) + if os.getenv("CAN_READ_SUPPRESSED") != None + else "0" + ) def url(env): @@ -143,7 +159,8 @@ def test_suppressed_not_accessible_with_bad_key(env): try: thunk() except: - failed_count += 1 + if EnvVars.CAN_READ_SUPPRESSED == "0": + failed_count += 1 assert failed_count == 3 # Restore it in case we need it in later tests. EnvVars.FAC_API_KEY_ID = TEMP_FAC_API_KEY_ID @@ -167,5 +184,6 @@ def test_suppressed_accessible_with_good_key(env): try: thunk() except: - failed_count += 1 + if EnvVars.CAN_READ_SUPPRESSED == "1": + failed_count += 1 assert failed_count == 0 From 24c2c5c1c7ff34798124bb41c6e1f7fd8032c0e1 Mon Sep 17 00:00:00 2001 From: Bobby Novak <176936850+rnovak338@users.noreply.github.com> Date: Thu, 7 Nov 2024 13:26:36 -0500 Subject: [PATCH 83/89] Linting --- .../sql/sling/bulk_data_export/create_stream_yaml.py | 1 - backend/dissemination/sql/test_api.py | 8 ++++---- backend/util/api_perf_test/perf.py | 11 +++++------ 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/backend/dissemination/sql/sling/bulk_data_export/create_stream_yaml.py b/backend/dissemination/sql/sling/bulk_data_export/create_stream_yaml.py index a8dfe0215c..a0f370a248 100644 --- a/backend/dissemination/sql/sling/bulk_data_export/create_stream_yaml.py +++ b/backend/dissemination/sql/sling/bulk_data_export/create_stream_yaml.py @@ -5,7 +5,6 @@ # to regenerate the sling file. import yaml -from pprint import pprint from datetime import datetime obj = {"streams": dict()} diff --git a/backend/dissemination/sql/test_api.py b/backend/dissemination/sql/test_api.py index 7ee0575fce..02189492cc 100644 --- a/backend/dissemination/sql/test_api.py +++ b/backend/dissemination/sql/test_api.py @@ -27,7 +27,7 @@ class EnvVars: RECORDS_REQUESTED = 5 CAN_READ_SUPPRESSED = ( str(os.getenv("CAN_READ_SUPPRESSED")) - if os.getenv("CAN_READ_SUPPRESSED") != None + if os.getenv("CAN_READ_SUPPRESSED") is not None else "0" ) @@ -125,7 +125,7 @@ def test_api_v1_0_3_not_exist(env): common_tables(f) print("This schema/API should not exist.") assert False - except: + except Exception: pass @@ -158,7 +158,7 @@ def test_suppressed_not_accessible_with_bad_key(env): ]: try: thunk() - except: + except Exception: if EnvVars.CAN_READ_SUPPRESSED == "0": failed_count += 1 assert failed_count == 3 @@ -183,7 +183,7 @@ def test_suppressed_accessible_with_good_key(env): ]: try: thunk() - except: + except Exception: if EnvVars.CAN_READ_SUPPRESSED == "1": failed_count += 1 assert failed_count == 0 diff --git a/backend/util/api_perf_test/perf.py b/backend/util/api_perf_test/perf.py index cb3e1ac3b0..6dbefa8c8c 100644 --- a/backend/util/api_perf_test/perf.py +++ b/backend/util/api_perf_test/perf.py @@ -3,7 +3,6 @@ import time from pprint import pprint import math -import json URI = "http://localhost:3000" @@ -64,11 +63,11 @@ def fetch_fa_time(api_version): "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", } t0 = time.time() - resp = requests.get(query, headers=headers) + requests.get(query, headers=headers) t1 = time.time() # We get back a list of one plan, and we want the total cost. total_cost += t1 - t0 - print(f"fetch_fa_time api {api_version} offset {offset} time {t1-t0}") + print(f"fetch_fa_time api {api_version} offset {offset} time {t1 - t0}") return math.floor(total_cost) @@ -84,12 +83,12 @@ def fetch_fa_time_by_year(api_version): "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", } t0 = time.time() - resp = requests.get(query, headers=headers) + requests.get(query, headers=headers) t1 = time.time() # We get back a list of one plan, and we want the total cost. total_cost += t1 - t0 print( - f"fetch_fa_time_by_year api {api_version} ay {audit_year} offset {offset} time {t1-t0}" + f"fetch_fa_time_by_year api {api_version} ay {audit_year} offset {offset} time {t1 - t0}" ) return math.floor(total_cost) @@ -122,7 +121,7 @@ def fetch_fa_batches_time(): "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", } t0 = time.time() - resp = requests.get(query, headers=headers) + requests.get(query, headers=headers) t1 = time.time() # We get back a list of one plan, and we want the total cost. total_cost += t1 - t0 From 8815dd26e90c3437f90993a190d92f939f3c40a5 Mon Sep 17 00:00:00 2001 From: Bobby Novak <176936850+rnovak338@users.noreply.github.com> Date: Thu, 7 Nov 2024 14:31:01 -0500 Subject: [PATCH 84/89] Document django admin access --- docs/user-management.md | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/docs/user-management.md b/docs/user-management.md index c7d999da42..89b82654b5 100644 --- a/docs/user-management.md +++ b/docs/user-management.md @@ -6,7 +6,9 @@ The first time you login to the application, a user record in the FAC database w ## Admin users -To promote a user to either staff or superuser status, we'll use a Django management command. If you're running locally, the management command can be executed directly. +### Local development + +To acquire admin access in the local environment, you can execute a management command directly. ```bash # Run make_staff to promote a user to staff status @@ -16,23 +18,13 @@ python manage.py make_staff user@example.com python manage.py make_super user@example.com ``` -To run it against a deployed instance, we'll SSH into the app, [configure our session](https://cloud.gov/docs/management/using-ssh/#application-debugging-tips), and then run the command. - -```bash -# SSH to cloud.gov **instance** -cf ssh gsa-fac-dev +### Deployed instances -# Configure session per cloud.gov docs -/tmp/lifecycle/shell +To add/remove/promote a user in the admin interface, you would modify the [staffusers.json](../backend/config/staffusers.json) list and submit a PR for this change. -# Run make_staff to promote a user to staff status -python manage.py make_staff user@example.com - -# Run make_super to promote a user to superuser status -python manage.py make_super user@example.com -``` +Once the application starts up, it will adjust the user access to the admin site based on the updated list. If you are included in this list, you will be able to access the site via the `/admin` page. -Once your user is promoted, you'll be able to access the admin site via the `/admin` page. +**NOTE** - The email addresses included in this list MUST have a Login.gov account and have used it to log in to the application at least once. Otherwise, the application will ignore the email address on startup. ## Non-admin users From f2084e9fd0413b6c1bc31180422638915cb3c3cb Mon Sep 17 00:00:00 2001 From: Bobby Novak <176936850+rnovak338@users.noreply.github.com> Date: Thu, 7 Nov 2024 14:51:50 -0500 Subject: [PATCH 85/89] Linting --- backend/config/settings.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/backend/config/settings.py b/backend/config/settings.py index cf006a6c38..40ae90cffc 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -229,6 +229,16 @@ DEBUG = False if ENVIRONMENT not in ["SANDBOX", "DEVELOPMENT", "PREVIEW", "STAGING", "PRODUCTION"]: + + # FIXME: This is now identical between local and cloud.gov, because we have + # a "fake" VCAP_SERVICES environment variable. Local DBs and S3 buckets + # can be configured the same way as their cloud equivalents. This can be + # refactored for simpler config loading in the app. + # + # During a build, there won't be an environment variable. Load the + # fake VCAP from the filesystem. + vcap = json.load(open("config/vcap_services_for_containers.json")) + DATABASES = { "default": dj_database_url.parse(get_db_url_from_vcap_services(vcap, "fac-db")), "fac-db": dj_database_url.parse(get_db_url_from_vcap_services(vcap, "fac-db")), From b04e944a081f84554b5a6d8eef142b23ea784e1c Mon Sep 17 00:00:00 2001 From: Bobby Novak <176936850+rnovak338@users.noreply.github.com> Date: Thu, 7 Nov 2024 19:35:53 -0500 Subject: [PATCH 86/89] More linting - bandit Added max timeout to request --- backend/util/api_perf_test/perf.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/util/api_perf_test/perf.py b/backend/util/api_perf_test/perf.py index 6dbefa8c8c..e999544020 100644 --- a/backend/util/api_perf_test/perf.py +++ b/backend/util/api_perf_test/perf.py @@ -25,7 +25,7 @@ def fetch_fa_exp(api_version): "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", } - resp = requests.get(query, headers=headers) + resp = requests.get(query, headers=headers, timeout=60) # We get back a list of one plan, and we want the total cost. total_cost += resp.json()[0]["Plan"]["Total Cost"] return math.floor(total_cost) @@ -47,7 +47,7 @@ def fetch_fa_by_year_exp(api_version): "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", } - resp = requests.get(query, headers=headers) + resp = requests.get(query, headers=headers, timeout=60) # We get back a list of one plan, and we want the total cost. total_cost += resp.json()[0]["Plan"]["Total Cost"] return math.floor(total_cost) @@ -63,7 +63,7 @@ def fetch_fa_time(api_version): "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", } t0 = time.time() - requests.get(query, headers=headers) + requests.get(query, headers=headers, timeout=60) t1 = time.time() # We get back a list of one plan, and we want the total cost. total_cost += t1 - t0 @@ -83,7 +83,7 @@ def fetch_fa_time_by_year(api_version): "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", } t0 = time.time() - requests.get(query, headers=headers) + requests.get(query, headers=headers, timeout=60) t1 = time.time() # We get back a list of one plan, and we want the total cost. total_cost += t1 - t0 @@ -104,7 +104,7 @@ def fetch_fa_batches_exp(): "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", } - resp = requests.get(query, headers=headers) + resp = requests.get(query, headers=headers, timeout=60) # We get back a list of one plan, and we want the total cost. total_cost += resp.json()[0]["Plan"]["Total Cost"] return math.floor(total_cost) @@ -121,7 +121,7 @@ def fetch_fa_batches_time(): "authorization": f"bearer {os.getenv('CYPRESS_API_GOV_JWT')}", } t0 = time.time() - requests.get(query, headers=headers) + requests.get(query, headers=headers, timeout=60) t1 = time.time() # We get back a list of one plan, and we want the total cost. total_cost += t1 - t0 From 02560471810eccaa64d575f4acd81bd7f6743ce4 Mon Sep 17 00:00:00 2001 From: Bobby Novak <176936850+rnovak338@users.noreply.github.com> Date: Thu, 7 Nov 2024 19:38:27 -0500 Subject: [PATCH 87/89] Reformat local python API tests to rest of the testing suite Now uses Django factory testing (like we do with our other tests that fall under `manage.py test`) for the tests that were written up by Matt. INCOMPLETE - check the "FIXME" comment. Currently the environment is hardcoded to "local" under the `ApiTests` class, and I haven't gotten a "success" response from the following tests. - `test_suppressed_not_accessible_with_bad_key` - `test_suppressed_accessible_with_good_key` --- backend/dissemination/sql/test_api.py | 227 ++++++++++++-------------- 1 file changed, 108 insertions(+), 119 deletions(-) diff --git a/backend/dissemination/sql/test_api.py b/backend/dissemination/sql/test_api.py index 02189492cc..47ac588f4f 100644 --- a/backend/dissemination/sql/test_api.py +++ b/backend/dissemination/sql/test_api.py @@ -1,19 +1,4 @@ -# -# Execute as a pytest. -# pytest -s --env local test_api.py -# -# set environment variable -# -# CAN_READ_SUPPRESSED=0 -# -# if you are testing with a key that *should* be rejected for -# suppressed audits, and -# -# CAN_READ_SUPPRESSED=1 -# -# if you have a key that should be able to read suppressed audits. -# The default is 0. - +from django.test import TestCase import os import requests import sys @@ -82,108 +67,112 @@ def limit(start, end): return {"Range-Unit": "items", "Range": f"{start}-{end}"} -# Asserts that an API response is: -# * A list -# * A list composed of objects that all contain the required keys -def good_resp(objs, keys): - assert isinstance(objs, list) - assert len(objs) == EnvVars.RECORDS_REQUESTED - for k in keys: - for o in objs: - # print(f"Checking {k} in {o}") - assert k in o - return True - - -# Constructs the base URL for making multiple API calls off of. -def cons(env, api_version): - def _helper(endpoint, keys): - base = url(env) - h = headers(env) | limit(0, EnvVars.RECORDS_REQUESTED - 1) | api(api_version) - r = requests.get(base + f"/{endpoint}", headers=h) - # print(r.request.url) - # print(r.request.headers) - good_resp(r.json(), keys) - - return _helper - - -# These tables are common to both the old API and -# the new public data API. -def common_tables(f): - f("general", ["report_id", "audit_year", "auditee_name"]) - f( - "federal_awards", - ["report_id", "amount_expended", "audit_report_type"], - ) - f("corrective_action_plans", ["report_id", "finding_ref_number", "auditee_uei"]) - - -def test_api_v1_0_3_not_exist(env): - f = cons(env, "api_v1_0_3") - try: - common_tables(f) - print("This schema/API should not exist.") - assert False - except Exception: - pass - - -def test_api_v1_1_0(env): - f = cons(env, "api_v1_1_0") - common_tables(f) - - -def test_api_v2_0_0(env): - f = cons(env, "api_v2_0_0") - common_tables(f) - - -def test_suppressed_not_accessible_with_bad_key(env): - # Stash the token, and wipe it out, so the API - # calls will fail. - TEMP_FAC_API_KEY_ID = EnvVars.FAC_API_KEY_ID - EnvVars.FAC_API_KEY_ID = str(uuid.uuid4()) - f = cons(env, "api_v2_0_0") - failed_count = 0 - for thunk in [ - lambda: f( - "suppressed_notes_to_sefa", ["report_id", "content", "is_minimis_rate_used"] - ), - lambda: f("suppressed_findings_text", ["report_id", "finding_ref_number"]), - lambda: f( - "suppressed_corrective_action_plans", - ["report_id", "finding_ref_number", "planned_action"], - ), - ]: - try: - thunk() - except Exception: - if EnvVars.CAN_READ_SUPPRESSED == "0": - failed_count += 1 - assert failed_count == 3 - # Restore it in case we need it in later tests. - EnvVars.FAC_API_KEY_ID = TEMP_FAC_API_KEY_ID - - -def test_suppressed_accessible_with_good_key(env): - # Stash the token, and wipe it out, so the API - # calls will fail. - f = cons(env, "api_v2_0_0") - failed_count = 0 - for thunk in [ - lambda: f( - "suppressed_notes_to_sefa", ["report_id", "content", "is_minimis_rate_used"] - ), - lambda: f("suppressed_findings_text", ["report_id", "finding_ref_number"]), - lambda: f( - "suppressed_corrective_action_plans", - ["report_id", "finding_ref_number", "planned_action"], - ), - ]: +class ApiTests(TestCase): + + ENV = "local" + + def good_resp(self, objs, keys): + """ + Asserts that an API response is: + * A list + * A list composed of objects that all contain the required keys + """ + self.assertIsInstance(objs, list) + self.assertEqual(len(objs), EnvVars.RECORDS_REQUESTED) + for k in keys: + for o in objs: + self.assertIn(k, o) + return True + + def cons(self, env, api_version): + """Constructs the base URL for making multiple API calls off of.""" + + # FIXME: currently, both tests that use this method fail over a "ConnectionRefusedError". + def _helper(endpoint, keys): + base = url(env) + h = ( + headers(env) + | limit(0, EnvVars.RECORDS_REQUESTED - 1) + | api(api_version) + ) + r = requests.get(base + f"/{endpoint}", headers=h) + self.good_resp(r.json(), keys) + + return _helper + + def common_tables(self, f): + """These tables are common to both the old API and the new public data API.""" + + f("general", ["report_id", "audit_year", "auditee_name"]) + f( + "federal_awards", + ["report_id", "amount_expended", "audit_report_type"], + ) + f("corrective_action_plans", ["report_id", "finding_ref_number", "auditee_uei"]) + + def test_api_v1_0_3_not_exist(self): + f = self.cons(self.ENV, "api_v1_0_3") try: - thunk() + self.common_tables(f) + print("This schema/API should not exist.") + self.assertTrue(False) except Exception: - if EnvVars.CAN_READ_SUPPRESSED == "1": - failed_count += 1 - assert failed_count == 0 + pass + + def test_api_v1_1_0(self): + f = self.cons(self.ENV, "api_v1_1_0") + self.common_tables(f) + + def test_api_v2_0_0(self): + f = self.cons(self.ENV, "api_v2_0_0") + self.common_tables(f) + + def test_suppressed_not_accessible_with_bad_key(self): + # Stash the token, and wipe it out, so the API + # calls will fail. + TEMP_FAC_API_KEY_ID = EnvVars.FAC_API_KEY_ID + EnvVars.FAC_API_KEY_ID = str(uuid.uuid4()) + f = self.cons(self.ENV, "api_v2_0_0") + failed_count = 0 + for thunk in [ + lambda: f( + "suppressed_notes_to_sefa", + ["report_id", "content", "is_minimis_rate_used"], + ), + lambda: f("suppressed_findings_text", ["report_id", "finding_ref_number"]), + lambda: f( + "suppressed_corrective_action_plans", + ["report_id", "finding_ref_number", "planned_action"], + ), + ]: + try: + thunk() + except Exception: + if EnvVars.CAN_READ_SUPPRESSED == "0": + failed_count += 1 + self.assertEqual(failed_count, 3) + # Restore it in case we need it in later tests. + EnvVars.FAC_API_KEY_ID = TEMP_FAC_API_KEY_ID + + def test_suppressed_accessible_with_good_key(self): + # Stash the token, and wipe it out, so the API + # calls will fail. + f = self.cons(self.ENV, "api_v2_0_0") + failed_count = 0 + for thunk in [ + lambda: f( + "suppressed_notes_to_sefa", + ["report_id", "content", "is_minimis_rate_used"], + ), + lambda: f("suppressed_findings_text", ["report_id", "finding_ref_number"]), + lambda: f( + "suppressed_corrective_action_plans", + ["report_id", "finding_ref_number", "planned_action"], + ), + ]: + try: + thunk() + except Exception: + if EnvVars.CAN_READ_SUPPRESSED == "1": + failed_count += 1 + self.assertEqual(failed_count, 0) From 015af342b2b1363b390193b75c9950863f77a53a Mon Sep 17 00:00:00 2001 From: Bobby Novak <176936850+rnovak338@users.noreply.github.com> Date: Thu, 7 Nov 2024 19:46:23 -0500 Subject: [PATCH 88/89] FE Linting --- .../cypress/support/dissemination-table-via-pdf.js | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/backend/cypress/support/dissemination-table-via-pdf.js b/backend/cypress/support/dissemination-table-via-pdf.js index d0df9bfbec..173a814fb0 100644 --- a/backend/cypress/support/dissemination-table-via-pdf.js +++ b/backend/cypress/support/dissemination-table-via-pdf.js @@ -2,17 +2,6 @@ Re-useable code for testing the dissemination table. */ -const API_GOV_JWT = Cypress.env('API_GOV_JWT') || ''; -const API_GOV_KEY = Cypress.env('API_GOV_KEY') || ''; -const API_GOV_USER_ID = Cypress.env('API_GOV_USER_ID'); -const API_GOV_KEY_ADMIN = Cypress.env('API_GOV_KEY_ADMIN'); -const API_GOV_USER_ID_ADMIN = Cypress.env('API_GOV_USER_ID_ADMIN'); -const API_GOV_URL = Cypress.env('API_GOV_URL'); -const API_VERSION = Cypress.env('API_VERSION'); -const ADMIN_API_VERSION = Cypress.env('ADMIN_API_VERSION'); - -const API_GOV_USER_EMAIL = `${API_GOV_USER_ID}@example.com`; - // We're testing a 2x2. Actually, this would be better as a table, as this is a 3D test. // is tribal is not tribal // ┌────────────────────────────────┬────────────────────────────────┐ From 04e19e4f6a92aeb33012d6ca0744972523514a69 Mon Sep 17 00:00:00 2001 From: Bobby Novak <176936850+rnovak338@users.noreply.github.com> Date: Thu, 14 Nov 2024 16:49:14 -0500 Subject: [PATCH 89/89] Linting --- .../reprocess_migration_cli_commands.py | 2 +- .../start_process_cli_commands.py | 4 ++- .../bulk_data_export/create_stream_yaml.py | 35 ++++++++++--------- backend/dissemination/sql/test_api.py | 4 +-- backend/pyproject.toml | 3 ++ backend/util/api_perf_test/perf.py | 8 ++--- 6 files changed, 31 insertions(+), 25 deletions(-) diff --git a/backend/census_historical_migration/throwaway_scripts/reprocess_migration_cli_commands.py b/backend/census_historical_migration/throwaway_scripts/reprocess_migration_cli_commands.py index 09005e5f9f..95a2a48320 100644 --- a/backend/census_historical_migration/throwaway_scripts/reprocess_migration_cli_commands.py +++ b/backend/census_historical_migration/throwaway_scripts/reprocess_migration_cli_commands.py @@ -1,7 +1,7 @@ import argparse import time -from util import ( +from census_historical_migration.throwaway_scripts.util import ( trigger_migration_workflow, ) import subprocess # nosec diff --git a/backend/census_historical_migration/throwaway_scripts/start_process_cli_commands.py b/backend/census_historical_migration/throwaway_scripts/start_process_cli_commands.py index b7be200650..12a20a64f9 100644 --- a/backend/census_historical_migration/throwaway_scripts/start_process_cli_commands.py +++ b/backend/census_historical_migration/throwaway_scripts/start_process_cli_commands.py @@ -2,7 +2,9 @@ import subprocess # nosec import time -from util import trigger_migration_workflow +from census_historical_migration.throwaway_scripts.util import ( + trigger_migration_workflow, +) # This throwaway script spits out code that can be # copy-pasted into a bash script, or directly into the command line. diff --git a/backend/dissemination/sql/sling/bulk_data_export/create_stream_yaml.py b/backend/dissemination/sql/sling/bulk_data_export/create_stream_yaml.py index a0f370a248..8b07ec32a3 100644 --- a/backend/dissemination/sql/sling/bulk_data_export/create_stream_yaml.py +++ b/backend/dissemination/sql/sling/bulk_data_export/create_stream_yaml.py @@ -7,21 +7,22 @@ import yaml from datetime import datetime -obj = {"streams": dict()} - -obj["source"] = "FAC_SNAPSHOT_URI" -obj["target"] = "BULK_DATA_EXPORT" -obj["defaults"] = { - "target_options": { - "format": "csv", - "compression": "gzip", - "file_max_rows": 0, - } +obj: dict = { + "streams": {}, + "source": "FAC_SNAPSHOT_URI", + "target": "BULK_DATA_EXPORT", + "defaults": { + "target_options": { + "format": "csv", + "compression": "gzip", + "file_max_rows": 0, + } + }, } -schema = "public_data_v1_0_0" +SCHEMA = "public_data_v1_0_0" -tables = [ +TABLES = [ "additional_eins", "additional_ueis", "combined", @@ -39,15 +40,15 @@ # it will not generate anything where data does not exist. # This future-proofs us for a year or two, so we don't have to worry # about updating the `sling` script that is created. -years = range(2016, 2031) +YEARS = range(2016, 2031) -for t in tables: +for t in TABLES: ndx = 0 - for y in years: - obj["streams"][f"{schema}.{t}.{ndx}"] = { + for y in YEARS: + obj["streams"][f"{SCHEMA}.{t}.{ndx}"] = { "object": f"bulk_export/{{MM}}/{y}_{t}.csv", - "sql": f"SELECT * FROM {schema}.{t} WHERE audit_year = '{y}'", + "sql": f"SELECT * FROM {SCHEMA}.{t} WHERE audit_year = '{y}'", "mode": "full-refresh", "target_options": { "format": "csv", diff --git a/backend/dissemination/sql/test_api.py b/backend/dissemination/sql/test_api.py index 47ac588f4f..69e246659e 100644 --- a/backend/dissemination/sql/test_api.py +++ b/backend/dissemination/sql/test_api.py @@ -95,7 +95,7 @@ def _helper(endpoint, keys): | limit(0, EnvVars.RECORDS_REQUESTED - 1) | api(api_version) ) - r = requests.get(base + f"/{endpoint}", headers=h) + r = requests.get(base + f"/{endpoint}", headers=h, timeout=60) self.good_resp(r.json(), keys) return _helper @@ -117,7 +117,7 @@ def test_api_v1_0_3_not_exist(self): print("This schema/API should not exist.") self.assertTrue(False) except Exception: - pass + self.assertTrue(True) def test_api_v1_1_0(self): f = self.cons(self.ENV, "api_v1_1_0") diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 1ac8180ed8..267a9480bd 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -35,11 +35,14 @@ disable = [ [tool.bandit] # TDB 2022-12-07 - test_schemas and test_validators use random number generation, which bandit doesn't like +# 2024-11-14 - create_stream_yaml and create_partition_statements will never be touched by users, yet bandit doesn't like the SQL injection risk. exclude_dirs = [ "audit/cross_validation/test_check_has_federal_awards.py", "audit/test_schemas.py", "audit/test_validators.py", "node_modules", + "dissemination/sql/fac-snapshot-db/create_partition_statements.py", + "dissemination/sql/sling/bulk_data_export/create_stream_yaml.py", "dissemination/test_search.py", "dissemination/tests.py", ".venv" diff --git a/backend/util/api_perf_test/perf.py b/backend/util/api_perf_test/perf.py index e999544020..9264fd6cd2 100644 --- a/backend/util/api_perf_test/perf.py +++ b/backend/util/api_perf_test/perf.py @@ -138,10 +138,10 @@ def make_ratios(d1, d2): if __name__ == "__main__": - results1 = {} - results2 = {} - results3 = {} - results4 = {} + results1: dict = {} + results2: dict = {} + results3: dict = {} + results4: dict = {} results1["api110_by_year"] = fetch_fa_by_year_exp("api_v1_1_0") results1["ap110"] = fetch_fa_exp("api_v1_1_0")