From 00648b7f64c8a13f1f714d1cccac542244a889a9 Mon Sep 17 00:00:00 2001 From: shahargl Date: Sun, 26 Sep 2021 10:12:49 +0300 Subject: [PATCH 1/5] Add the option to run the queries sync --- .gitignore | 5 ++++- main.py | 6 +++++- snowflake_connector.py | 6 +++++- utils.py | 15 +++++++++++++++ 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index ec4892f..aa180a3 100644 --- a/.gitignore +++ b/.gitignore @@ -129,4 +129,7 @@ dmypy.json .pyre/ # visual studio code -.vscode \ No newline at end of file +.vscode + +# PyCharm +.idea \ No newline at end of file diff --git a/main.py b/main.py index ae05076..234f61b 100644 --- a/main.py +++ b/main.py @@ -10,6 +10,7 @@ def main(): load_dotenv() # only on local run queries_list = os.environ['INPUT_QUERIES'].split(';') + sync = os.environ.get("INPUT_SYNC", False) warehouse = os.environ['INPUT_SNOWFLAKE_WAREHOUSE'] snowflake_account = os.environ['INPUT_SNOWFLAKE_ACCOUNT'] snowflake_username = os.environ['INPUT_SNOWFLAKE_USERNAME'] @@ -33,7 +34,10 @@ def main(): print(f"[!] Query id - {query_result.query_id}") print(f"[!] Running query ### - {query}") - json_results = asyncio.run(utils.gather_all_results(query_results)) + if not sync: + json_results = asyncio.run(utils.gather_all_results(query_results)) + else: + json_results = utils.gather_all_results_sync(query_results) utils.set_github_action_output('queries_results', json.dumps(json_results)) diff --git a/snowflake_connector.py b/snowflake_connector.py index 202f68a..e153e1f 100644 --- a/snowflake_connector.py +++ b/snowflake_connector.py @@ -64,4 +64,8 @@ async def fetch_results(self): while self.is_query_running(): await asyncio.sleep(0.1) - return self._fetch_results() \ No newline at end of file + return self._fetch_results() + + def fetch_results_sync(self): + self.cursor.get_results_from_sfqid(self.query_id) + return self.cursor.fetchall() \ No newline at end of file diff --git a/utils.py b/utils.py index 1acb1c2..79d691e 100644 --- a/utils.py +++ b/utils.py @@ -35,4 +35,19 @@ async def gather_all_results(query_result_list: List[QueryResult]) -> dict: print(row) json_total_results[done_task.get_name()].append(str(row)) + return json_total_results + + +def gather_all_results_sync(query_results_list: List[QueryResult]) -> dict: + """ + + Args: + query_results_list: + + Returns: + + """ + json_total_results = {} + for query in query_results_list: + json_total_results[query.query_id] = query.fetch_results_sync() return json_total_results \ No newline at end of file From 346b92fdfb7badb8806f05ec973e18e68efca472 Mon Sep 17 00:00:00 2001 From: shahargl Date: Sun, 26 Sep 2021 10:21:39 +0300 Subject: [PATCH 2/5] removed redundancies --- main.py | 26 +++++++++++++++++--------- snowflake_connector.py | 6 +----- utils.py | 15 --------------- 3 files changed, 18 insertions(+), 29 deletions(-) diff --git a/main.py b/main.py index 234f61b..a28c41b 100644 --- a/main.py +++ b/main.py @@ -27,19 +27,27 @@ def main(): con.set_db_warehouse(warehouse) query_results = [] - for query in queries_list: - query_result = con.query(query) - query_results.append(query_result) - print("### Running query ###") - print(f"[!] Query id - {query_result.query_id}") - print(f"[!] Running query ### - {query}") - + # default, run all queries async if not sync: + for query in queries_list: + query_result = con.query(query) + query_results.append(query_result) + print("### Running query ###") + print(f"[!] Query id - {query_result.query_id}") + print(f"[!] Running query ### - {query}") json_results = asyncio.run(utils.gather_all_results(query_results)) + # o/w, run them sync else: - json_results = utils.gather_all_results_sync(query_results) + json_results = {} + for query in queries_list: + query_result = con.query(query) + print("### Running query ###") + print(f"[!] Query id - {query_result.query_id}") + print(f"[!] Running query ### - {query}") + json_results[query_result.query_id] = query_result.fetch_all() utils.set_github_action_output('queries_results', json.dumps(json_results)) - + + if __name__ == '__main__': main() \ No newline at end of file diff --git a/snowflake_connector.py b/snowflake_connector.py index e153e1f..202f68a 100644 --- a/snowflake_connector.py +++ b/snowflake_connector.py @@ -64,8 +64,4 @@ async def fetch_results(self): while self.is_query_running(): await asyncio.sleep(0.1) - return self._fetch_results() - - def fetch_results_sync(self): - self.cursor.get_results_from_sfqid(self.query_id) - return self.cursor.fetchall() \ No newline at end of file + return self._fetch_results() \ No newline at end of file diff --git a/utils.py b/utils.py index 79d691e..1acb1c2 100644 --- a/utils.py +++ b/utils.py @@ -35,19 +35,4 @@ async def gather_all_results(query_result_list: List[QueryResult]) -> dict: print(row) json_total_results[done_task.get_name()].append(str(row)) - return json_total_results - - -def gather_all_results_sync(query_results_list: List[QueryResult]) -> dict: - """ - - Args: - query_results_list: - - Returns: - - """ - json_total_results = {} - for query in query_results_list: - json_total_results[query.query_id] = query.fetch_results_sync() return json_total_results \ No newline at end of file From 0dd474ba9a7f9a41e4a121384e117f1bf355f785 Mon Sep 17 00:00:00 2001 From: shahargl Date: Sun, 26 Sep 2021 10:26:37 +0300 Subject: [PATCH 3/5] Done --- action.yml | 3 +++ main.py | 2 +- snowflake_connector.py | 6 +++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/action.yml b/action.yml index d5666c2..a2734e5 100644 --- a/action.yml +++ b/action.yml @@ -25,6 +25,9 @@ inputs: queries: description: 'List of SQL queries, seperated by ;' required: true + sync: + description: 'Whether to run the queries sync (default is async)' + required: false outputs: queries_results: diff --git a/main.py b/main.py index a28c41b..00d0d10 100644 --- a/main.py +++ b/main.py @@ -44,7 +44,7 @@ def main(): print("### Running query ###") print(f"[!] Query id - {query_result.query_id}") print(f"[!] Running query ### - {query}") - json_results[query_result.query_id] = query_result.fetch_all() + json_results[query_result.query_id] = query_result.fetch_results_sync() utils.set_github_action_output('queries_results', json.dumps(json_results)) diff --git a/snowflake_connector.py b/snowflake_connector.py index 202f68a..67b7fc3 100644 --- a/snowflake_connector.py +++ b/snowflake_connector.py @@ -64,4 +64,8 @@ async def fetch_results(self): while self.is_query_running(): await asyncio.sleep(0.1) - return self._fetch_results() \ No newline at end of file + return self._fetch_results() + + def fetch_results_sync(self): + self.cursor.get_results_from_sfqid(self.query_id) + return self.cursor.fetchall() From bb3360e033091c6724dd28b24488b6d23611e9ee Mon Sep 17 00:00:00 2001 From: shahargl Date: Sun, 26 Sep 2021 11:26:05 +0300 Subject: [PATCH 4/5] UTs --- main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/main.py b/main.py index 00d0d10..855666d 100644 --- a/main.py +++ b/main.py @@ -9,6 +9,7 @@ def main(): load_dotenv() # only on local run + print(os.environ) queries_list = os.environ['INPUT_QUERIES'].split(';') sync = os.environ.get("INPUT_SYNC", False) warehouse = os.environ['INPUT_SNOWFLAKE_WAREHOUSE'] From cbf33c7c3f0d7d9c2c8271a3bb7ffe9382940501 Mon Sep 17 00:00:00 2001 From: Tal Borenstein <68807791+talboren@users.noreply.github.com> Date: Sun, 26 Sep 2021 13:32:07 +0300 Subject: [PATCH 5/5] Update sanity.yml --- .github/workflows/sanity.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sanity.yml b/.github/workflows/sanity.yml index ca07b77..156081b 100644 --- a/.github/workflows/sanity.yml +++ b/.github/workflows/sanity.yml @@ -15,7 +15,7 @@ jobs: with: snowflake_account: ${{ secrets.SNOWFLAKE_ACCOUNT }} snowflake_warehouse: ${{ secrets.SNOWFLAKE_WAREHOUSE }} - snowflake_username: ${{ secrets.SNOWFLAKE_USER }} + snowflake_username: ${{ secrets.SNOWFLAKE_USERNAME }} snowflake_password: ${{ secrets.SNOWFLAKE_PASSWORD }} queries: 'call system$wait(1); select CURRENT_VERSION(); @@ -45,7 +45,7 @@ jobs: with: snowflake_account: ${{ secrets.SNOWFLAKE_ACCOUNT }} snowflake_warehouse: ${{ secrets.SNOWFLAKE_WAREHOUSE }} - snowflake_username: ${{ secrets.SNOWFLAKE_USER }} + snowflake_username: ${{ secrets.SNOWFLAKE_USERNAME }} snowflake_password: ${{ secrets.SNOWFLAKE_PASSWORD }} queries: 'call system$wait(${{matrix.sleep}})'