From 4f9191b1ef723147bfa26a9027d0188e39c19a32 Mon Sep 17 00:00:00 2001 From: Justin Cohen Date: Wed, 23 Aug 2023 11:05:28 -0400 Subject: [PATCH 1/9] swap sqlalchemy.utils unquote_plus for urllib.parse unquote_plus --- progress_sa/pyodbc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/progress_sa/pyodbc.py b/progress_sa/pyodbc.py index 3de1e66..5a5ddb0 100644 --- a/progress_sa/pyodbc.py +++ b/progress_sa/pyodbc.py @@ -1,6 +1,7 @@ from progress_sa.base import ProgressDialect, ProgressExecutionContext from sqlalchemy.connectors.pyodbc import PyODBCConnector from sqlalchemy import util +from urllib.parse import unquote_plus import sys @@ -30,7 +31,7 @@ def create_connect_args(self, url): connect_args[param] = util.asbool(keys.pop(param)) if "odbc_connect" in keys: - connectors = [util.unquote_plus(keys.pop("odbc_connect"))] + connectors = [unquote_plus(keys.pop("odbc_connect"))] else: def check_quote(token): From e4d889aed700997961e9908e915ad8791d75ae33 Mon Sep 17 00:00:00 2001 From: Justin Cohen Date: Mon, 28 Aug 2023 09:54:07 -0400 Subject: [PATCH 2/9] add get_schema_names method --- progress_sa/base.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/progress_sa/base.py b/progress_sa/base.py index 0f96566..fd2dd24 100644 --- a/progress_sa/base.py +++ b/progress_sa/base.py @@ -197,6 +197,11 @@ def _get_default_schema_name(self, connection): def _get_raw_cursor(self, connection): return connection.engine.raw_connection().cursor() + + def get_schema_names(self, connection, **kw): + cursor = self._get_raw_cursor(connection) + schemas = cursor.execute('select distinct owner from sysprogress.SYSTABLES') + return [row[0] for row in schemas] def get_table_names(self, connection, schema): cursor = self._get_raw_cursor(connection) From 1ab6ecb6ad419dd50f40ea98b2d13e1793a5dff5 Mon Sep 17 00:00:00 2001 From: Justin Cohen Date: Mon, 28 Aug 2023 09:54:49 -0400 Subject: [PATCH 3/9] add **kw to get_table_names --- progress_sa/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/progress_sa/base.py b/progress_sa/base.py index fd2dd24..106ed2d 100644 --- a/progress_sa/base.py +++ b/progress_sa/base.py @@ -203,7 +203,7 @@ def get_schema_names(self, connection, **kw): schemas = cursor.execute('select distinct owner from sysprogress.SYSTABLES') return [row[0] for row in schemas] - def get_table_names(self, connection, schema): + def get_table_names(self, connection, schema, **kw): cursor = self._get_raw_cursor(connection) s = cursor.tables(schema=schema or '') return [row.table_name for row in s] From 83cf990b41952735a0fa937476280379ed0d20ad Mon Sep 17 00:00:00 2001 From: Justin Cohen Date: Mon, 28 Aug 2023 09:57:35 -0400 Subject: [PATCH 4/9] add method get_pk_constraint --- progress_sa/base.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/progress_sa/base.py b/progress_sa/base.py index 106ed2d..ce924fd 100644 --- a/progress_sa/base.py +++ b/progress_sa/base.py @@ -207,6 +207,7 @@ def get_table_names(self, connection, schema, **kw): cursor = self._get_raw_cursor(connection) s = cursor.tables(schema=schema or '') return [row.table_name for row in s] + def has_table(self, connection, tablename, schema=None): cursor = self._get_raw_cursor(connection) @@ -242,6 +243,25 @@ def get_primary_keys(self, connection, table_name, schema=None, **kw): def get_foreign_keys(self, *args, **kw): # does not seem to be implemented in the database. return [] + + def get_pk_constraint(self, connection, tablename, schema, **kw): + pkeys = [] + cursor = self._get_raw_cursor(connection) + c = cursor.execute('select * from SYSPROGRESS.SYS_TBL_CONSTRS') + constraint_name = None + + for row in c.fetchall(): + if "PRIMARY" in row[1]: + pkeys.append(row[4]) + if constraint_name is None: + constraint_name = row[0] + if pkeys: + return { + "constrained_columns": pkeys, + "name": constraint_name, + } + + def get_indexes(self, connection, table_name, schema=None, **kw): # implemented in the database, not terribly useful. From d0d2a230bbc97d3f2302eeb7bf3d3f50861f6e3a Mon Sep 17 00:00:00 2001 From: Justin Cohen Date: Mon, 28 Aug 2023 10:03:10 -0400 Subject: [PATCH 5/9] remove column size argument from coltype definition and add col type names that werent being taken into account into ischema_names dict --- progress_sa/base.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/progress_sa/base.py b/progress_sa/base.py index ce924fd..c5f817f 100644 --- a/progress_sa/base.py +++ b/progress_sa/base.py @@ -96,6 +96,10 @@ 'timestamp_timezone':TIMESTAMP, 'varbinary':VARCHAR, 'varchar':VARCHAR, + 'lvarbinary':VARCHAR, + 'varbina':VARCHAR, + 'lvarchar':VARCHAR, + 'decimal':FLOAT, } class ProgressExecutionContext(default.DefaultExecutionContext): @@ -227,7 +231,7 @@ def get_columns(self, connection, table_name, schema=None, **kw): type_name = column[index['type_name']] column_size = column[index['column_size']] - coltype = ischema_names[type_name](column_size) + coltype = ischema_names[type_name]() columns.append(dict(name=name, type=coltype, nullable=nullable, default=default, From c3186f324f9d119a0878de5bb24a457dbd2d0b93 Mon Sep 17 00:00:00 2001 From: Justin Cohen Date: Mon, 28 Aug 2023 10:07:06 -0400 Subject: [PATCH 6/9] update readme --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index a2752e0..9195ff4 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,9 @@ +# Disclaimer +I am fine tuning this Progress Openedge SQLalchemy dialect for use in a meltano based data infrastructure imporvement at my organization. + + + + # SQLAlchemy dialect for Progress OpenEdge An adaption of [this work](https://github.com/dholth/progress_sa) for Python3 and Progress OpenEdge 11.7 From cb62b17e253121cbfa30a2d921d05b2b92458e29 Mon Sep 17 00:00:00 2001 From: Justin Cohen Date: Mon, 28 Aug 2023 10:08:03 -0400 Subject: [PATCH 7/9] update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9195ff4..bd83408 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Disclaimer -I am fine tuning this Progress Openedge SQLalchemy dialect for use in a meltano based data infrastructure imporvement at my organization. +I am fine tuning this Progress Openedge SQLalchemy dialect for use in a meltano based data infrastructure improvement at my organization. From 7f3b3f1ab7ac93ee1a9a499b34986057cb2bdf7d Mon Sep 17 00:00:00 2001 From: Justin Cohen Date: Fri, 15 Sep 2023 14:52:25 -0400 Subject: [PATCH 8/9] change date type mapping to TIMESTAMP as dates are stored in progress oe as timestamp and must map correctly to postgres --- progress_sa/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/progress_sa/base.py b/progress_sa/base.py index c5f817f..8016de7 100644 --- a/progress_sa/base.py +++ b/progress_sa/base.py @@ -84,7 +84,7 @@ 'bigint':BIGINT, 'bit':BOOLEAN, 'character':CHAR, - 'date':DATE, + 'date':TIMESTAMP, 'float':FLOAT, # double precision. Also FLOAT(8) in SQL. 'integer':INTEGER, 'numeric':NUMERIC, From 8d96d17601e78580b664944ae76d94ac5bf2797a Mon Sep 17 00:00:00 2001 From: Justin Cohen Date: Wed, 18 Oct 2023 10:07:24 -0400 Subject: [PATCH 9/9] change pyodbc file to full in driver name from environment to control for variation in drive name and update readme with instructions to do so --- README.md | 2 ++ progress_sa/pyodbc.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index bd83408..31e1b98 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,8 @@ An adaption of [this work](https://github.com/dholth/progress_sa) for Python3 an $ cd progress_sa $ python setup.py install ``` +## DRIVER_NAME env variable +Due to variation in driver names, create an environment variable called PROGRESS_OE_DRIVER_NAME with your particular driver name and the driver will pull it in and use it. ## Usage diff --git a/progress_sa/pyodbc.py b/progress_sa/pyodbc.py index 5a5ddb0..87c13ec 100644 --- a/progress_sa/pyodbc.py +++ b/progress_sa/pyodbc.py @@ -3,6 +3,7 @@ from sqlalchemy import util from urllib.parse import unquote_plus import sys +import os class ProgressExecutionContext_pyodbc(ProgressExecutionContext): @@ -11,7 +12,7 @@ class ProgressExecutionContext_pyodbc(ProgressExecutionContext): class Progress_pyodbc(PyODBCConnector, ProgressDialect): # pyodbc_driver_name = 'Progress OpenEdge Wire Protocol' - pyodbc_driver_name = 'DataDirect 7.1 Progress OpenEdge Wire Protocol' + pyodbc_driver_name = os.environ.get("PROGRESS_OE_DRIVER_NAME") execution_ctx_cls = ProgressExecutionContext_pyodbc def __init__(self, **kwargs): super(Progress_pyodbc, self).__init__(**kwargs)