From e44d7e7a8ed10d13d3e72ba99f24368f90a81af0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= <16805946+edgarrmondragon@users.noreply.github.com> Date: Tue, 19 Nov 2024 18:05:53 -0600 Subject: [PATCH] refactor: Deprecate `sqlalchemy_url` and `dialect+driver` settings (#482) --- README.md | 54 ++++++++++++++++----------------- meltano.yml | 3 -- target_postgres/driver.py | 3 ++ target_postgres/target.py | 57 ++++++++++++++++------------------- target_postgres/tests/core.py | 8 +++-- 5 files changed, 60 insertions(+), 65 deletions(-) create mode 100644 target_postgres/driver.py diff --git a/README.md b/README.md index b67331e0..48809002 100644 --- a/README.md +++ b/README.md @@ -24,34 +24,32 @@ This target is tested with all actively supported [Python](https://devguide.pyth ## Settings -| Setting | Required | Default | Description | -| :------------------------------ | :------- | :---------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| host | False | None | Hostname for postgres instance. Note if sqlalchemy_url is set this will be ignored. | -| port | False | 5432 | The port on which postgres is awaiting connection. Note if sqlalchemy_url is set this will be ignored. | -| user | False | None | User name used to authenticate. Note if sqlalchemy_url is set this will be ignored. | -| password | False | None | Password used to authenticate. Note if sqlalchemy_url is set this will be ignored. | -| database | False | None | Database name. Note if sqlalchemy_url is set this will be ignored. | -| sqlalchemy_url | False | None | SQLAlchemy connection string. This will override using host, user, password, port, dialect, and all ssl settings. Note that you must escape password special characters properly. See https://docs.sqlalchemy.org/en/20/core/engines.html#escaping-special-characters-such-as-signs-in-passwords | -| dialect+driver | False | postgresql+psycopg | Dialect+driver see https://docs.sqlalchemy.org/en/20/core/engines.html. Generally just leave this alone. Note if sqlalchemy_url is set this will be ignored. | -| default_target_schema | False | melty | Postgres schema to send data to, example: tap-clickup | -| activate_version | False | 1 | If set to false, the tap will ignore activate version messages. If set to true, add_record_metadata must be set to true as well. | -| hard_delete | False | 0 | When activate version is sent from a tap this specefies if we should delete the records that don't match, or mark them with a date in the `_sdc_deleted_at` column. This config option is ignored if `activate_version` is set to false. | -| add_record_metadata | False | 1 | Note that this must be enabled for activate_version to work!This adds _sdc_extracted_at, _sdc_batched_at, and more to every table. See https://sdk.meltano.com/en/latest/implementation/record_metadata.html for more information. | -| interpret_content_encoding | False | 0 | If set to true, the target will interpret the content encoding of the schema to determine how to store the data. Using this option may result in a more efficient storage of the data but may also result in an error if the data is not encoded as expected. | -| ssl_enable | False | 0 | Whether or not to use ssl to verify the server's identity. Use ssl_certificate_authority and ssl_mode for further customization. To use a client certificate to authenticate yourself to the server, use ssl_client_certificate_enable instead. Note if sqlalchemy_url is set this will be ignored. | -| ssl_client_certificate_enable | False | 0 | Whether or not to provide client-side certificates as a method of authentication to the server. Use ssl_client_certificate and ssl_client_private_key for further customization. To use SSL to verify the server's identity, use ssl_enable instead. Note if sqlalchemy_url is set this will be ignored. | -| ssl_mode | False | verify-full | SSL Protection method, see [postgres documentation](https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-PROTECTION) for more information. Must be one of disable, allow, prefer, require, verify-ca, or verify-full. Note if sqlalchemy_url is set this will be ignored. | -| ssl_certificate_authority | False | ~/.postgresql/root.crl | The certificate authority that should be used to verify the server's identity. Can be provided either as the certificate itself (in .env) or as a filepath to the certificate. Note if sqlalchemy_url is set this will be ignored. | -| ssl_client_certificate | False | ~/.postgresql/postgresql.crt | The certificate that should be used to verify your identity to the server. Can be provided either as the certificate itself (in .env) or as a filepath to the certificate. Note if sqlalchemy_url is set this will be ignored. | -| ssl_client_private_key | False | ~/.postgresql/postgresql.key | The private key for the certificate you provided. Can be provided either as the certificate itself (in .env) or as a filepath to the certificate. Note if sqlalchemy_url is set this will be ignored. | -| ssl_storage_directory | False | .secrets | The folder in which to store SSL certificates provided as raw values. When a certificate/key is provided as a raw value instead of as a filepath, it must be written to a file before it can be used. This configuration option determines where that file is created. | -| ssh_tunnel | False | None | SSH Tunnel Configuration, this is a json object | -| ssh_tunnel.enable | False | 0 | Enable an ssh tunnel (also known as bastion host), see the other ssh_tunnel.* properties for more details | -| ssh_tunnel.host | False | None | Host of the bastion host, this is the host we'll connect to via ssh | -| ssh_tunnel.username | False | None | Username to connect to bastion host | -| ssh_tunnel.port | False | 22 | Port to connect to bastion host | -| ssh_tunnel.private_key | False | None | Private Key for authentication to the bastion host | -| ssh_tunnel.private_key_password | False | None | Private Key Password, leave None if no password is set | +| Setting | Required | Default | Description | +| :------------------------------ | :------- | :--------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| host | False | None | Hostname for postgres instance. | +| port | False | 5432 | The port on which postgres is awaiting connections. | +| user | False | None | User name used to authenticate. | +| password | False | None | Password used to authenticate. | +| database | False | None | Database name. | +| default_target_schema | False | melty | Postgres schema to send data to, example: tap-clickup | +| activate_version | False | 1 | If set to false, the tap will ignore activate version messages. If set to true, add_record_metadata must be set to true as well. | +| hard_delete | False | 0 | When activate version is sent from a tap this specefies if we should delete the records that don't match, or mark them with a date in the `_sdc_deleted_at` column. This config option is ignored if `activate_version` is set to false. | +| add_record_metadata | False | 1 | Note that this must be enabled for activate_version to work!This adds _sdc_extracted_at, _sdc_batched_at, and more to every table. See https://sdk.meltano.com/en/latest/implementation/record_metadata.html for more information. | +| interpret_content_encoding | False | 0 | If set to true, the target will interpret the content encoding of the schema to determine how to store the data. Using this option may result in a more efficient storage of the data but may also result in an error if the data is not encoded as expected. | +| ssl_enable | False | 0 | Whether or not to use ssl to verify the server's identity. Use ssl_certificate_authority and ssl_mode for further customization. To use a client certificate to authenticate yourself to the server, use ssl_client_certificate_enable instead. | +| ssl_client_certificate_enable | False | 0 | Whether or not to provide client-side certificates as a method of authentication to the server. Use ssl_client_certificate and ssl_client_private_key for further customization. To use SSL to verify the server's identity, use ssl_enable instead. | +| ssl_mode | False | verify-full | SSL Protection method, see [postgres documentation](https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-PROTECTION) for more information. Must be one of disable, allow, prefer, require, verify-ca, or verify-full. | +| ssl_certificate_authority | False | ~/.postgresql/root.crl | The certificate authority that should be used to verify the server's identity. Can be provided either as the certificate itself (in .env) or as a filepath to the certificate. | +| ssl_client_certificate | False | ~/.postgresql/postgresql.crt | The certificate that should be used to verify your identity to the server. Can be provided either as the certificate itself (in .env) or as a filepath to the certificate. | +| ssl_client_private_key | False | ~/.postgresql/postgresql.key | The private key for the certificate you provided. Can be provided either as the certificate itself (in .env) or as a filepath to the certificate. | +| ssl_storage_directory | False | .secrets | The folder in which to store SSL certificates provided as raw values. When a certificate/key is provided as a raw value instead of as a filepath, it must be written to a file before it can be used. This configuration option determines where that file is created. | +| ssh_tunnel | False | None | SSH Tunnel Configuration, this is a json object | +| ssh_tunnel.enable | False | 0 | Enable an ssh tunnel (also known as bastion host), see the other ssh_tunnel.* properties for more details | +| ssh_tunnel.host | False | None | Host of the bastion host, this is the host we'll connect to via ssh | +| ssh_tunnel.username | False | None | Username to connect to bastion host | +| ssh_tunnel.port | False | 22 | Port to connect to bastion host | +| ssh_tunnel.private_key | False | None | Private Key for authentication to the bastion host | +| ssh_tunnel.private_key_password | False | None | Private Key Password, leave None if no password is set | A full list of supported settings and capabilities is available by running: `target-postgres --about` diff --git a/meltano.yml b/meltano.yml index 64fa82a0..2c31ed81 100644 --- a/meltano.yml +++ b/meltano.yml @@ -35,9 +35,6 @@ plugins: namespace: target_postgres pip_url: -e . settings: - - name: sqlalchemy_url - kind: password - sensitive: true - name: ssl_enable kind: boolean sensitive: true diff --git a/target_postgres/driver.py b/target_postgres/driver.py new file mode 100644 index 00000000..b8df29b7 --- /dev/null +++ b/target_postgres/driver.py @@ -0,0 +1,3 @@ +"""This module contains the Postgres driver name for SQLAlchemy.""" + +PSYCOPG3 = "postgresql+psycopg" diff --git a/target_postgres/target.py b/target_postgres/target.py index 9fd08915..d2830da5 100644 --- a/target_postgres/target.py +++ b/target_postgres/target.py @@ -7,6 +7,7 @@ from singer_sdk import typing as th from singer_sdk.target_base import SQLTarget +from target_postgres.driver import PSYCOPG3 from target_postgres.sinks import PostgresSink if t.TYPE_CHECKING: @@ -40,6 +41,22 @@ def __init__( parse_env_config=parse_env_config, validate_config=validate_config, ) + # sqlalchemy_url and dialect+driver are now deprecated in favor of the + # individual host, port, user, password, and dialect+driver fields. + if self.config.get("sqlalchemy_url"): + self.logger.warning( + "The `sqlalchemy_url` configuration option is deprecated. " + "Please use the `host`, `port`, `user`, `password` " + "configuration options instead." + ) + + if (driver := self.config.get("dialect+driver")) and driver != PSYCOPG3: + self.logger.warning( + "The `dialect+driver` configuration option is deprecated. " + f"Please set it to `{PSYCOPG3}`, as this will be the hard-coded " + "value in the future." + ) + # There's a few ways to do this in JSON Schema but it is schema draft dependent. # https://stackoverflow.com/questions/38717933/jsonschema-attribute-conditionally-required # noqa: E501 assert (self.config.get("sqlalchemy_url") is not None) or ( @@ -98,49 +115,34 @@ def __init__( th.Property( "host", th.StringType, - description=( - "Hostname for postgres instance. " - + "Note if sqlalchemy_url is set this will be ignored." - ), + description="Hostname for postgres instance.", ), th.Property( "port", th.IntegerType, default=5432, - description=( - "The port on which postgres is awaiting connection. " - + "Note if sqlalchemy_url is set this will be ignored." - ), + description="The port on which postgres is awaiting connections.", ), th.Property( "user", th.StringType, - description=( - "User name used to authenticate. " - + "Note if sqlalchemy_url is set this will be ignored." - ), + description="User name used to authenticate.", ), th.Property( "password", th.StringType, - description=( - "Password used to authenticate. " - "Note if sqlalchemy_url is set this will be ignored." - ), + description="Password used to authenticate.", ), th.Property( "database", th.StringType, - description=( - "Database name. " - + "Note if sqlalchemy_url is set this will be ignored." - ), + description="Database name.", ), th.Property( "sqlalchemy_url", th.StringType, description=( - "SQLAlchemy connection string. " + "DEPRECATED. SQLAlchemy connection string. " + "This will override using host, user, password, port, " + "dialect, and all ssl settings. Note that you must escape password " + "special characters properly. See " @@ -150,12 +152,11 @@ def __init__( th.Property( "dialect+driver", th.StringType, - default="postgresql+psycopg", + default=PSYCOPG3, description=( - "Dialect+driver see " + "DEPRECATED. Dialect+driver see " + "https://docs.sqlalchemy.org/en/20/core/engines.html. " - + "Generally just leave this alone. " - + "Note if sqlalchemy_url is set this will be ignored." + + "Generally just leave this alone." ), ), th.Property( @@ -215,7 +216,6 @@ def __init__( + " ssl_certificate_authority and ssl_mode for further customization." + " To use a client certificate to authenticate yourself to the server," + " use ssl_client_certificate_enable instead." - + " Note if sqlalchemy_url is set this will be ignored." ), ), th.Property( @@ -227,7 +227,6 @@ def __init__( + " authentication to the server. Use ssl_client_certificate and" + " ssl_client_private_key for further customization. To use SSL to" + " verify the server's identity, use ssl_enable instead." - + " Note if sqlalchemy_url is set this will be ignored." ), ), th.Property( @@ -238,7 +237,6 @@ def __init__( "SSL Protection method, see [postgres documentation](https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-PROTECTION)" + " for more information. Must be one of disable, allow, prefer," + " require, verify-ca, or verify-full." - + " Note if sqlalchemy_url is set this will be ignored." ), ), th.Property( @@ -249,7 +247,6 @@ def __init__( "The certificate authority that should be used to verify the server's" + " identity. Can be provided either as the certificate itself (in" + " .env) or as a filepath to the certificate." - + " Note if sqlalchemy_url is set this will be ignored." ), ), th.Property( @@ -260,7 +257,6 @@ def __init__( "The certificate that should be used to verify your identity to the" + " server. Can be provided either as the certificate itself (in .env)" + " or as a filepath to the certificate." - + " Note if sqlalchemy_url is set this will be ignored." ), ), th.Property( @@ -271,7 +267,6 @@ def __init__( "The private key for the certificate you provided. Can be provided" + " either as the certificate itself (in .env) or as a filepath to the" + " certificate." - + " Note if sqlalchemy_url is set this will be ignored." ), ), th.Property( diff --git a/target_postgres/tests/core.py b/target_postgres/tests/core.py index 613625c7..f426b55c 100644 --- a/target_postgres/tests/core.py +++ b/target_postgres/tests/core.py @@ -8,7 +8,6 @@ def postgres_config(): return { - "dialect+driver": "postgresql+psycopg", "host": "localhost", "user": "postgres", "password": "postgres", @@ -29,7 +28,6 @@ def postgres_config(): def postgres_config_no_ssl(): return { - "dialect+driver": "postgresql+psycopg", "host": "localhost", "user": "postgres", "password": "postgres", @@ -43,7 +41,11 @@ def postgres_config_no_ssl(): def postgres_config_ssh_tunnel(): return { - "sqlalchemy_url": "postgresql://postgres:postgres@10.5.0.5:5432/main", + "host": "10.5.0.5", + "user": "postgres", + "password": "postgres", + "database": "main", + "port": 5432, "ssh_tunnel": { "enable": True, "host": "127.0.0.1",