From 3a9e5c0067501979d9712070ca9b6195ffa23bff Mon Sep 17 00:00:00 2001 From: Alejandro Rodriguez Date: Fri, 28 Oct 2022 15:12:11 -0400 Subject: [PATCH] Remove Redshift prod-dw references, bump version, and make generate --- README.md | 2 +- config/config.go | 18 +---------- config/config_validate.go | 1 - controller/controller.go | 9 ------ db/postgres.go | 13 -------- gen-go/client/client.go | 2 +- gen-go/models/analytics_database.go | 4 +-- gen-go/models/analytics_latency_configs.go | 36 --------------------- gen-js/index.d.ts | 2 +- gen-js/index.js | 2 +- gen-js/package.json | 2 +- helpers/helpers.go | 2 -- launch/analytics-latency-config-service.yml | 5 --- swagger.yml | 8 +---- 14 files changed, 8 insertions(+), 98 deletions(-) diff --git a/README.md b/README.md index 2cf7859..0d9ca7d 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Defining checks in analytics-latency-config-service can be accomplished by addin ] ``` -`analytics-latency-config-service` then reads from this config to surface latency info to other workers/services. `schema` + `table` identifies the table, and `latency.timestamp_column` identifies the time a row enters Redshift. `latency.thresholds` configures the different tiers of latency thresholds maximum amount of latency acceptable for the table's data in [Go time format](https://golang.org/pkg/time/#ParseDuration). +`analytics-latency-config-service` then reads from this config to surface latency info to other workers/services. `schema` + `table` identifies the table, and `latency.timestamp_column` identifies the time a row enters the source database. `latency.thresholds` configures the different tiers of latency thresholds maximum amount of latency acceptable for the table's data in [Go time format](https://golang.org/pkg/time/#ParseDuration). For tables that are not explicitly declared in the config, `default_threshold` and `default_timestamp_column` will be used as substitutes for the above values. diff --git a/config/config.go b/config/config.go index 2f9b5c3..67ba630 100644 --- a/config/config.go +++ b/config/config.go @@ -11,17 +11,7 @@ import ( ) var ( - // We have two redshift databases: - // One that holds all the data and views (prod) - // And one that holds timeline (fast-prod) - // RedshiftProd* are for the former - RedshiftProdHost string - RedshiftProdPort string - RedshiftProdDatabase string - RedshiftProdUsername string - RedshiftProdPassword string - - // RedshiftFast* are for the latter + // RedshiftFast* holds timeline (fast-prod) RedshiftFastHost string RedshiftFastPort string RedshiftFastDatabase string @@ -79,12 +69,6 @@ func Init() { // InitDBs reads environment DB variables and initializes the config. func InitDBs() { - RedshiftProdHost = requiredEnv("REDSHIFT_PROD_HOST") - RedshiftProdPort = requiredEnv("REDSHIFT_PROD_PORT") - RedshiftProdDatabase = requiredEnv("REDSHIFT_PROD_DATABASE") - RedshiftProdUsername = requiredEnv("REDSHIFT_PROD_USER") - RedshiftProdPassword = requiredEnv("REDSHIFT_PROD_PASSWORD") - RedshiftFastHost = requiredEnv("REDSHIFT_FAST_HOST") RedshiftFastPort = requiredEnv("REDSHIFT_FAST_PORT") RedshiftFastDatabase = requiredEnv("REDSHIFT_FAST_DATABASE") diff --git a/config/config_validate.go b/config/config_validate.go index 42004a7..fc9e7ba 100644 --- a/config/config_validate.go +++ b/config/config_validate.go @@ -10,7 +10,6 @@ import ( func validateLatencyConfig(configs models.AnalyticsLatencyConfigs) { dbs := []models.AnalyticsDatabase{ - models.AnalyticsDatabaseRedshiftProd, models.AnalyticsDatabaseRedshiftFast, models.AnalyticsDatabaseRdsInternal, models.AnalyticsDatabaseRdsExternal, diff --git a/controller/controller.go b/controller/controller.go index 03c4b0b..74b7eeb 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -14,7 +14,6 @@ import ( // Controller implements server.Controller type Controller struct { - redshiftProdConnection db.DBClient redshiftFastConnection db.DBClient rdsExternalConnection db.DBClient rdsInternalConnection db.DBClient @@ -24,8 +23,6 @@ type Controller struct { func (c *Controller) getDatabaseConnection(database models.AnalyticsDatabase) (db.DBClient, error) { switch database { - case models.AnalyticsDatabaseRedshiftProd: - return c.redshiftProdConnection, nil case models.AnalyticsDatabaseRedshiftFast: return c.redshiftFastConnection, nil case models.AnalyticsDatabaseRdsInternal: @@ -41,11 +38,6 @@ func (c *Controller) getDatabaseConnection(database models.AnalyticsDatabase) (d func New() (*Controller, error) { var mErrors *multierror.Error - redshiftProdConnection, err := db.NewRedshiftProdClient() - if err != nil { - mErrors = multierror.Append(mErrors, fmt.Errorf("redshift-prod-failed-init: %s", err.Error())) - } - redshiftFastConnection, err := db.NewRedshiftFastClient() if err != nil { mErrors = multierror.Append(mErrors, fmt.Errorf("redshift-fast-failed-init: %s", err.Error())) @@ -73,7 +65,6 @@ func New() (*Controller, error) { } return &Controller{ - redshiftProdConnection: redshiftProdConnection, redshiftFastConnection: redshiftFastConnection, rdsExternalConnection: rdsExternalConnection, rdsInternalConnection: rdsInternalConnection, diff --git a/db/postgres.go b/db/postgres.go index ff159bb..fd87528 100644 --- a/db/postgres.go +++ b/db/postgres.go @@ -51,19 +51,6 @@ func NewPostgresClient(info PostgresCredentials, clusterName string) (*Postgres, return &Postgres{session, clusterName}, nil } -// NewRedshiftProdClient initializes a client to fresh prod -func NewRedshiftProdClient() (*Postgres, error) { - info := PostgresCredentials{ - Host: config.RedshiftProdHost, - Port: config.RedshiftProdPort, - Username: config.RedshiftProdUsername, - Password: config.RedshiftProdPassword, - Database: config.RedshiftProdDatabase, - } - - return NewPostgresClient(info, "redshift-prod") -} - // NewRedshiftFastClient initializes a client to fast prod func NewRedshiftFastClient() (*Postgres, error) { info := PostgresCredentials{ diff --git a/gen-go/client/client.go b/gen-go/client/client.go index 0023460..791827d 100644 --- a/gen-go/client/client.go +++ b/gen-go/client/client.go @@ -23,7 +23,7 @@ var _ = strconv.FormatInt var _ = bytes.Compare // Version of the client. -const Version = "0.3.0" +const Version = "0.4.0" // VersionHeader is sent with every request. const VersionHeader = "X-Client-Version" diff --git a/gen-go/models/analytics_database.go b/gen-go/models/analytics_database.go index 8704a37..06669f8 100644 --- a/gen-go/models/analytics_database.go +++ b/gen-go/models/analytics_database.go @@ -19,8 +19,6 @@ import ( type AnalyticsDatabase string const ( - // AnalyticsDatabaseRedshiftProd captures enum value "RedshiftProd" - AnalyticsDatabaseRedshiftProd AnalyticsDatabase = "RedshiftProd" // AnalyticsDatabaseRedshiftFast captures enum value "RedshiftFast" AnalyticsDatabaseRedshiftFast AnalyticsDatabase = "RedshiftFast" // AnalyticsDatabaseRdsInternal captures enum value "RdsInternal" @@ -36,7 +34,7 @@ var analyticsDatabaseEnum []interface{} func init() { var res []AnalyticsDatabase - if err := json.Unmarshal([]byte(`["RedshiftProd","RedshiftFast","RdsInternal","RdsExternal","Snowflake"]`), &res); err != nil { + if err := json.Unmarshal([]byte(`["RedshiftFast","RdsInternal","RdsExternal","Snowflake"]`), &res); err != nil { panic(err) } for _, v := range res { diff --git a/gen-go/models/analytics_latency_configs.go b/gen-go/models/analytics_latency_configs.go index f625d67..299ba2f 100644 --- a/gen-go/models/analytics_latency_configs.go +++ b/gen-go/models/analytics_latency_configs.go @@ -31,10 +31,6 @@ type AnalyticsLatencyConfigs struct { // Required: true RedshiftFast []*SchemaConfig `json:"redshiftFast"` - // redshift prod - // Required: true - RedshiftProd []*SchemaConfig `json:"redshiftProd"` - // snowflake // Required: true Snowflake []*SchemaConfig `json:"snowflake"` @@ -59,11 +55,6 @@ func (m *AnalyticsLatencyConfigs) Validate(formats strfmt.Registry) error { res = append(res, err) } - if err := m.validateRedshiftProd(formats); err != nil { - // prop - res = append(res, err) - } - if err := m.validateSnowflake(formats); err != nil { // prop res = append(res, err) @@ -156,33 +147,6 @@ func (m *AnalyticsLatencyConfigs) validateRedshiftFast(formats strfmt.Registry) return nil } -func (m *AnalyticsLatencyConfigs) validateRedshiftProd(formats strfmt.Registry) error { - - if err := validate.Required("redshiftProd", "body", m.RedshiftProd); err != nil { - return err - } - - for i := 0; i < len(m.RedshiftProd); i++ { - - if swag.IsZero(m.RedshiftProd[i]) { // not required - continue - } - - if m.RedshiftProd[i] != nil { - - if err := m.RedshiftProd[i].Validate(formats); err != nil { - if ve, ok := err.(*errors.Validation); ok { - return ve.ValidateName("redshiftProd" + "." + strconv.Itoa(i)) - } - return err - } - } - - } - - return nil -} - func (m *AnalyticsLatencyConfigs) validateSnowflake(formats strfmt.Registry) error { if err := validate.Required("snowflake", "body", m.Snowflake); err != nil { diff --git a/gen-js/index.d.ts b/gen-js/index.d.ts index cc3d1a0..3042891 100644 --- a/gen-js/index.d.ts +++ b/gen-js/index.d.ts @@ -107,7 +107,7 @@ declare namespace AnalyticsLatencyConfigService { namespace Models { - type AnalyticsDatabase = ("RedshiftProd" | "RedshiftFast" | "RdsInternal" | "RdsExternal" | "Snowflake"); + type AnalyticsDatabase = ("RedshiftFast" | "RdsInternal" | "RdsExternal" | "Snowflake"); type AnalyticsLatencyConfigs = any; diff --git a/gen-js/index.js b/gen-js/index.js index 8d8849a..44c56ee 100644 --- a/gen-js/index.js +++ b/gen-js/index.js @@ -619,7 +619,7 @@ module.exports.Errors = Errors; module.exports.DefaultCircuitOptions = defaultCircuitOptions; -const version = "0.3.0"; +const version = "0.4.0"; const versionHeader = "X-Client-Version"; module.exports.Version = version; module.exports.VersionHeader = versionHeader; diff --git a/gen-js/package.json b/gen-js/package.json index 1a5e145..513768f 100644 --- a/gen-js/package.json +++ b/gen-js/package.json @@ -1,6 +1,6 @@ { "name": "@clever/analytics-latency-config-service", - "version": "0.3.0", + "version": "0.4.0", "description": "Service for surfacing latency config settings for the analytics pipeline.", "main": "index.js", "dependencies": { diff --git a/helpers/helpers.go b/helpers/helpers.go index 5cf8080..c85dce2 100644 --- a/helpers/helpers.go +++ b/helpers/helpers.go @@ -76,8 +76,6 @@ func GetThresholdTierErrorValue(tier models.ThresholdTier) (int, error) { // GetDatabaseConfig returns the database-specific config func GetDatabaseConfig(configs models.AnalyticsLatencyConfigs, database models.AnalyticsDatabase) ([]*models.SchemaConfig, error) { switch database { - case models.AnalyticsDatabaseRedshiftProd: - return configs.RedshiftProd, nil case models.AnalyticsDatabaseRedshiftFast: return configs.RedshiftFast, nil case models.AnalyticsDatabaseRdsInternal: diff --git a/launch/analytics-latency-config-service.yml b/launch/analytics-latency-config-service.yml index ac70a2f..06b6a21 100644 --- a/launch/analytics-latency-config-service.yml +++ b/launch/analytics-latency-config-service.yml @@ -1,11 +1,6 @@ env: - TRACING_ACCESS_TOKEN - TRACING_INGEST_URL -- REDSHIFT_PROD_DATABASE -- REDSHIFT_PROD_USER -- REDSHIFT_PROD_PASSWORD -- REDSHIFT_PROD_HOST -- REDSHIFT_PROD_PORT - REDSHIFT_FAST_DATABASE - REDSHIFT_FAST_USER - REDSHIFT_FAST_PASSWORD diff --git a/swagger.yml b/swagger.yml index 1b094bd..799e468 100644 --- a/swagger.yml +++ b/swagger.yml @@ -4,7 +4,7 @@ info: description: Service for surfacing latency config settings for the analytics pipeline. # when changing the version here, make sure to run # `make generate` to generate clients and server - version: 0.3.0 + version: 0.4.0 x-npm-package: '@clever/analytics-latency-config-service' schemes: - http @@ -185,16 +185,11 @@ definitions: AnalyticsLatencyConfigs: required: - - redshiftProd - redshiftFast - rdsInternal - rdsExternal - snowflake properties: - redshiftProd: - type: array - items: - $ref: "#/definitions/SchemaConfig" redshiftFast: type: array items: @@ -216,7 +211,6 @@ definitions: type: string description: Analytics databases, in config enum: - - "RedshiftProd" - "RedshiftFast" - "RdsInternal" - "RdsExternal"