diff --git a/.github/workflows/rill-cloud.yml b/.github/workflows/rill-cloud.yml index a29b8f473af..781a8fca29e 100644 --- a/.github/workflows/rill-cloud.yml +++ b/.github/workflows/rill-cloud.yml @@ -13,6 +13,7 @@ on: env: RELEASE: ${{ startsWith(github.ref, 'refs/tags/v') }} + DEPLOY_CLOUD: 1 jobs: release: @@ -39,10 +40,15 @@ jobs: - name: Build & Publish Rill docker image run: |- + if [ "$RELEASE" == "true" ] || [ "$GITHUB_REF_NAME" == "main" ]; then + echo "DEPLOY_CLOUD=1" >> $GITHUB_ENV + else + echo "DEPLOY_CLOUD=$(git branch -r | grep release-0 | sort | tail -1 | grep -c $GITHUB_REF_NAME)" >> $GITHUB_ENV + fi + if [ ${RELEASE} == "false" ]; then echo "Fetch tags to get the last tagged version" - git fetch --all --tags; - echo "LATEST_BRANCH=$(git branch -r | grep release-0 | sort | tail -1 | grep -c ${GITHUB_REF_NAME})" >> $GITHUB_ENV + git fetch --all --tags; fi # Embed DuckDB extensions in the Rill binary @@ -64,7 +70,7 @@ jobs: fi - name: Trigger Rill Cloud deployment - if: env.LATEST_BRANCH == '1' + if: env.DEPLOY_CLOUD == '1' run: |- set -e curl -X POST https://api.github.com/repos/rilldata/rill-helm-charts/dispatches \ diff --git a/admin/provisioner/clickhousestatic/provisioner.go b/admin/provisioner/clickhousestatic/provisioner.go index 1432e3c8649..0a8b1fe3508 100644 --- a/admin/provisioner/clickhousestatic/provisioner.go +++ b/admin/provisioner/clickhousestatic/provisioner.go @@ -102,12 +102,20 @@ func (p *Provisioner) Provision(ctx context.Context, r *provisioner.Resource, op return nil, fmt.Errorf("failed to create clickhouse database: %w", err) } - // Idempotently create the user + // Idempotently create the user. _, err = p.ch.ExecContext(ctx, fmt.Sprintf("CREATE USER IF NOT EXISTS %s IDENTIFIED WITH sha256_password BY ? DEFAULT DATABASE %s GRANTEES NONE", user, dbName), password) if err != nil { return nil, fmt.Errorf("failed to create clickhouse user: %w", err) } + // When creating the user, the password assignment is not idempotent (if there are two concurrent invocations, we don't know which password was used). + // By adding the password separately, we ensure all passwords will work. + // NOTE: Requires ClickHouse 24.9 or later. + _, err = p.ch.ExecContext(ctx, fmt.Sprintf("ALTER USER %s ADD IDENTIFIED WITH sha256_password BY ?", user), password) + if err != nil { + return nil, fmt.Errorf("failed to add password for clickhouse user: %w", err) + } + // Grant privileges on the database to the user _, err = p.ch.ExecContext(ctx, fmt.Sprintf(` GRANT diff --git a/admin/provisioner/clickhousestatic/provisioner_test.go b/admin/provisioner/clickhousestatic/provisioner_test.go index da86898a47d..69aaa5a9cc7 100644 --- a/admin/provisioner/clickhousestatic/provisioner_test.go +++ b/admin/provisioner/clickhousestatic/provisioner_test.go @@ -17,11 +17,11 @@ import ( "go.uber.org/zap" ) -func Test(t *testing.T) { +func TestClickHouseStatic(t *testing.T) { // Create a test ClickHouse cluster container, err := testcontainersclickhouse.Run( context.Background(), - "clickhouse/clickhouse-server:24.6.2.17", + "clickhouse/clickhouse-server:24.11.1.2557", // Add a user config file that enables access management for the "default" user testcontainers.CustomizeRequestOption(func(req *testcontainers.GenericContainerRequest) error { req.Files = append(req.Files, testcontainers.ContainerFile{ diff --git a/cli/pkg/local/app.go b/cli/pkg/local/app.go index 93df78ad80b..071ae89fbae 100644 --- a/cli/pkg/local/app.go +++ b/cli/pkg/local/app.go @@ -8,7 +8,6 @@ import ( "net/http" "os" "path/filepath" - "strconv" "time" "github.com/c2h5oh/datasize" @@ -197,26 +196,13 @@ func NewApp(ctx context.Context, opts *AppOptions) (*App, error) { } } - // If the OLAP is the default OLAP (DuckDB in stage.db), we make it relative to the project directory (not the working directory) - defaultOLAP := false olapCfg := make(map[string]string) - if opts.OlapDriver == DefaultOLAPDriver && opts.OlapDSN == DefaultOLAPDSN { - defaultOLAP = true - val, err := isExternalStorageEnabled(vars) - if err != nil { - return nil, err - } - olapCfg["external_table_storage"] = strconv.FormatBool(val) - } - if opts.OlapDriver == "duckdb" { + if opts.OlapDSN != DefaultOLAPDSN { + return nil, fmt.Errorf("setting DSN for DuckDB is not supported") + } // Set default DuckDB pool size to 4 olapCfg["pool_size"] = "4" - if !defaultOLAP { - // dsn is automatically computed by duckdb driver so we set only when non default dsn is passed - olapCfg["dsn"] = opts.OlapDSN - olapCfg["error_on_incompatible_version"] = "true" - } } // Add OLAP connector @@ -609,14 +595,3 @@ func (s skipFieldZapEncoder) AddString(key, val string) { s.Encoder.AddString(key, val) } } - -// isExternalStorageEnabled determines if external storage can be enabled. -func isExternalStorageEnabled(variables map[string]string) (bool, error) { - // check if flag explicitly passed - val, ok := variables["connector.duckdb.external_table_storage"] - if !ok { - // mark enabled by default - return true, nil - } - return strconv.ParseBool(val) -} diff --git a/docs/docs/build/credentials/credentials.md b/docs/docs/build/credentials/credentials.md index 39e13e414e6..530435e9c35 100644 --- a/docs/docs/build/credentials/credentials.md +++ b/docs/docs/build/credentials/credentials.md @@ -8,7 +8,7 @@ Rill requires credentials to connect to remote data sources such as private buck At a high level, configuring credentials and credentials management in Rill can be broken down into three categories: - Setting credentials for Rill Developer -- Setting credentials for a Rill Cloud project +- [Setting credentials for a Rill Cloud project](/manage/variables-and-credentials) - Pushing and pulling credentials to / from Rill Cloud ## Setting credentials for Rill Developer @@ -28,37 +28,38 @@ If you plan to deploy a project (to Rill Cloud), it is not recommended to pass i ::: -## Setting credentials for a Rill Cloud project +## Variables -When deploying a project, Rill Cloud will need appropriate credentials passed to it that can be used to read from a source (or to be used to establish a connection with the OLAP engine). This can be done either by: -1. Using [rill env configure](../../reference/cli/env/configure.md) to pass in connector level credentials (make sure to run this CLI command from the root directory of your Rill project) -2. Specifying all corresponding source level and OLAP engine credentials in the project's `.env` file and "pushing it" to Rill Cloud ([see below](#pushing-and-pulling-credentials-to--from-rill-cloud)) +Project variables work exactly the same way as credentials and can be defined when starting rill via `--env key=value` or set in the .env file in the project directory. -For more details, please refer to the corresponding [connector](../../reference/connectors/connectors.md) or [OLAP engine](../../reference/olap-engines/olap-engines.md) page. +```bash +variable=xyz +``` -:::tip Best practice +This variable will then be usable and referenceable for [templating](../../deploy/templating.md) purposes in the local instance of your project. -For projects that you deploy to production, it is recommended to use service accounts when possible instead of individual user credentials. +:::info Fun Fact +Connector credentials are essentially a form of project variable, prefixed using the `connector..` syntax. For example, `connector.druid.dsn` and `connector.clickhouse.dsn` are both hardcoded project variables (that happen to correspond to the [Druid](/reference/olap-engines/druid.md) and [ClickHouse](/reference/olap-engines/clickhouse.md) OLAP engines respectively). ::: -## Pushing and pulling credentials to / from Rill Cloud +:::tip Avoid committing sensitive information to Git -If you have a project deployed to Rill Cloud, Rill provides the ability to **sync** the credentials between your local instance of Rill Developer and Rill cloud. This provides the ability to quickly reuse existing credentials, if configured, instead of having to manually input credentials each time. This can be accomplished by leveraging the `rill env push` and `rill env pull` CLI commands respectively. +It's never a good idea to commit sensitive information to Git and goes against security best practices. Similar to credentials, if there are sensitive variables that you don't want to commit publicly to your `rill.yaml` configuration file (and thus potentially accessible by others), it's recommended to set them in your `.env` file directly and/or use `rill env set` via the CLI (and then optionally push / pull them as necessary). -### rill env push +::: -As a project admin, you can either use `rill env configure` after deploying a project or `rill env push` to specify a particular set of credentials that your Rill Cloud project will use. If choosing the latter, you can update your *`/.env`* file with the appropriate variables and credentials that are required. Alternatively, if this file has already been updated, you can run `rill env push` from your project's root directory. -- Rill Cloud will use the specified credentials and variables in this `.env` file for the deployed project. -- Other users will also be able to use `rill env pull` to retrieve these defined credentials for local use (with Rill Developer). +## Deploying to Rill Cloud -:::warning Overriding Cloud credentials +:::tip Ready to Deploy? +Please see our [deploy credentials page](/deploy/deploy-credentials#configure-environmental-variables-and-credentials-on-rill-cloud) to configure your credentials on Rill Cloud. +::: -If a credential and/or variable has already been configured in Rill Cloud, Rill will warn you about overriding if you attempt to push a new value in your `.env` file. This is because overriding credentials can impact your deployed project and/or other users (if they pull these credentials locally). -![Pushing credentials that already exist to Rill Cloud](/img/build/credentials/rill-env-push.png) -::: +## Pulling Credentials and Variables from a Deployed Project on Rill Cloud + +If you are making changes to an already deployed instance from Rill Cloud, it is possible to **sync** the credentials and variables from the Rill Cloud to your local instance of Rill Developer. ### rill env pull @@ -72,26 +73,17 @@ Please note when you run `rill env pull`, Rill will *automatically override any ::: -### variables - -Project variables work exactly the same way as credentials and can be pushed to Cloud / pulled locally using the same `rill env push` and `rill env pull` commands specifically. To do this, if you're in Rill Developer and want to set a variable through your `/.env` file (and save): -```bash -variable=xyz -``` - -This variable should then be usable and referenceable for [templating](../../deploy/templating.md) purposes in the local instance of your project. Then, if you want to push these variable definitions to your deployed Cloud project, you can use `rill env push`. Similarly, if these variables had already been set in Rill Cloud for your project, you can use `rill env pull` to clone these variables locally (in your `.env` file). - -:::info Fun Fact - -Connector credentials are essentially a form of project variable, prefixed using the `connector..` syntax. For example, `connector.druid.dsn` and `connector.clickhouse.dsn` are both hardcoded project variables (that happen to correspond to the [Druid](/reference/olap-engines/druid.md) and [ClickHouse](/reference/olap-engines/clickhouse.md) OLAP engines respectively). +### rill env push -::: +As a project admin, you can either use `rill env configure` after deploying a project or `rill env push` to specify a particular set of credentials that your Rill Cloud project will use. If choosing the latter, you can update your *`/.env`* file with the appropriate variables and credentials that are required. Alternatively, if this file has already been updated, you can run `rill env push` from your project's root directory. +- Rill Cloud will use the specified credentials and variables in this `.env` file for the deployed project. +- Other users will also be able to use `rill env pull` to retrieve these defined credentials for local use (with Rill Developer). -:::tip Avoid committing sensitive information to Git +:::warning Overriding Cloud credentials -It's never a good idea to commit sensitive information to Git and goes against security best practices. Similar to credentials, if there are sensitive variables that you don't want to commit publicly to your `rill.yaml` configuration file (and thus potentially accessible by others), it's recommended to set them in your `.env` file directly and/or use `rill env set` via the CLI (and then optionally push / pull them as necessary). +If a credential and/or variable has already been configured in Rill Cloud, Rill will warn you about overriding if you attempt to push a new value in your `.env` file. This is because overriding credentials can impact your deployed project and/or other users (if they pull these credentials locally). +![Pushing credentials that already exist to Rill Cloud](/img/build/credentials/rill-env-push.png) ::: - diff --git a/docs/docs/build/dashboards/customize.md b/docs/docs/build/dashboards/customize.md index c43af1b66f1..dfa43804612 100644 --- a/docs/docs/build/dashboards/customize.md +++ b/docs/docs/build/dashboards/customize.md @@ -16,36 +16,35 @@ For a full list of available dashboard properties and configurations, please see ::: -**`time_ranges`** +**`time_ranges:`** One of the more important configurations, available time ranges allow you to change the defaults in the time dropdown for periods to select. Updating this list allows users to quickly change between the most common analyses like day over day, recent weeks, or period to date. The range must be a valid [ISO 8601 duration](https://en.wikipedia.org/wiki/ISO_8601#Durations) or one of the [Rill ISO 8601 extensions](../../reference/rill-iso-extensions.md#extensions). ```yaml - available_time_ranges: - - PT15M - - PT1H - - P7D - - P4W - - rill-TD ## Today - - rill-WTD ## Week-To-date +- PT15M +- PT1H +- P7D +- P4W +- rill-TD ## Today +- rill-WTD ## Week-To-date ``` -**`time_zones`** +**`time_zones:`** Rill will automatically select several time zones that should be pinned to the top of the time zone selector. It should be a list of [IANA time zone identifiers](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones). You can add or remove from this list for the relevant time zones for your team. ```yaml - - America/Los_Angeles - - America/Chicago - - America/New_York - - Europe/London - - Europe/Paris - - Asia/Jerusalem - - Europe/Moscow - - Asia/Kolkata - - Asia/Shanghai - - Asia/Tokyo - - Australia/Sydney +- America/Los_Angeles +- America/Chicago +- America/New_York +- Europe/London +- Europe/Paris +- Asia/Jerusalem +- Europe/Moscow +- Asia/Kolkata +- Asia/Shanghai +- Asia/Tokyo +- Australia/Sydney ``` ### Setting Default Views for Dashboards @@ -122,8 +121,8 @@ title: Title of your explore dashboard description: a description metrics_view: -dimensions: '*' #can use regex -measures: '*' #can use regex +dimensions: '*' #can use expressions +measures: '*' #can use expressions theme: #your default theme @@ -132,10 +131,7 @@ time_zones: #was available_time_zones defaults: #define all the defaults within here dimensions: - measures: - time_range: - comparison_mode: - comparison_dimension: + ... security: access: #only access can be set on dashboard level, see metric view for detailed access policies diff --git a/docs/docs/build/dashboards/dashboards.md b/docs/docs/build/dashboards/dashboards.md index d8b87ec79f0..afcff4abff1 100644 --- a/docs/docs/build/dashboards/dashboards.md +++ b/docs/docs/build/dashboards/dashboards.md @@ -4,16 +4,18 @@ description: Create dashboards using source data and models with time, dimension sidebar_label: Create Explore Dashboards sidebar_position: 00 --- + + +
+ + +In Rill, explore dashboards are used to visually understand your data with real-time filtering based on your defined dimensions and measures in your metrics view. In the explore dashboard YAML, you can define which measures and dimensions are visible as well as define the default view when a user sees your dashboard. :::tip Starting in version 0.50, metrics views has been separated from explore dashboards. This allows for a cleaner, more accessible metrics layer and the ability to build various dashboards and components on top of a single metrics view. For more information on what a metrics view is please see: [What is a Metrics View?](/concepts/metrics-layer) For migration steps, see [Migrations](/latest-changes/v50-dashboard-changes#how-to-migrate-your-current-dashboards). ::: -In Rill, explore dashboards are used to visually understand your data with real-time filtering based on your defined dimensions and measures in your metrics view. In the explore dashboard YAML, you can define which measures and dimensions are visible as well as define the default view when a user sees your dashboard. - -![img](/img/build/dashboard/explore-dashboard.png) - * _**metrics_view**_ - A metrics view that powers the dashboard * _**measures**_ - `*` Which measures to include or exclude from the metrics view, using a wildcard will include all. * _**dimensions**_ - `*` Which dimensions to include or exclude from the metrics view, using a wildcard will include all. @@ -29,11 +31,8 @@ title: Title of your Explore Dashboard description: a description for your explore dashboard metrics_view: my_metricsview -dimensions: '*' #can use regex -measures: '*' #can use regex - -time_ranges: #was available_time_ranges, list the time of available time ranges that can be selected in your dashboard -time_zones: #was available_time_zones, list the time zones that are selectable in the dashboard +dimensions: '*' #can use expressions +measures: '*' #can use expressions defaults: #define all the defaults within here, was default_* in previous dashboard YAML dimensions: @@ -44,8 +43,6 @@ security: ``` - - :::note Dashboard Properties For more details about available configurations and properties, check our [Dashboard YAML](/reference/project-files/explore-dashboards) reference page. ::: diff --git a/docs/docs/build/structure/structure.md b/docs/docs/build/structure/structure.md index 3706fbd56a4..0d7bff80af1 100644 --- a/docs/docs/build/structure/structure.md +++ b/docs/docs/build/structure/structure.md @@ -26,6 +26,12 @@ However, if you'd like to create a resource outside one of these native folders, ::: +## Navigating Upstream / Downstream Objects + + +
+When selecting between a source, model, metrics view and dashboard, you can view the upstream/downstream objects to the current view. For example, if you are selecting a metrics view, you can see all of the dashboards (in a dropdown) that are built on the metrics view. Likewise, if your model references several sources, this will be available to select. + ## Moving resources within your project From the UI, within the file explorer, you should be able to drag resources / objects around and move them from / to folders as necessary. diff --git a/docs/docs/deploy/deploy-credentials.md b/docs/docs/deploy/deploy-credentials.md index b328382251b..4be56646c53 100644 --- a/docs/docs/deploy/deploy-credentials.md +++ b/docs/docs/deploy/deploy-credentials.md @@ -14,6 +14,22 @@ As a general best practice, it is strongly recommended to use service accounts a ::: +## Configure Environmental Variables and Credentials on Rill Cloud + +Once you are ready to deploy or have already deployed and are experiencing issues connecting to your source, you will need to run the following command, `rill env configure`. When running this command, Rill will detect any connectors that are being used by the project and prompt you to fill in the required fields. When completed, this will be pushed to your Rill Cloud Deployment and automatically refresh the required objects. + + +```bash +$rill env configure +Finish deploying your project by providing access to the connectors. Rill requires credentials for the following connectors: + + - your connectors here (used by models and sources) + +Configuring connector "bigquery": +... + +Updated project variables +``` ## Service Accounts Service accounts are non-human user accounts that provide an identity for processes or services running on a server to interact with external resources, such as databases, APIs, and cloud services. Unlike personal user accounts, service accounts are intended for use by software applications or automated tools and do not require interactive login. In the context of Rill, service accounts are credentials that should be used for projects deployed to Rill Cloud. diff --git a/docs/docs/home/FAQ.md b/docs/docs/home/FAQ.md index e70059de402..1bbea71fb1f 100644 --- a/docs/docs/home/FAQ.md +++ b/docs/docs/home/FAQ.md @@ -95,58 +95,3 @@ You can follow the same steps as above. The button will have changed from `deplo ### How do I share my dashboard to other users? You will need to [invite users to your organization/project](https://docs.rilldata.com/manage/user-management#option-1---admin-invites-user) or send them a URL for them to [request access to your dashboard](https://docs.rilldata.com/manage/user-management#option-2---user-requests-access). If you just want them to see the contents of your dashboard, you can look into using [public URLs](https://docs.rilldata.com/explore/share-url). - - -## Rill Cloud Trial - -### What is Rill Cloud Trial? - -We now offer a self serve option for our users using Rill Cloud to get started. Before signing up for a [Teams plan](https://www.rilldata.com/pricing), you can create an account and start your free trial. Note that the banner will show you the remaining days for your trial and will update as the expiration gets closer! - -![img](/img/FAQ/rill-trial-banner.png) - -### When does my trial start? - -Your trial will start when you deploy a project to Rill Cloud. An Organization will be autoamatically created during this process using your email. - -### How long does my Rill Cloud Trial last? - -Currently, a Rill Cloud Trial lasts for 30 days. If you have any comments or concerns, please reach out to us on our [various platforms](../contact.md)! - -### What is included in the free trial? - -The free trial is locked at 2 projects and up to 10GB of data each. You can invite as many users as required. - -### What happens to my project if I do not pay on time? - -Your projects will hibernate. Your project metadata will still be available once you've activated your team plan. - -![expired](/img/FAQ/expired-project.png) - -### What is project hibernation? - -When a project is inactive for a specific number of days or your trial has expired, we automatically hibernate the project. What this means is that all of your information and metadata is saved and resource consumption will be zero. You will need to unhibernate the project to gain access to the dashboard again. If the project is hibernated due to payment issues, the project will stay in this state until payment is confirmed. Once the payment is confirmed, you can redeploy the project with the following: - -``` -rill project hibernate --redeploy -``` - -## Rill Cloud Team Plan (Billing) - -### How do I activate my Team Plan? - -You can activate your Team Plan via the Billing page from the settings in Rill Cloud. If you have not already, you will be prompted to add a payment method via Stripe. Please note that the cost of your plan depends on how much data you will be ingesting into Rill. For more information on costs, please refer to our [pricing page](https://www.rilldata.com/pricing). - -![billing](/img/FAQ/rill-org-billing.png) - - -### What does the Team Plan include? - -The base Team Plan starts at $250/month with 10GB of data included. For data over 10GB, this is priced at $25/GB. For more information, please refer to our [pricing page](https://www.rilldata.com/pricing). - - -### How can I see my current data usage? - -This is viewable via the Usage page in the Organization Settings page. - -! [usage] () \ No newline at end of file diff --git a/docs/docs/manage/variables-and-credentials.md b/docs/docs/manage/variables-and-credentials.md new file mode 100644 index 00000000000..9dc06923085 --- /dev/null +++ b/docs/docs/manage/variables-and-credentials.md @@ -0,0 +1,95 @@ +--- +title: Environmental Variables and Credentials in Rill Cloud +sidebar_label: Variables and Credentials +sidebar_position: 50 +--- + +The credentials in a deployed Rill Cloud projects can be managed on the Settings page or via the CLI. If you have yet to deploy your credentials, please follow the steps in our [deploy credentials page](/deploy/deploy-credentials#configure-environmental-variables-and-credentials-on-rill-cloud). + +## Modifying Variables and Credentials via the Settings Page + +![img](/img/tutorials/admin/env-var-ui.png) + +### Adding / Editing Environmental Variables +![img](/img/manage/var-and-creds/add-variable.png) + +:::tip Can't find the .env file? +By default, the hidden files will not be visible in the finder window. In order to view hidden files, you will need to enable "show hidden files". +Keyboard shortcut: Command + Shift + . +::: + +## Pushing and pulling credentials to / from Rill Cloud via the CLI + +If you have a project deployed to Rill Cloud, Rill provides the ability to **sync** the credentials between your local instance of Rill Developer and Rill cloud. This provides the ability to quickly reuse existing credentials, if configured, instead of having to manually input credentials each time. This can be accomplished by leveraging the `rill env push` and `rill env pull` CLI commands respectively. + +:::tip Avoid committing sensitive information to Git + +It's never a good idea to commit sensitive information to Git and goes against security best practices. Similar to credentials, if there are sensitive variables that you don't want to commit publicly to your `rill.yaml` configuration file (and thus potentially accessible by others), it's recommended to set them in your `.env` file directly and/or use `rill env set` via the CLI (and then optionally push / pull them as necessary). + +::: + +### `rill env push` + +As a project admin, you can either use `rill env configure` after deploying a project or `rill env push` to specify a particular set of credentials that your Rill Cloud project will use. If choosing the latter, you can update your *`/.env`* file with the appropriate variables and credentials that are required. Alternatively, if this file has already been updated, you can run `rill env push` from your project's root directory. +- Rill Cloud will use the specified credentials and variables in this `.env` file for the deployed project. +- Other users will also be able to use `rill env pull` to retrieve these defined credentials for local use (with Rill Developer). + +:::warning Overriding Cloud credentials + +If a credential and/or variable has already been configured in Rill Cloud, Rill will warn you about overriding if you attempt to push a new value in your `.env` file. This is because overriding credentials can impact your deployed project and/or other users (if they pull these credentials locally). + +::: + +### `rill env pull` + +For projects that have been deployed to Rill Cloud, an added benefit of our Rill Developer-Cloud architecture is that credentials that have been configured can be pulled locally for easier reuse (instead of having to manually reconfigure these credentials in Rill Developer). To do this, you can run `rill env pull` from your project's root directory to retrieve the latest credentials (after cloning the project's git repository to your local environment). + +```bash +rill env pull +Updated .env file with cloud credentials from project "". +``` + +:::info Overriding local credentials + +Please note when you run `rill env pull`, Rill will *automatically override any existing credentials or variables* that have been configured in your project's `.env` file if there is a match in the key name. This may result in unexpected behavior if you are using different credentials locally. + +::: + +### Credentials Naming Schema + +Connector credentials are essentially a form of project variable, prefixed using the `connector..` syntax. For example, `connector.druid.dsn` and `connector.clickhouse.dsn` are both hardcoded project variables (that happen to correspond to the [Druid](/reference/olap-engines/druid.md) and [ClickHouse](/reference/olap-engines/clickhouse.md) OLAP engines respectively). Please see below for each source and its required properties. + + +
+| **Source Name** | Property | Example | +| :-----------------------: | :-------------------------: | :------------------- | +| **GCS** |`GOOGLE_APPLICATION_CREDENTIALS`| `connector.gcs.google_application_credentials` | +| |`GCS_BUCKET_NAME`| `connector.gcs.gcs_bucket_name` | +| **AWS S3** | `AWS_ACCESS_KEY_ID` | `connector.s3.aws_access_key_id` | +| | `AWS_SECRET_ACCESS_KEY` |`connector.s3.aws_secret_access_key` | +| **Azure** |`AZURE_STORAGE_ACCOUNT`|`connector.azure.azure_storage_account`| +| |`AZURE_STORAGE_KEY`|`connector.azure.azure_storage_key`| +| |`AZURE_CLIENT_ID`|`connector.azure.azure_client_id`| +| |`AZURE_CLIENT_SECRET`|`connector.azure.azure_client_secret`| +| |`AZURE_TENANT_ID`|`connector.azure.azure_tenant_id`| +| **Big Query** | `GOOGLE_APPLICATION_CREDENTIALS` |`connector.bigquery.google_application_credentials` | +| **Snowflake** |`DSN`|`connector.snowflake.dsn`| +| **ClickHouse** |`HOST`|`connector.clickhouse.host `| +| |`PORT`|`connector.clickhouse.port `| +| |`USERNAME`|`connector.clickhouse.username `| +| |`PASSWORD`|`connector.clickhouse.password `| +| |`SSL`|`connector.clickhouse.ssl `| +| |`DATABASE`|`connector.clickhouse.database `| +... + +
\ No newline at end of file diff --git a/docs/docs/reference/project-files/explore-dashboards.md b/docs/docs/reference/project-files/explore-dashboards.md index f14d8710994..7ac1da43614 100644 --- a/docs/docs/reference/project-files/explore-dashboards.md +++ b/docs/docs/reference/project-files/explore-dashboards.md @@ -7,7 +7,6 @@ hide_table_of_contents: true In your Rill project directory, create a explore dashboard, `.yaml`, file in the `dashboards` directory. Rill will ingest the dashboard definition next time you run `rill start`. - ## Properties **`type`** — Refers to the resource type and must be `explore` _(required)_. @@ -18,15 +17,30 @@ In your Rill project directory, create a explore dashboard, `.ya **`display_name`** - Refers to the display name for the metrics view _(required)_. -**`description`** - A description for the project. _(optional)_. +**`description`** - A description for the project _(optional)_. + +**`dimensions`** - List of dimension names. Use `'*'` to select all dimensions (default) _(optional)_. + - **`regex`** - Select dimensions using a regular expression _(optional)_. + - **`exclude`** - Select all dimensions *except* those listed here _(optional)_. -**`dimensions`** - List of dimensions to include, defaults to `*`. _(optional)_. +```yaml +# Example: Select a dimension +dimensions: + - country - - **`exclude`** - Inversely a list of dimensions to exclude. Will ignore include if exclude is specified. _(optional)_. +# Example: Select all dimensions except one +dimensions: + exclude: + - country -**`measures`** - List of measures to include, defaults to `*`. _(optional)_. +# Example: Select all dimensions that match a regex +dimensions: + regex: "^public_.*$" +``` - - **`exclude`** - Inversely a list of measures to exclude. Will ignore include if exclude is specified. _(optional)_. +**`measures`** - List of measure names. Use `'*'` to select all measures (default) _(optional)_. + - **`regex`** - Select measures using a regular expression (see `dimensions` above for an example) _(optional)_. + - **`exclude`** - Select all measures *except* those listed here (see `dimensions` above for an example) _(optional)_. **`defaults`** - defines the defaults YAML struct @@ -81,6 +95,12 @@ In your Rill project directory, create a explore dashboard, `.ya **`time_zones`** — Refers to the time zones that should be pinned to the top of the time zone selector. It should be a list of [IANA time zone identifiers](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones). By adding one or more time zones will make the dashboard time zone aware and allow users to change current time zone within the dashboard _(optional)_. **`theme`** — Refers to the default theme to apply to the dashboard. A valid theme must be defined in the project. Read this [page](./themes.md) for more detailed information about themes _(optional)_. +```yaml +theme: + colors: + primary: hsl(180, 100%, 50%) + secondary: lightgreen +``` **`security`** - Defines a [security policy](/manage/security) for the dashboard _(optional)_. - **`access`** - Expression indicating if the user should be granted access to the dashboard. If not defined, it will resolve to `false` and the dashboard won't be accessible to anyone. Needs to be a valid SQL expression that evaluates to a boolean _(optional)_. \ No newline at end of file diff --git a/docs/docs/reference/project-files/metrics-view.md b/docs/docs/reference/project-files/metrics-view.md index 369b98b9296..8cf686b2338 100644 --- a/docs/docs/reference/project-files/metrics-view.md +++ b/docs/docs/reference/project-files/metrics-view.md @@ -53,6 +53,13 @@ In your Rill project directory, create a metrics view, `.yaml`, fi - **Example**: to show a measure using fixed point formatting with 2 digits after the decimal point, your measure specification would include: `format_d3: ".2f"`. - **Example**: to show a measure using grouped thousands with two significant digits, your measure specification would include: `format_d3: ",.2r"`. - **`format_d3_locale`** — locale configuration passed through to D3, enabling changing the currency symbol among other things. For details, see the docs for D3's [`formatLocale`](https://d3js.org/d3-format#formatLocale). _(optional)_ + +```yaml + format_d3: "$," + format_d3_locale: + grouping: [3, 2] + currency: ["₹", ""] +``` - **`format_preset`** — controls the formatting of this measure according to option specified below. Measures cannot have both `format_preset` and `format_d3` entries. _(optional; if neither `format_preset` nor `format_d3` is supplied, measures will be formatted with the `humanize` preset)_ - `humanize` — round off numbers in an opinionated way to thousands (K), millions (M), billions (B), etc. - `none` — raw output diff --git a/docs/docs/tutorials/administration/project/credentials-env-variable-management.md b/docs/docs/tutorials/administration/project/credentials-env-variable-management.md index 9118102e8f1..b7b7c768ea3 100644 --- a/docs/docs/tutorials/administration/project/credentials-env-variable-management.md +++ b/docs/docs/tutorials/administration/project/credentials-env-variable-management.md @@ -13,12 +13,9 @@ Please review the documentation on [Credential Managment](https://docs.rilldata. ## Managing Credentials and Variables on Rill Cloud - -
-Historically (pre 0.48), management was only possible via the CLI. Now, it is also possible to do so via the UI! - -
+![env-ui](/img/tutorials/admin/env-var-ui.png) +After pushing your initial credentials into Rill Cloud, you will be able to view them in the Settings page. In the above screenshot, we have already run `rill env configure` from the CLI so it has populated the required credentials in the UI via the .env file in your project directory. ## Credentials diff --git a/docs/docs/tutorials/administration/project/env-var-ui.png b/docs/docs/tutorials/administration/project/env-var-ui.png new file mode 100644 index 00000000000..904e1b69435 Binary files /dev/null and b/docs/docs/tutorials/administration/project/env-var-ui.png differ diff --git a/docs/docs/tutorials/other/component-variable-freedom.md b/docs/docs/tutorials/other/component-variable-freedom.md deleted file mode 100644 index 4899a1c77e9..00000000000 --- a/docs/docs/tutorials/other/component-variable-freedom.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -title: "Using Variables in Components, to what extent?" -sidebar_label: "Filter your Components for your own personal View" -sidebar_position: 10 -hide_table_of_contents: false -tags: - - Canvas Dashboard - - Canvas Component ---- - -## An idea - -What if you create a component that is COMPLETELY based off of variables? -The source is selectable and depending on which source your selecting, this filters the filters on which columns are measures are available. - - -selector_source.yaml -> outputs 'source_name' to all components - -source_name -> selector_dimensions.yaml -> outputs dimensions -> all components -source_name -> selector_measure.yaml -> outputs measures -> all components - -source_name and dimension -> selector_group_by -> outputs a groupby for components defaults to "" - -source_name, measure -x, measure_0 -x, measure_1 -y, measure_0 -y, measure_1 - diff --git a/docs/docs/tutorials/rill_basics/4-metrics-view.md b/docs/docs/tutorials/rill_basics/4-metrics-view.md index 439fd267682..958bf9e1665 100644 --- a/docs/docs/tutorials/rill_basics/4-metrics-view.md +++ b/docs/docs/tutorials/rill_basics/4-metrics-view.md @@ -134,3 +134,84 @@ measures: Measure are the numeric aggreagtes of columns from your data model. These function will use DuckDB SQL aggregation functions and expressions. Similar to dimensions, you will need to create an expression based on the column of your underlying model or table. Our first measure will be: `SUM(added_lines)`. + +## Adding more Functionality + +Let's add further dimensions and measure to the metrics layer and see the changes to the explore dashboard. + +### Dimensions + +From our dataset, we can add more dimensions to allow more filtering and exploration of the measures we will create. + + Add the following dimensions, with title and description. + - author_name + - author_timezone + - filename + +### Measures + + We can definitely create better aggregations for some more meaningful data based on these commits. + - sum(added_lines) + - sum(deleted_lines) + + +You may need to reference the metrics view YAML reference guide to figure out the above. Your final output should look something like this! + +![finished](/img/tutorials/103/Completed-100-dashboard.png) + + +
+ Working Metrics View YAML + ```yaml +# Metrics View YAML +# Reference documentation: https://docs.rilldata.com/reference/project-files/metrics_views + +version: 1 +type: metrics_view + +table: commits___model + +timeseries: author_date # Select an actual timestamp column (if any) from your table + +dimensions: + - column: author_name + name: author_name + label: The Author's Name + description: The name of the author of the commit + + - column: author_timezone + label: "The Author's TZ" + description: "The Author's Timezone" + + - column: filename + label: "The filename" + description: "The name of the modified filename" + +measures: + - expression: SUM(added_lines) + name: added_lines + label: Sum of Added lines + format_preset: humanize + description: The aggregate sum of added_lines column. + valid_percent_of_total: true + + - expression: "SUM(deleted_lines)" + label: "Sum of deleted lines" + description: "The aggregate sum of deleted_lines column." + +``` + +
+ + +### Completed visual metrics editor + +If you decide to build out the metrics view via the UI, it should look something like below! + +![img](/img/tutorials/103/visual-metric-editor.png) + + +import DocsRating from '@site/src/components/DocsRating'; + +--- + diff --git a/docs/docs/tutorials/rill_basics/5-dashboard.md b/docs/docs/tutorials/rill_basics/5-dashboard.md index d1296e7e4fa..b6f4808baa8 100644 --- a/docs/docs/tutorials/rill_basics/5-dashboard.md +++ b/docs/docs/tutorials/rill_basics/5-dashboard.md @@ -7,99 +7,47 @@ tags: - OLAP:DuckDB --- -## Create the Explore dashboard -At this point our metrics view is ready! Let's rename the metrics to `commits___model_metrics.yaml` and select `Create Explore`. +At this point our metrics view is ready! Let's select `Create Explore Dashboard`. This will automatically populate the explore dashboard to select all the of created metrics and dimension in your metrics view. We can make changes to the view via the YAML or visual dashboard editor. +![img](/img/tutorials/103/Completed-100-dashboard.png) -This will create an explore-dashboards folder with a very simple YAML file. Let's go ahead a select preview to see what it looks like. You should see something similar to the below. +## Making Changes +### Visual Explore Dashboard -![simple](/img/tutorials/103/simple-dashboard.png) +![img](/img/tutorials/103/visual-dashboard-tutorial.png) +On the right panel, you are able to select measures, dimensions, time ranges, and various other components that control the view of your dashboard. In the main area, you will see a preview of what your dashboard will look like. You can also select the code view to make any needed changes and/or set more advanced settings as found in our [explore dashboard YAML reference.](https://docs.rilldata.com/reference/project-files/explore-dashboards) -We can definitely do better than that! +### YAML +By default, the page will contain the basics parameters as seen below. You can add more advanced settings as you require for you use case. +```YAML +# Explore YAML +# Reference documentation: https://docs.rilldata.com/reference/project-files/explore-dashboards ---- -For a quick summary on the different components that we modified, and its respective parts in the dashboard UI. - - -
- - -## Adding more Functionality - -Let's add further dimensions and measure to the metrics layer and see the changes to the explore dashboard. - -### Dimensions - -From our dataset, we can add more dimensions to allow more filtering and exploration of the measures we will create. - - Add the following dimensions, with title and description. - - author_name - - author_timezone - - filename - -### Measures - - We can definitely create better aggregations for some more meaningful data based on these commits. - - sum(added_lines) - - sum(deleted_lines) +type: explore +title: "commits___model_metrics dashboard" +metrics_view: commits___model_metrics -You may need to reference the metrics view YAML reference guide to figure out the above. Your final output should look something like this! - -![finished](/img/tutorials/103/Completed-100-dashboard.png) - - -
- Working Metrics View YAML - ```yaml -# Metrics View YAML -# Reference documentation: https://docs.rilldata.com/reference/project-files/metrics_views - -version: 1 -type: metrics_view - -table: commits___model - -timeseries: author_date # Select an actual timestamp column (if any) from your table - -dimensions: - - column: author_name - name: author_name - label: The Author's Name - description: The name of the author of the commit +dimensions: '*' +measures: '*' +``` - - column: author_timezone - label: "The Author's TZ" - description: "The Author's Timezone" - - column: filename - label: "The filename" - description: "The name of the modified filename" -measures: - - expression: SUM(added_lines) - name: added_lines - label: Sum of Added lines - format_preset: humanize - description: The aggregate sum of added_lines column. - valid_percent_of_total: true +### Explore Dasboard Components - - expression: "SUM(deleted_lines)" - label: "Sum of deleted lines" - description: "The aggregate sum of deleted_lines column." +For a quick summary on the different components of an explore dashboard, and its respective parts in the dashboard UI. -``` + +
-
+--- -### Completed visual metrics editor -If you decide to build out the metrics view via the UI, it should look something like below! -![img](/img/tutorials/103/visual-metric-editor.png) diff --git a/docs/docs/tutorials/rill_clickhouse/2-r_ch_connect.md b/docs/docs/tutorials/rill_clickhouse/2-r_ch_connect.md index 1f8e8fb240c..ff1c843d5c1 100644 --- a/docs/docs/tutorials/rill_clickhouse/2-r_ch_connect.md +++ b/docs/docs/tutorials/rill_clickhouse/2-r_ch_connect.md @@ -25,17 +25,8 @@ Depending what you choose, the contents of your connection will change and I rec ::: -### Changing the default OLAP engine -Let's navigate to the rill.yaml file and add the following. - -```yaml -compiler: rillv1 - -title: "Rill and ClickHouse Project" -olap_connector: clickhouse -``` - -Once updated, we can create the clickhouse connection by selection `+Add Data` > `ClickHouse` and fill in the components on the UI. +### Connect to ClickHouse +We can create the clickhouse connection by selection `+Add Data` > `ClickHouse` and fill in the components on the UI. ![clickhouse](/img/tutorials/ch/clickhouse-connector.png) :::tip @@ -47,7 +38,14 @@ You can obtain the credentials from your ClickHouse Cloud account by clicking th ``` ::: -Once this is created, a `clickhouse.yaml` file will appear in the `connectors` folder. +Once this is created, a `clickhouse.yaml` file will appear in the `connectors` folder and the following will be added to your rill.yaml. + +```yaml +compiler: rillv1 + +title: "Rill and ClickHouse Project" +olap_connector: clickhouse #automatically added +``` Example for a locally running ClickHouse server: ```yaml diff --git a/docs/docs/tutorials/rill_clickhouse/4-r_ch_dashboard.md b/docs/docs/tutorials/rill_clickhouse/4-r_ch_dashboard.md index 626d4dea557..7319c4de593 100644 --- a/docs/docs/tutorials/rill_clickhouse/4-r_ch_dashboard.md +++ b/docs/docs/tutorials/rill_clickhouse/4-r_ch_dashboard.md @@ -10,7 +10,7 @@ tags: ### Create the explore dashboard -When you're ready, you can create the visualization on top of the metric layer. Let's select `Create Explore dashboard`. This will create a simple explore-dashboards/uk_price_paid_metrics_explore.yaml file that reads in all the dimensions and measures. For more information on the available key-pairs, please refer to the [reference documentation.](https://docs.rilldata.com/reference/project-files/explores) +When you're ready, you can create the visualization on top of the metric layer. Let's select `Create Explore dashboard`. This will create a simple explore-dashboards/uk_price_paid_metrics_explore.yaml file that reads in all the dimensions and measures. For more information on the available key-pairs, please refer to the [reference documentation.](https://docs.rilldata.com/reference/project-files/explore-dashboards) --- diff --git a/docs/sidebars.js b/docs/sidebars.js index d4d8e95be79..905afaa961c 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -32,7 +32,7 @@ const sidebars = { { type: 'category', label: 'Rill Developer to Rill Cloud in 6 steps!', - description: 'Rill Developer to to Rill Cloud', + description: 'Rill Developer to Rill Cloud', items: [ 'tutorials/rill_basics/launch', diff --git a/docs/static/img/build/dashboard/explore-dashboard.gif b/docs/static/img/build/dashboard/explore-dashboard.gif new file mode 100644 index 00000000000..e623a80df40 Binary files /dev/null and b/docs/static/img/build/dashboard/explore-dashboard.gif differ diff --git a/docs/static/img/build/structure/breadcrumb.png b/docs/static/img/build/structure/breadcrumb.png new file mode 100644 index 00000000000..b3ea8ba8b4f Binary files /dev/null and b/docs/static/img/build/structure/breadcrumb.png differ diff --git a/docs/static/img/manage/var-and-creds/add-variable.png b/docs/static/img/manage/var-and-creds/add-variable.png new file mode 100644 index 00000000000..0229dd4178e Binary files /dev/null and b/docs/static/img/manage/var-and-creds/add-variable.png differ diff --git a/docs/static/img/tutorials/103/visual-dashboard-tutorial.png b/docs/static/img/tutorials/103/visual-dashboard-tutorial.png new file mode 100644 index 00000000000..e9a80dd7b9f Binary files /dev/null and b/docs/static/img/tutorials/103/visual-dashboard-tutorial.png differ diff --git a/docs/static/img/tutorials/admin/env-var-ui.png b/docs/static/img/tutorials/admin/env-var-ui.png new file mode 100644 index 00000000000..904e1b69435 Binary files /dev/null and b/docs/static/img/tutorials/admin/env-var-ui.png differ diff --git a/go.mod b/go.mod index e5cda852bea..07bbe2e372b 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 - github.com/ClickHouse/clickhouse-go/v2 v2.20.0 + github.com/ClickHouse/clickhouse-go/v2 v2.30.0 github.com/ForceCLI/force v1.0.5-0.20231227180521-1b251cf1a8b0 github.com/Masterminds/sprig/v3 v3.2.3 github.com/MicahParks/keyfunc v1.9.0 @@ -90,7 +90,7 @@ require ( github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 github.com/startreedata/pinot-client-go v0.4.0 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.10.0 github.com/stripe/stripe-go/v79 v79.6.0 github.com/testcontainers/testcontainers-go v0.33.0 github.com/testcontainers/testcontainers-go/modules/clickhouse v0.33.0 @@ -99,15 +99,15 @@ require ( go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.52.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0 go.opentelemetry.io/contrib/instrumentation/runtime v0.52.0 - go.opentelemetry.io/otel v1.32.0 + go.opentelemetry.io/otel v1.33.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.27.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 go.opentelemetry.io/otel/exporters/prometheus v0.49.0 - go.opentelemetry.io/otel/metric v1.32.0 + go.opentelemetry.io/otel/metric v1.33.0 go.opentelemetry.io/otel/sdk v1.27.0 go.opentelemetry.io/otel/sdk/metric v1.27.0 - go.opentelemetry.io/otel/trace v1.32.0 + go.opentelemetry.io/otel/trace v1.33.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 go.uber.org/zap/exp v0.2.0 @@ -115,7 +115,7 @@ require ( golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 golang.org/x/oauth2 v0.22.0 golang.org/x/sync v0.8.0 - golang.org/x/sys v0.26.0 + golang.org/x/sys v0.28.0 golang.org/x/text v0.19.0 google.golang.org/api v0.184.0 google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 @@ -147,7 +147,7 @@ require ( github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect - github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/mtibben/percent v0.2.1 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect @@ -165,7 +165,7 @@ require ( cloud.google.com/go/auth v0.5.1 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect - github.com/ClickHouse/ch-go v0.61.3 // indirect + github.com/ClickHouse/ch-go v0.63.1 // indirect github.com/ForceCLI/config v0.0.0-20230217143549-9149d42a3c99 // indirect github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/Masterminds/goutils v1.1.1 // indirect @@ -217,7 +217,7 @@ require ( github.com/docker/cli v27.0.3+incompatible // indirect github.com/docker/compose/v2 v2.28.1 // indirect github.com/docker/distribution v2.8.3+incompatible // indirect - github.com/docker/docker v27.1.2+incompatible // indirect + github.com/docker/docker v27.3.0+incompatible // indirect github.com/docker/docker-credential-helpers v0.8.0 // indirect github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect github.com/docker/go-metrics v0.0.1 // indirect @@ -330,7 +330,7 @@ require ( github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect - github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pingcap/errors v0.11.5-0.20221009092201-b66cddb77c32 // indirect github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22 // indirect @@ -355,7 +355,7 @@ require ( github.com/shibumi/go-pathspec v1.3.0 // indirect github.com/shirou/gopsutil/v3 v3.24.5 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect - github.com/shopspring/decimal v1.3.1 // indirect + github.com/shopspring/decimal v1.4.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/skeema/knownhosts v1.2.2 // indirect github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect @@ -382,6 +382,7 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0 // indirect diff --git a/go.sum b/go.sum index 8b5d1163329..145cb7ee353 100644 --- a/go.sum +++ b/go.sum @@ -656,10 +656,10 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mx github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/ClickHouse/ch-go v0.61.3 h1:MmBwUhXrAOBZK7n/sWBzq6FdIQ01cuF2SaaO8KlDRzI= -github.com/ClickHouse/ch-go v0.61.3/go.mod h1:1PqXjMz/7S1ZUaKvwPA3i35W2bz2mAMFeCi6DIXgGwQ= -github.com/ClickHouse/clickhouse-go/v2 v2.20.0 h1:bvlLQ31XJfl7MxIqAq2l1G6JhHYzqEXdvfpMeU6bkKc= -github.com/ClickHouse/clickhouse-go/v2 v2.20.0/go.mod h1:VQfyA+tCwCRw2G7ogfY8V0fq/r0yJWzy8UDrjiP/Lbs= +github.com/ClickHouse/ch-go v0.63.1 h1:s2JyZvWLTCSAGdtjMBBmAgQQHMco6pawLJMOXi0FODM= +github.com/ClickHouse/ch-go v0.63.1/go.mod h1:I1kJJCL3WJcBMGe1m+HVK0+nREaG+JOYYBWjrDrF3R0= +github.com/ClickHouse/clickhouse-go/v2 v2.30.0 h1:AG4D/hW39qa58+JHQIFOSnxyL46H6h2lrmGGk17dhFo= +github.com/ClickHouse/clickhouse-go/v2 v2.30.0/go.mod h1:i9ZQAojcayW3RsdCb3YR+n+wC2h65eJsZCscZ1Z1wyo= github.com/ForceCLI/config v0.0.0-20230217143549-9149d42a3c99 h1:H2axnitaP3Dw+tocMHPQHjM2wJ/+grF8sOIQGaJeEsg= github.com/ForceCLI/config v0.0.0-20230217143549-9149d42a3c99/go.mod h1:WHFXv3VIHldTnYGmWAXAxsu4O754A9Zakq4DedI8PSA= github.com/ForceCLI/force v1.0.5-0.20231227180521-1b251cf1a8b0 h1:XPYvEs+GpfNekTXPfOfkUWpbRYpOVorykDs6IPzlax8= @@ -1120,8 +1120,8 @@ github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBi github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.17+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v27.1.2+incompatible h1:AhGzR1xaQIy53qCkxARaFluI00WPGtXn0AJuoQsVYTY= -github.com/docker/docker v27.1.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.3.0+incompatible h1:BNb1QY6o4JdKpqwi9IB+HUYcRRrVN4aGFUTvDmWYK1A= +github.com/docker/docker v27.3.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/docker-credential-helpers v0.8.0 h1:YQFtbBQb4VrpoPxhFuzEBPQ9E16qz5SpHLS+uswaCp8= github.com/docker/docker-credential-helpers v0.8.0/go.mod h1:UGFXcuoQ5TxPiB54nHOZ32AWRqQdECoh/Mg0AlEYb40= @@ -1674,8 +1674,8 @@ github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0f github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= -github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jhump/gopoet v0.0.0-20190322174617-17282ff210b3/go.mod h1:me9yfT6IJSlOL3FCfrg+L6yzUEZ+5jW6WHt4Sk+UPUI= @@ -2020,8 +2020,8 @@ github.com/phpdave11/gofpdf v1.4.2/go.mod h1:zpO6xFn9yxo3YLyMvW8HcKWVdbNqgIfOOp2 github.com/phpdave11/gofpdi v1.0.12/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/phpdave11/gofpdi v1.0.13/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= -github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= +github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20221009092201-b66cddb77c32 h1:m5ZsBa5o/0CkzZXfXLaThzKuR85SnHHetqBCpzQ30h8= @@ -2132,8 +2132,8 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= @@ -2176,8 +2176,9 @@ github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= @@ -2266,8 +2267,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stripe/stripe-go/v79 v79.6.0 h1:qSBV2f2rpLEEZTdTlVLzdmQJZNmfoo2E3hUEkFT8GBc= github.com/stripe/stripe-go/v79 v79.6.0/go.mod h1:cuH6X0zC8peY6f1AubHwgJ/fJSn2dh5pfiCr6CjyKVU= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= @@ -2396,6 +2397,8 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.28.0/go.mod h1:vEhqr0m4eTc+DWxfsXoXue2GBgV2uUwVznkGIHW/e5w= @@ -2410,8 +2413,8 @@ go.opentelemetry.io/contrib/instrumentation/runtime v0.52.0 h1:UaQVCH34fQsyDjlgS go.opentelemetry.io/contrib/instrumentation/runtime v0.52.0/go.mod h1:Ks4aHdMgu1vAfEY0cIBHcGx2l1S0+PwFm2BE/HRzqSk= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs= -go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U= -go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg= +go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= +go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4= go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 h1:ZtfnDL+tUrs1F0Pzfwbg2d59Gru9NCH3bgSHBM6LDwU= @@ -2432,8 +2435,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0/go.mod h go.opentelemetry.io/otel/exporters/prometheus v0.49.0 h1:Er5I1g/YhfYv9Affk9nJLfH/+qCCVVg1f2R9AbJfqDQ= go.opentelemetry.io/otel/exporters/prometheus v0.49.0/go.mod h1:KfQ1wpjf3zsHjzP149P4LyAwWRupc6c7t1ZJ9eXpKQM= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M= -go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8= +go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= +go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs= @@ -2445,8 +2448,8 @@ go.opentelemetry.io/otel/sdk/metric v1.27.0 h1:5uGNOlpXi+Hbo/DRoI31BSb1v+OGcpv2N go.opentelemetry.io/otel/sdk/metric v1.27.0/go.mod h1:we7jJVrYN2kh3mVBlswtPU22K0SA+769l93J6bsyvqw= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk= -go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM= -go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8= +go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= +go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.11.0/go.mod h1:QpEjXPrNQzrFDZgoTo49dgHR9RYRSrg3NAKnUGl9YpQ= go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= @@ -2862,8 +2865,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/proto/gen/rill/runtime/v1/runtime.swagger.yaml b/proto/gen/rill/runtime/v1/runtime.swagger.yaml index d5578f954af..ff4c3b2f797 100644 --- a/proto/gen/rill/runtime/v1/runtime.swagger.yaml +++ b/proto/gen/rill/runtime/v1/runtime.swagger.yaml @@ -3099,6 +3099,7 @@ definitions: - CODE_FLOAT32 - CODE_FLOAT64 - CODE_TIMESTAMP + - CODE_INTERVAL - CODE_DATE - CODE_TIME - CODE_STRING diff --git a/proto/gen/rill/runtime/v1/schema.pb.go b/proto/gen/rill/runtime/v1/schema.pb.go index 449b7fa0926..77c603b1cdb 100644 --- a/proto/gen/rill/runtime/v1/schema.pb.go +++ b/proto/gen/rill/runtime/v1/schema.pb.go @@ -41,6 +41,7 @@ const ( Type_CODE_FLOAT32 Type_Code = 12 Type_CODE_FLOAT64 Type_Code = 13 Type_CODE_TIMESTAMP Type_Code = 14 + Type_CODE_INTERVAL Type_Code = 27 Type_CODE_DATE Type_Code = 15 Type_CODE_TIME Type_Code = 16 Type_CODE_STRING Type_Code = 17 @@ -73,6 +74,7 @@ var ( 12: "CODE_FLOAT32", 13: "CODE_FLOAT64", 14: "CODE_TIMESTAMP", + 27: "CODE_INTERVAL", 15: "CODE_DATE", 16: "CODE_TIME", 17: "CODE_STRING", @@ -102,6 +104,7 @@ var ( "CODE_FLOAT32": 12, "CODE_FLOAT64": 13, "CODE_TIMESTAMP": 14, + "CODE_INTERVAL": 27, "CODE_DATE": 15, "CODE_TIME": 16, "CODE_STRING": 17, @@ -392,7 +395,7 @@ var file_rill_runtime_v1_schema_proto_rawDesc = []byte{ 0x0a, 0x1c, 0x72, 0x69, 0x6c, 0x6c, 0x2f, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2f, 0x76, 0x31, 0x2f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0f, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x22, - 0xd6, 0x05, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x2e, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, + 0xe9, 0x05, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x2e, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1a, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x2e, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x6e, 0x75, 0x6c, 0x6c, @@ -408,7 +411,7 @@ var file_rill_runtime_v1_schema_proto_rawDesc = []byte{ 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x33, 0x0a, 0x08, 0x6d, 0x61, 0x70, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x61, 0x70, 0x54, - 0x79, 0x70, 0x65, 0x52, 0x07, 0x6d, 0x61, 0x70, 0x54, 0x79, 0x70, 0x65, 0x22, 0xc9, 0x03, 0x0a, + 0x79, 0x70, 0x65, 0x52, 0x07, 0x6d, 0x61, 0x70, 0x54, 0x79, 0x70, 0x65, 0x22, 0xdc, 0x03, 0x0a, 0x04, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x14, 0x0a, 0x10, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x42, 0x4f, 0x4f, 0x4c, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, @@ -427,46 +430,47 @@ var file_rill_runtime_v1_schema_proto_rawDesc = []byte{ 0x10, 0x1a, 0x12, 0x10, 0x0a, 0x0c, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x46, 0x4c, 0x4f, 0x41, 0x54, 0x33, 0x32, 0x10, 0x0c, 0x12, 0x10, 0x0a, 0x0c, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x46, 0x4c, 0x4f, 0x41, 0x54, 0x36, 0x34, 0x10, 0x0d, 0x12, 0x12, 0x0a, 0x0e, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x54, - 0x49, 0x4d, 0x45, 0x53, 0x54, 0x41, 0x4d, 0x50, 0x10, 0x0e, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, - 0x44, 0x45, 0x5f, 0x44, 0x41, 0x54, 0x45, 0x10, 0x0f, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x44, - 0x45, 0x5f, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x10, 0x12, 0x0f, 0x0a, 0x0b, 0x43, 0x4f, 0x44, 0x45, - 0x5f, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x11, 0x12, 0x0e, 0x0a, 0x0a, 0x43, 0x4f, 0x44, - 0x45, 0x5f, 0x42, 0x59, 0x54, 0x45, 0x53, 0x10, 0x12, 0x12, 0x0e, 0x0a, 0x0a, 0x43, 0x4f, 0x44, - 0x45, 0x5f, 0x41, 0x52, 0x52, 0x41, 0x59, 0x10, 0x13, 0x12, 0x0f, 0x0a, 0x0b, 0x43, 0x4f, 0x44, - 0x45, 0x5f, 0x53, 0x54, 0x52, 0x55, 0x43, 0x54, 0x10, 0x14, 0x12, 0x0c, 0x0a, 0x08, 0x43, 0x4f, - 0x44, 0x45, 0x5f, 0x4d, 0x41, 0x50, 0x10, 0x15, 0x12, 0x10, 0x0a, 0x0c, 0x43, 0x4f, 0x44, 0x45, - 0x5f, 0x44, 0x45, 0x43, 0x49, 0x4d, 0x41, 0x4c, 0x10, 0x16, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, - 0x44, 0x45, 0x5f, 0x4a, 0x53, 0x4f, 0x4e, 0x10, 0x17, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x44, - 0x45, 0x5f, 0x55, 0x55, 0x49, 0x44, 0x10, 0x18, 0x22, 0x8f, 0x01, 0x0a, 0x0a, 0x53, 0x74, 0x72, - 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, - 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, - 0x54, 0x79, 0x70, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, - 0x64, 0x73, 0x1a, 0x46, 0x0a, 0x05, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, - 0x29, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, - 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, - 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x71, 0x0a, 0x07, 0x4d, 0x61, - 0x70, 0x54, 0x79, 0x70, 0x65, 0x12, 0x30, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x5f, 0x74, 0x79, 0x70, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, - 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x07, - 0x6b, 0x65, 0x79, 0x54, 0x79, 0x70, 0x65, 0x12, 0x34, 0x0a, 0x0a, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x72, 0x69, - 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x79, - 0x70, 0x65, 0x52, 0x09, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x54, 0x79, 0x70, 0x65, 0x42, 0xbe, 0x01, - 0x0a, 0x13, 0x63, 0x6f, 0x6d, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, - 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x42, 0x0b, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x50, 0x72, 0x6f, - 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x3c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x72, 0x69, 0x6c, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x72, 0x69, 0x6c, 0x6c, 0x2f, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x72, 0x69, 0x6c, 0x6c, 0x2f, 0x72, 0x75, - 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2f, 0x76, 0x31, 0x3b, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, - 0x76, 0x31, 0xa2, 0x02, 0x03, 0x52, 0x52, 0x58, 0xaa, 0x02, 0x0f, 0x52, 0x69, 0x6c, 0x6c, 0x2e, - 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x56, 0x31, 0xca, 0x02, 0x0f, 0x52, 0x69, 0x6c, - 0x6c, 0x5c, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x1b, 0x52, - 0x69, 0x6c, 0x6c, 0x5c, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5c, 0x56, 0x31, 0x5c, 0x47, - 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x11, 0x52, 0x69, 0x6c, - 0x6c, 0x3a, 0x3a, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x3a, 0x3a, 0x56, 0x31, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x49, 0x4d, 0x45, 0x53, 0x54, 0x41, 0x4d, 0x50, 0x10, 0x0e, 0x12, 0x11, 0x0a, 0x0d, 0x43, 0x4f, + 0x44, 0x45, 0x5f, 0x49, 0x4e, 0x54, 0x45, 0x52, 0x56, 0x41, 0x4c, 0x10, 0x1b, 0x12, 0x0d, 0x0a, + 0x09, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x44, 0x41, 0x54, 0x45, 0x10, 0x0f, 0x12, 0x0d, 0x0a, 0x09, + 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x10, 0x12, 0x0f, 0x0a, 0x0b, 0x43, + 0x4f, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x11, 0x12, 0x0e, 0x0a, 0x0a, + 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x42, 0x59, 0x54, 0x45, 0x53, 0x10, 0x12, 0x12, 0x0e, 0x0a, 0x0a, + 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x41, 0x52, 0x52, 0x41, 0x59, 0x10, 0x13, 0x12, 0x0f, 0x0a, 0x0b, + 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x53, 0x54, 0x52, 0x55, 0x43, 0x54, 0x10, 0x14, 0x12, 0x0c, 0x0a, + 0x08, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x4d, 0x41, 0x50, 0x10, 0x15, 0x12, 0x10, 0x0a, 0x0c, 0x43, + 0x4f, 0x44, 0x45, 0x5f, 0x44, 0x45, 0x43, 0x49, 0x4d, 0x41, 0x4c, 0x10, 0x16, 0x12, 0x0d, 0x0a, + 0x09, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x4a, 0x53, 0x4f, 0x4e, 0x10, 0x17, 0x12, 0x0d, 0x0a, 0x09, + 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x55, 0x55, 0x49, 0x44, 0x10, 0x18, 0x22, 0x8f, 0x01, 0x0a, 0x0a, + 0x53, 0x74, 0x72, 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x66, 0x69, + 0x65, 0x6c, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x72, 0x69, 0x6c, + 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x72, + 0x75, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x06, 0x66, + 0x69, 0x65, 0x6c, 0x64, 0x73, 0x1a, 0x46, 0x0a, 0x05, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x12, + 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x12, 0x29, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x15, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, + 0x76, 0x31, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, 0x71, 0x0a, + 0x07, 0x4d, 0x61, 0x70, 0x54, 0x79, 0x70, 0x65, 0x12, 0x30, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x5f, + 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x72, 0x69, 0x6c, + 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x79, 0x70, + 0x65, 0x52, 0x07, 0x6b, 0x65, 0x79, 0x54, 0x79, 0x70, 0x65, 0x12, 0x34, 0x0a, 0x0a, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, + 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, + 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x09, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x54, 0x79, 0x70, 0x65, + 0x42, 0xbe, 0x01, 0x0a, 0x13, 0x63, 0x6f, 0x6d, 0x2e, 0x72, 0x69, 0x6c, 0x6c, 0x2e, 0x72, 0x75, + 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x76, 0x31, 0x42, 0x0b, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x3c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x72, 0x69, 0x6c, 0x6c, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x72, 0x69, 0x6c, + 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x72, 0x69, 0x6c, 0x6c, + 0x2f, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2f, 0x76, 0x31, 0x3b, 0x72, 0x75, 0x6e, 0x74, + 0x69, 0x6d, 0x65, 0x76, 0x31, 0xa2, 0x02, 0x03, 0x52, 0x52, 0x58, 0xaa, 0x02, 0x0f, 0x52, 0x69, + 0x6c, 0x6c, 0x2e, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x56, 0x31, 0xca, 0x02, 0x0f, + 0x52, 0x69, 0x6c, 0x6c, 0x5c, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5c, 0x56, 0x31, 0xe2, + 0x02, 0x1b, 0x52, 0x69, 0x6c, 0x6c, 0x5c, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x5c, 0x56, + 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x11, + 0x52, 0x69, 0x6c, 0x6c, 0x3a, 0x3a, 0x52, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x3a, 0x3a, 0x56, + 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/proto/rill/runtime/v1/schema.proto b/proto/rill/runtime/v1/schema.proto index 3e3e0d04675..b602c8f8a3d 100644 --- a/proto/rill/runtime/v1/schema.proto +++ b/proto/rill/runtime/v1/schema.proto @@ -22,6 +22,7 @@ message Type { CODE_FLOAT32 = 12; CODE_FLOAT64 = 13; CODE_TIMESTAMP = 14; + CODE_INTERVAL = 27; CODE_DATE = 15; CODE_TIME = 16; CODE_STRING = 17; diff --git a/runtime/compilers/rillv1/parse_alert.go b/runtime/compilers/rillv1/parse_alert.go index aae9e56c410..97b8cb30242 100644 --- a/runtime/compilers/rillv1/parse_alert.go +++ b/runtime/compilers/rillv1/parse_alert.go @@ -276,6 +276,9 @@ func (p *Parser) parseAlert(node *Node) error { // NOTE: After calling insertResource, an error must not be returned. Any validation should be done before calling it. r.AlertSpec.DisplayName = tmp.DisplayName + if r.AlertSpec.DisplayName == "" { + r.AlertSpec.DisplayName = ToDisplayName(node.Name) + } if schedule != nil { r.AlertSpec.RefreshSchedule = schedule } diff --git a/runtime/compilers/rillv1/parse_canvas.go b/runtime/compilers/rillv1/parse_canvas.go index c81d490b412..c020b3f3473 100644 --- a/runtime/compilers/rillv1/parse_canvas.go +++ b/runtime/compilers/rillv1/parse_canvas.go @@ -109,6 +109,9 @@ func (p *Parser) parseCanvas(node *Node) error { // NOTE: After calling insertResource, an error must not be returned. Any validation should be done before calling it. r.CanvasSpec.DisplayName = tmp.DisplayName + if r.CanvasSpec.DisplayName == "" { + r.CanvasSpec.DisplayName = ToDisplayName(node.Name) + } r.CanvasSpec.Columns = tmp.Columns r.CanvasSpec.Gap = tmp.Gap r.CanvasSpec.Variables = variables diff --git a/runtime/compilers/rillv1/parse_component.go b/runtime/compilers/rillv1/parse_component.go index 70652f6e65d..fc12cc8f857 100644 --- a/runtime/compilers/rillv1/parse_component.go +++ b/runtime/compilers/rillv1/parse_component.go @@ -62,6 +62,9 @@ func (p *Parser) parseComponent(node *Node) error { // NOTE: After calling insertResource, an error must not be returned. Any validation should be done before calling it. r.ComponentSpec = spec + if r.ComponentSpec.DisplayName == "" { + r.ComponentSpec.DisplayName = ToDisplayName(node.Name) + } return nil } diff --git a/runtime/compilers/rillv1/parse_explore.go b/runtime/compilers/rillv1/parse_explore.go index fa9fc7941df..93d5276fc8e 100644 --- a/runtime/compilers/rillv1/parse_explore.go +++ b/runtime/compilers/rillv1/parse_explore.go @@ -266,6 +266,9 @@ func (p *Parser) parseExplore(node *Node) error { // NOTE: After calling insertResource, an error must not be returned. Any validation should be done before calling it. r.ExploreSpec.DisplayName = tmp.DisplayName + if r.ExploreSpec.DisplayName == "" { + r.ExploreSpec.DisplayName = ToDisplayName(node.Name) + } r.ExploreSpec.Description = tmp.Description r.ExploreSpec.MetricsView = tmp.MetricsView r.ExploreSpec.Dimensions = dimensions diff --git a/runtime/compilers/rillv1/parse_metrics_view.go b/runtime/compilers/rillv1/parse_metrics_view.go index 7039beb4f03..8d671f8e9d7 100644 --- a/runtime/compilers/rillv1/parse_metrics_view.go +++ b/runtime/compilers/rillv1/parse_metrics_view.go @@ -530,6 +530,10 @@ func (p *Parser) parseMetricsView(node *Node) error { dim.DisplayName = dim.Label } + if dim.DisplayName == "" { + dim.DisplayName = ToDisplayName(dim.Name) + } + if (dim.Column == "" && dim.Expression == "") || (dim.Column != "" && dim.Expression != "") { return fmt.Errorf("exactly one of column or expression should be set for dimension: %q", dim.Name) } @@ -563,6 +567,10 @@ func (p *Parser) parseMetricsView(node *Node) error { measure.DisplayName = measure.Label } + if measure.DisplayName == "" { + measure.DisplayName = ToDisplayName(measure.Name) + } + lower := strings.ToLower(measure.Name) if _, ok := names[lower]; ok { return fmt.Errorf("found duplicate dimension or measure name %q", measure.Name) @@ -783,6 +791,9 @@ func (p *Parser) parseMetricsView(node *Node) error { spec.Table = tmp.Table spec.Model = tmp.Model spec.DisplayName = tmp.DisplayName + if spec.DisplayName == "" { + spec.DisplayName = ToDisplayName(node.Name) + } spec.Description = tmp.Description spec.TimeDimension = tmp.TimeDimension spec.WatermarkExpression = tmp.Watermark diff --git a/runtime/compilers/rillv1/parse_report.go b/runtime/compilers/rillv1/parse_report.go index b447f8ca390..d65b70a5d9a 100644 --- a/runtime/compilers/rillv1/parse_report.go +++ b/runtime/compilers/rillv1/parse_report.go @@ -184,6 +184,9 @@ func (p *Parser) parseReport(node *Node) error { // NOTE: After calling insertResource, an error must not be returned. Any validation should be done before calling it. r.ReportSpec.DisplayName = tmp.DisplayName + if r.ReportSpec.DisplayName == "" { + r.ReportSpec.DisplayName = ToDisplayName(node.Name) + } if schedule != nil { r.ReportSpec.RefreshSchedule = schedule } diff --git a/runtime/compilers/rillv1/parse_rillyaml.go b/runtime/compilers/rillv1/parse_rillyaml.go index bbf2c984b0a..bdcaf4a3549 100644 --- a/runtime/compilers/rillv1/parse_rillyaml.go +++ b/runtime/compilers/rillv1/parse_rillyaml.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "io" + "strings" "github.com/rilldata/rill/runtime/pkg/env" "gopkg.in/yaml.v3" @@ -40,6 +42,7 @@ type VariableDef struct { // rillYAML is the raw YAML structure of rill.yaml type rillYAML struct { + Compiler string `yaml:"compiler"` // Title of the project DisplayName string `yaml:"display_name"` // Title of the project @@ -82,6 +85,17 @@ type rillYAML struct { Features yaml.Node `yaml:"features"` // Paths to expose over HTTP (defaults to ./public) PublicPaths []string `yaml:"public_paths"` + // Paths to ignore when watching for changes. + // This is ignored in this parser because it's consumed directly by the repo driver. + IgnorePaths []string `yaml:"ignore_paths"` + // A list of mock users to test against dashboard security policies. + // This is ignored in this parser because it's consumed directly by the local application. + MockUsers []struct { + Email string `yaml:"email"` + Name string `yaml:"name"` + Admin bool `yaml:"admin"` + Groups []string `yaml:"groups"` + } `yaml:"mock_users"` } // parseRillYAML parses rill.yaml @@ -92,10 +106,19 @@ func (p *Parser) parseRillYAML(ctx context.Context, path string) error { } tmp := &rillYAML{} + if err := yaml.Unmarshal([]byte(data), tmp); err != nil { return newYAMLError(err) } + dec := yaml.NewDecoder(strings.NewReader(data)) + dec.KnownFields(true) + + err = dec.Decode(tmp) + if err != nil && !errors.Is(err, io.EOF) { + return newYAMLError(err) + } + // Look for environment-specific overrides for k, v := range tmp.Env { // nolint: gocritic // Using a pointer changes parser behavior if v.Kind == yaml.ScalarNode { diff --git a/runtime/compilers/rillv1/parser_test.go b/runtime/compilers/rillv1/parser_test.go index 47442823359..4feaaeb743b 100644 --- a/runtime/compilers/rillv1/parser_test.go +++ b/runtime/compilers/rillv1/parser_test.go @@ -276,14 +276,16 @@ schema: default Refs: []ResourceName{{Kind: ResourceKindModel, Name: "m2"}}, Paths: []string{"/metrics/d1.yaml"}, MetricsViewSpec: &runtimev1.MetricsViewSpec{ - Connector: "duckdb", - Model: "m2", + Connector: "duckdb", + Model: "m2", + DisplayName: "D1", Dimensions: []*runtimev1.MetricsViewSpec_DimensionV2{ - {Name: "a", Column: "a"}, + {Name: "a", DisplayName: "A", Column: "a"}, }, Measures: []*runtimev1.MetricsViewSpec_MeasureV2{ { Name: "b", + DisplayName: "B", Expression: "count(*)", Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE, FormatD3: "0,0", @@ -1069,13 +1071,14 @@ security: Name: ResourceName{Kind: ResourceKindMetricsView, Name: "mv1"}, Paths: []string{"/mv1.yaml"}, MetricsViewSpec: &runtimev1.MetricsViewSpec{ - Connector: "duckdb", - Table: "t1", + Connector: "duckdb", + Table: "t1", + DisplayName: "Mv1", Dimensions: []*runtimev1.MetricsViewSpec_DimensionV2{ - {Name: "a", Column: "a"}, + {Name: "a", DisplayName: "A", Column: "a"}, }, Measures: []*runtimev1.MetricsViewSpec_MeasureV2{ - {Name: "b", Expression: "count(*)", Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE}, + {Name: "b", DisplayName: "B", Expression: "count(*)", Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE}, }, FirstDayOfWeek: 7, SecurityRules: []*runtimev1.SecurityRule{ @@ -1091,13 +1094,14 @@ security: Name: ResourceName{Kind: ResourceKindMetricsView, Name: "mv2"}, Paths: []string{"/mv2.yaml"}, MetricsViewSpec: &runtimev1.MetricsViewSpec{ - Connector: "duckdb", - Table: "t2", + Connector: "duckdb", + Table: "t2", + DisplayName: "Mv2", Dimensions: []*runtimev1.MetricsViewSpec_DimensionV2{ - {Name: "a", Column: "a"}, + {Name: "a", DisplayName: "A", Column: "a"}, }, Measures: []*runtimev1.MetricsViewSpec_MeasureV2{ - {Name: "b", Expression: "count(*)", Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE}, + {Name: "b", DisplayName: "B", Expression: "count(*)", Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE}, }, FirstDayOfWeek: 1, SecurityRules: []*runtimev1.SecurityRule{ @@ -1209,13 +1213,14 @@ security: Name: ResourceName{Kind: ResourceKindMetricsView, Name: "d1"}, Paths: []string{"/metrics/d1.yaml"}, MetricsViewSpec: &runtimev1.MetricsViewSpec{ - Connector: "duckdb", - Table: "t1", + Connector: "duckdb", + Table: "t1", + DisplayName: "D1", Dimensions: []*runtimev1.MetricsViewSpec_DimensionV2{ - {Name: "a", Column: "a"}, + {Name: "a", DisplayName: "A", Column: "a"}, }, Measures: []*runtimev1.MetricsViewSpec_MeasureV2{ - {Name: "b", Expression: "count(*)", Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE}, + {Name: "b", DisplayName: "B", Expression: "count(*)", Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE}, }, SecurityRules: []*runtimev1.SecurityRule{ {Rule: &runtimev1.SecurityRule_Access{Access: &runtimev1.SecurityRuleAccess{ @@ -1497,13 +1502,14 @@ measures: Refs: nil, // NOTE: This is what we're testing – that it avoids inferring the missing "d1" as a self-reference Paths: []string{"/metrics/d1.yaml"}, MetricsViewSpec: &runtimev1.MetricsViewSpec{ - Connector: "duckdb", - Table: "d1", + Connector: "duckdb", + Table: "d1", + DisplayName: "D1", Dimensions: []*runtimev1.MetricsViewSpec_DimensionV2{ - {Name: "a", Column: "a"}, + {Name: "a", DisplayName: "A", Column: "a"}, }, Measures: []*runtimev1.MetricsViewSpec_MeasureV2{ - {Name: "b", Expression: "count(*)", Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE}, + {Name: "b", DisplayName: "B", Expression: "count(*)", Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE}, }, }, }, @@ -1568,6 +1574,7 @@ theme: Paths: []string{"/explores/e1.yaml"}, Refs: []ResourceName{{Kind: ResourceKindMetricsView, Name: "missing"}, {Kind: ResourceKindTheme, Name: "t1"}}, ExploreSpec: &runtimev1.ExploreSpec{ + DisplayName: "E1", MetricsView: "missing", DimensionsSelector: &runtimev1.FieldSelector{Selector: &runtimev1.FieldSelector_All{All: true}}, MeasuresSelector: &runtimev1.FieldSelector{Selector: &runtimev1.FieldSelector_All{All: true}}, @@ -1579,6 +1586,7 @@ theme: Paths: []string{"/explores/e2.yaml"}, Refs: []ResourceName{{Kind: ResourceKindMetricsView, Name: "missing"}}, ExploreSpec: &runtimev1.ExploreSpec{ + DisplayName: "E2", MetricsView: "missing", DimensionsSelector: &runtimev1.FieldSelector{Selector: &runtimev1.FieldSelector_All{All: true}}, MeasuresSelector: &runtimev1.FieldSelector{Selector: &runtimev1.FieldSelector_All{All: true}}, @@ -1664,6 +1672,7 @@ items: Paths: []string{"/components/c1.yaml"}, Refs: []ResourceName{{Kind: ResourceKindAPI, Name: "MetricsViewAggregation"}}, ComponentSpec: &runtimev1.ComponentSpec{ + DisplayName: "C1", Resolver: "api", ResolverProperties: must(structpb.NewStruct(map[string]any{"api": "MetricsViewAggregation", "args": map[string]any{"metrics_view": "foo"}})), Renderer: "vega_lite", @@ -1675,6 +1684,7 @@ items: Paths: []string{"/components/c2.yaml"}, Refs: []ResourceName{{Kind: ResourceKindAPI, Name: "MetricsViewAggregation"}}, ComponentSpec: &runtimev1.ComponentSpec{ + DisplayName: "C2", Resolver: "api", ResolverProperties: must(structpb.NewStruct(map[string]any{"api": "MetricsViewAggregation", "args": map[string]any{"metrics_view": "bar"}})), Renderer: "vega_lite", @@ -1685,6 +1695,7 @@ items: Name: ResourceName{Kind: ResourceKindComponent, Name: "c3"}, Paths: []string{"/components/c3.yaml"}, ComponentSpec: &runtimev1.ComponentSpec{ + DisplayName: "C3", Resolver: "metrics_sql", ResolverProperties: must(structpb.NewStruct(map[string]any{"sql": "SELECT 1"})), Renderer: "line_chart", @@ -1709,8 +1720,9 @@ items: {Kind: ResourceKindComponent, Name: "d1--component-2"}, }, CanvasSpec: &runtimev1.CanvasSpec{ - Columns: 4, - Gap: 3, + DisplayName: "D1", + Columns: 4, + Gap: 3, Items: []*runtimev1.CanvasItem{ {Component: "c1"}, {Component: "c2", Width: asPtr(uint32(1)), Height: asPtr(uint32(2))}, @@ -1901,33 +1913,38 @@ measures: MetricsViewSpec: &runtimev1.MetricsViewSpec{ Connector: "duckdb", Table: "t1", + DisplayName: "D1", TimeDimension: "t", Dimensions: []*runtimev1.MetricsViewSpec_DimensionV2{ - {Name: "foo", Column: "foo"}, + {Name: "foo", DisplayName: "Foo", Column: "foo"}, }, Measures: []*runtimev1.MetricsViewSpec_MeasureV2{ { - Name: "a", - Expression: "count(*)", - Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE, + Name: "a", + DisplayName: "A", + Expression: "count(*)", + Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE, }, { Name: "b", + DisplayName: "B", Expression: "a+1", Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_DERIVED, ReferencedMeasures: []string{"a"}, }, { Name: "c", + DisplayName: "C", Expression: "sum(a)", Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_DERIVED, PerDimensions: []*runtimev1.MetricsViewSpec_DimensionSelector{{Name: "foo"}}, ReferencedMeasures: []string{"a"}, }, { - Name: "d", - Expression: "a/lag(a)", - Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_DERIVED, + Name: "d", + DisplayName: "D", + Expression: "a/lag(a)", + Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_DERIVED, Window: &runtimev1.MetricsViewSpec_MeasureWindow{ Partition: true, OrderBy: []*runtimev1.MetricsViewSpec_DimensionSelector{{Name: "t", TimeGrain: runtimev1.TimeGrain_TIME_GRAIN_DAY}}, diff --git a/runtime/compilers/rillv1/util.go b/runtime/compilers/rillv1/util.go new file mode 100644 index 00000000000..150c56056cc --- /dev/null +++ b/runtime/compilers/rillv1/util.go @@ -0,0 +1,25 @@ +package rillv1 + +import ( + "strings" + + "golang.org/x/text/cases" + "golang.org/x/text/language" +) + +// toDisplayName converts a snake_case name to a display name by replacing underscores and dashes with spaces and capitalizing every word. +func ToDisplayName(name string) string { + // Don't transform names that start with an underscore (since it's probably internal). + if name != "" && name[0] == '_' { + return name + } + + // Replace underscores and dashes with spaces. + name = strings.ReplaceAll(name, "_", " ") + name = strings.ReplaceAll(name, "-", " ") + + // Capitalize the first letter. + name = cases.Title(language.English).String(name) + + return name +} diff --git a/runtime/compilers/rillv1/util_test.go b/runtime/compilers/rillv1/util_test.go new file mode 100644 index 00000000000..3a52c3ec0b7 --- /dev/null +++ b/runtime/compilers/rillv1/util_test.go @@ -0,0 +1,27 @@ +package rillv1 + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestToDisplayName(t *testing.T) { + tests := []struct { + name string + expected string + }{ + {"", ""}, + {"foo", "Foo"}, + {"foo_bar", "Foo Bar"}, + {"foo-bar", "Foo Bar"}, + {"_foo", "_foo"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + actual := ToDisplayName(test.name) + require.Equal(t, test.expected, actual) + }) + } +} diff --git a/runtime/connections_test.go b/runtime/connections_test.go index 01e704957af..23a5de4f2b9 100644 --- a/runtime/connections_test.go +++ b/runtime/connections_test.go @@ -23,7 +23,7 @@ connectors: AWS_ACCESS_KEY_ID: us-east-1 aws_secret_access_key: xxxx -variables: +vars: foo: bar allow_host_access: false `, diff --git a/runtime/controller_test.go b/runtime/controller_test.go index 8d1c6a91ef9..1fe8c72968e 100644 --- a/runtime/controller_test.go +++ b/runtime/controller_test.go @@ -9,7 +9,7 @@ import ( runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" "github.com/rilldata/rill/runtime" - "github.com/rilldata/rill/runtime/drivers" + "github.com/rilldata/rill/runtime/compilers/rillv1" "github.com/rilldata/rill/runtime/testruntime" "github.com/stretchr/testify/require" "google.golang.org/protobuf/types/known/structpb" @@ -101,10 +101,11 @@ measures: // Verify the metrics view mvSpec := &runtimev1.MetricsViewSpec{ - Connector: "duckdb", - Model: "bar", - Dimensions: []*runtimev1.MetricsViewSpec_DimensionV2{{Name: "a", Column: "a"}}, - Measures: []*runtimev1.MetricsViewSpec_MeasureV2{{Name: "b", Expression: "count(*)", Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE}}, + Connector: "duckdb", + Model: "bar", + Dimensions: []*runtimev1.MetricsViewSpec_DimensionV2{{Name: "a", DisplayName: "A", Column: "a"}}, + Measures: []*runtimev1.MetricsViewSpec_MeasureV2{{Name: "b", DisplayName: "B", Expression: "count(*)", Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE}}, + DisplayName: "Foobar", } testruntime.RequireResource(t, rt, id, &runtimev1.Resource{ Meta: &runtimev1.ResourceMeta{ @@ -118,11 +119,12 @@ measures: Spec: mvSpec, State: &runtimev1.MetricsViewState{ ValidSpec: &runtimev1.MetricsViewSpec{ - Connector: "duckdb", - Model: "bar", - Table: "bar", - Dimensions: []*runtimev1.MetricsViewSpec_DimensionV2{{Name: "a", Column: "a"}}, - Measures: []*runtimev1.MetricsViewSpec_MeasureV2{{Name: "b", Expression: "count(*)", Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE}}, + Connector: "duckdb", + Table: "bar", + Model: "bar", + DisplayName: "Foobar", + Dimensions: []*runtimev1.MetricsViewSpec_DimensionV2{{Name: "a", DisplayName: "A", Column: "a"}}, + Measures: []*runtimev1.MetricsViewSpec_MeasureV2{{Name: "b", DisplayName: "B", Expression: "count(*)", Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE}}, }, }, }, @@ -243,7 +245,7 @@ path: data/foo.csv // Delete the underlying table olap, release, err := rt.OLAP(context.Background(), id, "") require.NoError(t, err) - err = olap.Exec(context.Background(), &drivers.Statement{Query: "DROP TABLE foo;"}) + err = olap.DropTable(context.Background(), "foo") require.NoError(t, err) release() testruntime.RequireNoOLAPTable(t, rt, id, "foo") @@ -489,7 +491,8 @@ select 1 testruntime.ReconcileParserAndWait(t, rt, id) testruntime.RequireReconcileState(t, rt, id, 2, 0, 0) // Assert that the model is a table now - testruntime.RequireIsView(t, olap, "bar", false) + // TODO : fix with information schema fix + // testruntime.RequireIsView(t, olap, "bar", false) // Mark the model as not materialized testruntime.PutFiles(t, rt, id, map[string]string{ @@ -504,51 +507,6 @@ select 1 testruntime.RequireIsView(t, olap, "bar", true) } -func TestModelCTE(t *testing.T) { - // Create a model that references a source - rt, id := testruntime.NewInstance(t) - testruntime.PutFiles(t, rt, id, map[string]string{ - "/data/foo.csv": `a,b,c,d,e -1,2,3,4,5 -1,2,3,4,5 -1,2,3,4,5 -`, - "/sources/foo.yaml": ` -connector: local_file -path: data/foo.csv -`, - "/models/bar.sql": `SELECT * FROM foo`, - }) - testruntime.ReconcileParserAndWait(t, rt, id) - testruntime.RequireReconcileState(t, rt, id, 3, 0, 0) - model, modelRes := newModel("SELECT * FROM foo", "bar", "foo") - testruntime.RequireResource(t, rt, id, modelRes) - testruntime.RequireOLAPTable(t, rt, id, "bar") - - // Update model to have a CTE with alias different from the source - testruntime.PutFiles(t, rt, id, map[string]string{ - "/models/bar.sql": `with CTEAlias as (select * from foo) select * from CTEAlias`, - }) - testruntime.ReconcileParserAndWait(t, rt, id) - testruntime.RequireReconcileState(t, rt, id, 3, 0, 0) - model.Spec.InputProperties = must(structpb.NewStruct(map[string]any{"sql": `with CTEAlias as (select * from foo) select * from CTEAlias`})) - testruntime.RequireResource(t, rt, id, modelRes) - testruntime.RequireOLAPTable(t, rt, id, "bar") - - // Update model to have a CTE with alias same as the source - testruntime.PutFiles(t, rt, id, map[string]string{ - "/models/bar.sql": `with foo as (select * from memory.foo) select * from foo`, - }) - testruntime.ReconcileParserAndWait(t, rt, id) - testruntime.RequireReconcileState(t, rt, id, 3, 0, 0) - model.Spec.InputProperties = must(structpb.NewStruct(map[string]any{"sql": `with foo as (select * from memory.foo) select * from foo`})) - modelRes.Meta.Refs = []*runtimev1.ResourceName{} - testruntime.RequireResource(t, rt, id, modelRes) - // Refs are removed but the model is valid. - // TODO: is this expected? - testruntime.RequireOLAPTable(t, rt, id, "bar") -} - func TestRename(t *testing.T) { // Rename model A to B and model B to A, verify success // Rename model A to B and source B to A, verify success @@ -696,7 +654,7 @@ path: data/foo.csv "/metrics/dash.yaml": ` version: 1 type: metrics_view -display_name: dash +display_name: Dash model: bar3 dimensions: - column: b @@ -847,7 +805,7 @@ path: data/foo.csv "/metrics/dash.yaml": ` version: 1 type: metrics_view -display_name: dash +display_name: Dash model: bar dimensions: - column: b @@ -868,7 +826,7 @@ measures: "/metrics/dash.yaml": ` version: 1 type: metrics_view -display_name: dash +display_name: Dash model: bar dimensions: - column: b @@ -890,7 +848,7 @@ measures: "/metrics/dash.yaml": ` version: 1 type: metrics_view -display_name: dash +display_name: Dash model: bar dimensions: - column: b @@ -912,7 +870,7 @@ measures: "/metrics/dash.yaml": ` version: 1 type: metrics_view -display_name: dash +display_name: Dash model: bar dimensions: - column: b @@ -935,7 +893,7 @@ measures: "/metrics/dash.yaml": ` version: 1 type: metrics_view -display_name: dash +display_name: Dash model: bar dimensions: - column: b @@ -956,7 +914,7 @@ measures: "/metrics/dash.yaml": ` version: 1 type: metrics_view -display_name: dash +display_name: Dash model: bar dimensions: - column: b @@ -977,7 +935,7 @@ measures: "/metrics/dash.yaml": ` version: 1 type: metrics_view -display_name: dash +display_name: Dash model: bar dimensions: - column: b @@ -998,7 +956,7 @@ measures: "/metrics/dash.yaml": ` version: 1 type: metrics_view -display_name: dash +display_name: Dash model: bar dimensions: - column: b @@ -1044,7 +1002,7 @@ path: data/foo.csv "/metrics/dash.yaml": ` version: 1 type: metrics_view -display_name: dash +display_name: Dash model: bar dimensions: - column: b @@ -1136,7 +1094,7 @@ path: data/foo.csv "/metrics/dash.yaml": ` version: 1 type: metrics_view -display_name: dash +display_name: Dash model: bar dimensions: - column: b @@ -1272,7 +1230,7 @@ path: data/foo.csv "/metrics/dash.yaml": ` version: 1 type: metrics_view -display_name: dash +display_name: Dash model: bar dimensions: - column: country @@ -1447,7 +1405,7 @@ func newMetricsView(name, model string, measures, dimensions []string) (*runtime Spec: &runtimev1.MetricsViewSpec{ Connector: "duckdb", Model: model, - DisplayName: name, + DisplayName: rillv1.ToDisplayName(name), Measures: make([]*runtimev1.MetricsViewSpec_MeasureV2, len(measures)), Dimensions: make([]*runtimev1.MetricsViewSpec_DimensionV2, len(dimensions)), }, @@ -1456,7 +1414,7 @@ func newMetricsView(name, model string, measures, dimensions []string) (*runtime Connector: "duckdb", Table: model, Model: model, - DisplayName: name, + DisplayName: rillv1.ToDisplayName(name), Measures: make([]*runtimev1.MetricsViewSpec_MeasureV2, len(measures)), Dimensions: make([]*runtimev1.MetricsViewSpec_DimensionV2, len(dimensions)), }, @@ -1464,25 +1422,30 @@ func newMetricsView(name, model string, measures, dimensions []string) (*runtime } for i, measure := range measures { + name := fmt.Sprintf("measure_%d", i) metrics.Spec.Measures[i] = &runtimev1.MetricsViewSpec_MeasureV2{ - Name: fmt.Sprintf("measure_%d", i), - Expression: measure, - Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE, + Name: name, + DisplayName: rillv1.ToDisplayName(name), + Expression: measure, + Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE, } metrics.State.ValidSpec.Measures[i] = &runtimev1.MetricsViewSpec_MeasureV2{ - Name: fmt.Sprintf("measure_%d", i), - Expression: measure, - Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE, + Name: name, + DisplayName: rillv1.ToDisplayName(name), + Expression: measure, + Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE, } } for i, dimension := range dimensions { metrics.Spec.Dimensions[i] = &runtimev1.MetricsViewSpec_DimensionV2{ - Name: dimension, - Column: dimension, + Name: dimension, + DisplayName: rillv1.ToDisplayName(dimension), + Column: dimension, } metrics.State.ValidSpec.Dimensions[i] = &runtimev1.MetricsViewSpec_DimensionV2{ - Name: dimension, - Column: dimension, + Name: dimension, + DisplayName: rillv1.ToDisplayName(dimension), + Column: dimension, } } metricsRes := &runtimev1.Resource{ diff --git a/runtime/drivers/admin/admin.go b/runtime/drivers/admin/admin.go index be694cf7fd1..27c1ecd643c 100644 --- a/runtime/drivers/admin/admin.go +++ b/runtime/drivers/admin/admin.go @@ -237,11 +237,6 @@ func (h *Handle) AsTransporter(from, to drivers.Handle) (drivers.Transporter, bo return nil, false } -// AsSQLStore implements drivers.Handle. -func (h *Handle) AsSQLStore() (drivers.SQLStore, bool) { - return nil, false -} - // AsNotifier implements drivers.Handle. func (h *Handle) AsNotifier(properties map[string]any) (drivers.Notifier, error) { return nil, drivers.ErrNotNotifier diff --git a/runtime/drivers/athena/athena.go b/runtime/drivers/athena/athena.go index 959f3fa72dd..1bc9cbc7180 100644 --- a/runtime/drivers/athena/athena.go +++ b/runtime/drivers/athena/athena.go @@ -217,11 +217,6 @@ func (c *Connection) AsWarehouse() (drivers.Warehouse, bool) { return c, true } -// AsSQLStore implements drivers.Connection. -func (c *Connection) AsSQLStore() (drivers.SQLStore, bool) { - return nil, false -} - // AsNotifier implements drivers.Handle. func (c *Connection) AsNotifier(properties map[string]any) (drivers.Notifier, error) { return nil, drivers.ErrNotNotifier diff --git a/runtime/drivers/azure/azure.go b/runtime/drivers/azure/azure.go index e561d57ed36..372180fb794 100644 --- a/runtime/drivers/azure/azure.go +++ b/runtime/drivers/azure/azure.go @@ -228,11 +228,6 @@ func (c *Connection) AsWarehouse() (drivers.Warehouse, bool) { return nil, false } -// AsSQLStore implements drivers.Connection. -func (c *Connection) AsSQLStore() (drivers.SQLStore, bool) { - return nil, false -} - // AsNotifier implements drivers.Connection. func (c *Connection) AsNotifier(properties map[string]any) (drivers.Notifier, error) { return nil, drivers.ErrNotNotifier diff --git a/runtime/drivers/bigquery/bigquery.go b/runtime/drivers/bigquery/bigquery.go index 0bb4bad68e6..2d47751d1d0 100644 --- a/runtime/drivers/bigquery/bigquery.go +++ b/runtime/drivers/bigquery/bigquery.go @@ -184,11 +184,6 @@ func (c *Connection) AsObjectStore() (drivers.ObjectStore, bool) { return nil, false } -// AsSQLStore implements drivers.Connection. -func (c *Connection) AsSQLStore() (drivers.SQLStore, bool) { - return nil, false -} - // AsModelExecutor implements drivers.Handle. func (c *Connection) AsModelExecutor(instanceID string, opts *drivers.ModelExecutorOptions) (drivers.ModelExecutor, bool) { return nil, false diff --git a/runtime/drivers/clickhouse/clickhouse.go b/runtime/drivers/clickhouse/clickhouse.go index 7ce672e59e9..e844cce89ee 100644 --- a/runtime/drivers/clickhouse/clickhouse.go +++ b/runtime/drivers/clickhouse/clickhouse.go @@ -379,12 +379,6 @@ func (c *connection) AsWarehouse() (drivers.Warehouse, bool) { return nil, false } -// AsSQLStore implements drivers.Connection. -// Use OLAPStore instead. -func (c *connection) AsSQLStore() (drivers.SQLStore, bool) { - return nil, false -} - // AsNotifier implements drivers.Connection. func (c *connection) AsNotifier(properties map[string]any) (drivers.Notifier, error) { return nil, drivers.ErrNotNotifier diff --git a/runtime/drivers/clickhouse/context.go b/runtime/drivers/clickhouse/context.go index b50f72854d8..c3fa5d8678d 100644 --- a/runtime/drivers/clickhouse/context.go +++ b/runtime/drivers/clickhouse/context.go @@ -31,7 +31,12 @@ func connFromContext(ctx context.Context) *sqlx.Conn { // This is used to use certain session aware features like temporary tables. func (c *connection) sessionAwareContext(ctx context.Context) context.Context { if c.opts.Protocol == clickhouse.HTTP { - settings := maps.Clone(c.opts.Settings) + var settings map[string]any + if len(c.opts.Settings) == 0 { + settings = make(map[string]any) + } else { + settings = maps.Clone(c.opts.Settings) + } settings["session_id"] = uuid.New().String() return clickhouse.Context(ctx, clickhouse.WithSettings(settings)) } diff --git a/runtime/drivers/clickhouse/model_executor_localfile_self.go b/runtime/drivers/clickhouse/model_executor_localfile_self.go index 694e29316c8..92dd1a20ca1 100644 --- a/runtime/drivers/clickhouse/model_executor_localfile_self.go +++ b/runtime/drivers/clickhouse/model_executor_localfile_self.go @@ -103,14 +103,12 @@ func (e *localFileToSelfExecutor) Execute(ctx context.Context, opts *drivers.Mod if opts.Env.StageChanges || outputProps.Typ == "DICTIONARY" { stagingTableName = stagingTableNameFor(tableName) } - if t, err := e.c.InformationSchema().Lookup(ctx, "", "", stagingTableName); err == nil { - _ = e.c.DropTable(ctx, stagingTableName, t.View) - } + _ = e.c.DropTable(ctx, stagingTableName) // create the table err = e.c.createTable(ctx, stagingTableName, "", outputProps) if err != nil { - _ = e.c.DropTable(ctx, stagingTableName, false) + _ = e.c.DropTable(ctx, stagingTableName) return nil, fmt.Errorf("failed to create model: %w", err) } @@ -131,7 +129,7 @@ func (e *localFileToSelfExecutor) Execute(ctx context.Context, opts *drivers.Mod if outputProps.Typ == "DICTIONARY" { err = e.c.createDictionary(ctx, tableName, fmt.Sprintf("SELECT * FROM %s", safeSQLName(stagingTableName)), outputProps) // drop the temp table - _ = e.c.DropTable(ctx, stagingTableName, false) + _ = e.c.DropTable(ctx, stagingTableName) if err != nil { return nil, fmt.Errorf("failed to create dictionary: %w", err) } diff --git a/runtime/drivers/clickhouse/model_executor_self.go b/runtime/drivers/clickhouse/model_executor_self.go index b8a106d12b0..daf7116c545 100644 --- a/runtime/drivers/clickhouse/model_executor_self.go +++ b/runtime/drivers/clickhouse/model_executor_self.go @@ -65,14 +65,16 @@ func (e *selfToSelfExecutor) Execute(ctx context.Context, opts *drivers.ModelExe // Drop the staging view/table if it exists. // NOTE: This intentionally drops the end table if not staging changes. - if t, err := e.c.InformationSchema().Lookup(ctx, "", "", stagingTableName); err == nil { - _ = e.c.DropTable(ctx, stagingTableName, t.View) - } + _ = e.c.DropTable(ctx, stagingTableName) // Create the table - err := e.c.CreateTableAsSelect(ctx, stagingTableName, asView, inputProps.SQL, mustToMap(outputProps)) + opts := &drivers.CreateTableOptions{ + View: asView, + TableOpts: mustToMap(outputProps), + } + err := e.c.CreateTableAsSelect(ctx, stagingTableName, inputProps.SQL, opts) if err != nil { - _ = e.c.DropTable(ctx, stagingTableName, asView) + _ = e.c.DropTable(ctx, stagingTableName) return nil, fmt.Errorf("failed to create model: %w", err) } @@ -85,7 +87,13 @@ func (e *selfToSelfExecutor) Execute(ctx context.Context, opts *drivers.ModelExe } } else { // Insert into the table - err := e.c.InsertTableAsSelect(ctx, tableName, inputProps.SQL, false, true, outputProps.IncrementalStrategy, outputProps.UniqueKey) + opts := &drivers.InsertTableOptions{ + ByName: false, + InPlace: true, + Strategy: outputProps.IncrementalStrategy, + UniqueKey: outputProps.UniqueKey, + } + err := e.c.InsertTableAsSelect(ctx, tableName, inputProps.SQL, opts) if err != nil { return nil, fmt.Errorf("failed to incrementally insert into table: %w", err) } diff --git a/runtime/drivers/clickhouse/model_manager.go b/runtime/drivers/clickhouse/model_manager.go index dcb3e5f7d16..41533abbe28 100644 --- a/runtime/drivers/clickhouse/model_manager.go +++ b/runtime/drivers/clickhouse/model_manager.go @@ -88,11 +88,23 @@ func (p *ModelOutputProperties) Validate(opts *drivers.ModelExecuteOptions) erro } switch p.IncrementalStrategy { - case drivers.IncrementalStrategyUnspecified, drivers.IncrementalStrategyAppend: + case drivers.IncrementalStrategyUnspecified, drivers.IncrementalStrategyAppend, drivers.IncrementalStrategyPartitionOverwrite, drivers.IncrementalStrategyMerge: default: return fmt.Errorf("invalid incremental strategy %q", p.IncrementalStrategy) } + // if incremntal strategy is partition overwrite, partition_by is required + if p.IncrementalStrategy == drivers.IncrementalStrategyPartitionOverwrite && p.PartitionBy == "" { + return fmt.Errorf(`must specify a "partition_by" when "incremental_strategy" is %q`, p.IncrementalStrategy) + } + + // ClickHouse enforces the requirement of either a primary key or an ORDER BY clause for the ReplacingMergeTree engine. + // When using the incremental strategy as 'merge', the engine must be ReplacingMergeTree. + // This ensures that duplicate rows are eventually replaced, maintaining data consistency. + if p.IncrementalStrategy == drivers.IncrementalStrategyMerge && !(strings.Contains(p.Engine, "ReplacingMergeTree") || strings.Contains(p.EngineFull, "ReplacingMergeTree")) { + return fmt.Errorf(`must use "ReplacingMergeTree" engine when "incremental_strategy" is %q`, p.IncrementalStrategy) + } + if p.IncrementalStrategy == drivers.IncrementalStrategyUnspecified { p.IncrementalStrategy = drivers.IncrementalStrategyAppend } @@ -192,7 +204,7 @@ func (c *connection) Exists(ctx context.Context, res *drivers.ModelResult) (bool return false, fmt.Errorf("connector is not an OLAP") } - _, err := olap.InformationSchema().Lookup(ctx, "", "", res.Table) + _, err := olap.InformationSchema().Lookup(ctx, c.config.Database, "", res.Table) return err == nil, nil } @@ -202,17 +214,14 @@ func (c *connection) Delete(ctx context.Context, res *drivers.ModelResult) error return fmt.Errorf("connector is not an OLAP") } - stagingTable, err := olap.InformationSchema().Lookup(ctx, "", "", stagingTableNameFor(res.Table)) - if err == nil { - _ = c.DropTable(ctx, stagingTable.Name, stagingTable.View) - } + _ = c.DropTable(ctx, stagingTableNameFor(res.Table)) - table, err := olap.InformationSchema().Lookup(ctx, "", "", res.Table) + table, err := olap.InformationSchema().Lookup(ctx, c.config.Database, "", res.Table) if err != nil { return err } - return c.DropTable(ctx, table.Name, table.View) + return c.DropTable(ctx, table.Name) } func (c *connection) MergePartitionResults(a, b *drivers.ModelResult) (*drivers.ModelResult, error) { @@ -250,7 +259,7 @@ func olapForceRenameTable(ctx context.Context, c *connection, fromName string, f // Renaming a table to the same name with different casing is not supported. Workaround by renaming to a temporary name first. if strings.EqualFold(fromName, toName) { tmpName := fmt.Sprintf("__rill_tmp_rename_%s_%s", typ, toName) - err := c.RenameTable(ctx, fromName, tmpName, fromIsView) + err := c.RenameTable(ctx, fromName, tmpName) if err != nil { return err } @@ -258,7 +267,7 @@ func olapForceRenameTable(ctx context.Context, c *connection, fromName string, f } // Do the rename - return c.RenameTable(ctx, fromName, toName, fromIsView) + return c.RenameTable(ctx, fromName, toName) } func boolPtr(b bool) *bool { diff --git a/runtime/drivers/clickhouse/olap.go b/runtime/drivers/clickhouse/olap.go index 55fb2fbac66..7bf26106d24 100644 --- a/runtime/drivers/clickhouse/olap.go +++ b/runtime/drivers/clickhouse/olap.go @@ -2,6 +2,7 @@ package clickhouse import ( "context" + "crypto/md5" "errors" "fmt" "strings" @@ -34,7 +35,7 @@ func (c *connection) Dialect() drivers.Dialect { return drivers.DialectClickHouse } -func (c *connection) WithConnection(ctx context.Context, priority int, longRunning, tx bool, fn drivers.WithConnectionFunc) error { +func (c *connection) WithConnection(ctx context.Context, priority int, longRunning bool, fn drivers.WithConnectionFunc) error { // Check not nested if connFromContext(ctx) != nil { panic("nested WithConnection") @@ -208,9 +209,9 @@ func (c *connection) AlterTableColumn(ctx context.Context, tableName, columnName } // CreateTableAsSelect implements drivers.OLAPStore. -func (c *connection) CreateTableAsSelect(ctx context.Context, name string, view bool, sql string, tableOpts map[string]any) error { +func (c *connection) CreateTableAsSelect(ctx context.Context, name, sql string, opts *drivers.CreateTableOptions) error { outputProps := &ModelOutputProperties{} - if err := mapstructure.WeakDecode(tableOpts, outputProps); err != nil { + if err := mapstructure.WeakDecode(opts.TableOpts, outputProps); err != nil { return fmt.Errorf("failed to parse output properties: %w", err) } var onClusterClause string @@ -239,11 +240,11 @@ func (c *connection) CreateTableAsSelect(ctx context.Context, name string, view } // InsertTableAsSelect implements drivers.OLAPStore. -func (c *connection) InsertTableAsSelect(ctx context.Context, name, sql string, byName, inPlace bool, strategy drivers.IncrementalStrategy, uniqueKey []string) error { - if !inPlace { +func (c *connection) InsertTableAsSelect(ctx context.Context, name, sql string, opts *drivers.InsertTableOptions) error { + if !opts.InPlace { return fmt.Errorf("clickhouse: inserts does not support inPlace=false") } - if strategy == drivers.IncrementalStrategyAppend { + if opts.Strategy == drivers.IncrementalStrategyAppend { return c.Exec(ctx, &drivers.Statement{ Query: fmt.Sprintf("INSERT INTO %s %s", safeSQLName(name), sql), Priority: 1, @@ -251,13 +252,113 @@ func (c *connection) InsertTableAsSelect(ctx context.Context, name, sql string, }) } - // merge strategy is also not supported for clickhouse - return fmt.Errorf("incremental insert strategy %q not supported", strategy) + if opts.Strategy == drivers.IncrementalStrategyPartitionOverwrite { + _, onCluster, err := informationSchema{c: c}.entityType(ctx, c.config.Database, name) + if err != nil { + return err + } + onClusterClause := "" + if onCluster { + onClusterClause = "ON CLUSTER " + safeSQLName(c.config.Cluster) + } + // Get the engine info of the given table + engine, err := c.getTableEngine(ctx, name) + if err != nil { + return err + } + // Distributed table cannot be altered directly, so we need to alter the local table + if engine == "Distributed" { + name = localTableName(name) + } + // create temp table with the same schema using a deterministic name + tempName := fmt.Sprintf("__rill_temp_%s_%x", name, md5.Sum([]byte(sql))) + err = c.Exec(ctx, &drivers.Statement{ + Query: fmt.Sprintf("CREATE OR REPLACE TABLE %s %s AS %s", safeSQLName(tempName), onClusterClause, name), + Priority: 1, + }) + if err != nil { + return err + } + // clean up the temp table + defer func() { + var cancel context.CancelFunc + + // If the original context is cancelled, create a new context for cleanup + if ctx.Err() != nil { + ctx, cancel = context.WithTimeout(context.Background(), 15*time.Second) + } else { + cancel = func() {} + } + defer cancel() + + err = c.Exec(ctx, &drivers.Statement{ + Query: fmt.Sprintf("DROP TABLE %s %s", safeSQLName(tempName), onClusterClause), + Priority: 1, + }) + if err != nil { + c.logger.Warn("clickhouse: failed to drop temp table", zap.String("name", tempName), zap.Error(err)) + } + }() + // insert into temp table + err = c.Exec(ctx, &drivers.Statement{ + Query: fmt.Sprintf("INSERT INTO %s %s", safeSQLName(tempName), sql), + Priority: 1, + LongRunning: true, + }) + if err != nil { + return err + } + // list partitions from the temp table + partitions, err := c.getTablePartitions(ctx, tempName) + if err != nil { + return err + } + // iterate over partitions and replace them in the main table + for _, part := range partitions { + // alter the main table to replace the partition + err = c.Exec(ctx, &drivers.Statement{ + Query: fmt.Sprintf("ALTER TABLE %s %s REPLACE PARTITION ? FROM %s", safeSQLName(name), onClusterClause, safeSQLName(tempName)), + Args: []any{part}, + Priority: 1, + }) + if err != nil { + return err + } + } + return nil + } + + if opts.Strategy == drivers.IncrementalStrategyMerge { + _, onCluster, err := informationSchema{c: c}.entityType(ctx, c.config.Database, name) + if err != nil { + return err + } + onClusterClause := "" + if onCluster { + onClusterClause = "ON CLUSTER " + safeSQLName(c.config.Cluster) + } + // get the engine info of the given table + engine, err := c.getTableEngine(ctx, name) + if err != nil { + return err + } + if !strings.Contains(engine, "ReplacingMergeTree") { + return fmt.Errorf("clickhouse: merge strategy requires ReplacingMergeTree engine") + } + + // insert into table using the merge strategy + return c.Exec(ctx, &drivers.Statement{ + Query: fmt.Sprintf("INSERT INTO %s %s %s", safeSQLName(name), onClusterClause, sql), + Priority: 1, + LongRunning: true, + }) + } + return fmt.Errorf("incremental insert strategy %q not supported", opts.Strategy) } // DropTable implements drivers.OLAPStore. -func (c *connection) DropTable(ctx context.Context, name string, _ bool) error { - typ, onCluster, err := informationSchema{c: c}.entityType(ctx, "", name) +func (c *connection) DropTable(ctx context.Context, name string) error { + typ, onCluster, err := informationSchema{c: c}.entityType(ctx, c.config.Database, name) if err != nil { return err } @@ -310,8 +411,8 @@ func (c *connection) MayBeScaledToZero(ctx context.Context) bool { } // RenameTable implements drivers.OLAPStore. -func (c *connection) RenameTable(ctx context.Context, oldName, newName string, view bool) error { - typ, onCluster, err := informationSchema{c: c}.entityType(ctx, "", oldName) +func (c *connection) RenameTable(ctx context.Context, oldName, newName string) error { + typ, onCluster, err := informationSchema{c: c}.entityType(ctx, c.config.Database, oldName) if err != nil { return err } @@ -330,10 +431,14 @@ func (c *connection) RenameTable(ctx context.Context, oldName, newName string, v return c.renameTable(ctx, oldName, newName, onClusterClause) } // capture the full engine of the old distributed table + args := []any{c.config.Database, oldName} + if c.config.Database == "" { + args = []any{nil, oldName} + } var engineFull string res, err := c.Execute(ctx, &drivers.Statement{ - Query: "SELECT engine_full FROM system.tables WHERE database = currentDatabase() AND name = ?", - Args: []any{oldName}, + Query: "SELECT engine_full FROM system.tables WHERE database = coalesce(?, currentDatabase()) AND name = ?", + Args: args, Priority: 100, }) if err != nil { @@ -382,9 +487,13 @@ func (c *connection) RenameTable(ctx context.Context, oldName, newName string, v func (c *connection) renameView(ctx context.Context, oldName, newName, onCluster string) error { // clickhouse does not support renaming views so we capture the OLD view's select statement and use it to create new view + args := []any{c.config.Database, oldName} + if c.config.Database == "" { + args = []any{nil, oldName} + } res, err := c.Execute(ctx, &drivers.Statement{ - Query: "SELECT as_select FROM system.tables WHERE database = currentDatabase() AND name = ?", - Args: []any{oldName}, + Query: "SELECT as_select FROM system.tables WHERE database = coalesce(?, currentDatabase()) AND name = ?", + Args: args, Priority: 100, }) if err != nil { @@ -440,7 +549,7 @@ func (c *connection) renameTable(ctx context.Context, oldName, newName, onCluste return err } // drop the old table - return c.DropTable(context.Background(), oldName, false) + return c.DropTable(context.Background(), oldName) } func (c *connection) createTable(ctx context.Context, name, sql string, outputProps *ModelOutputProperties) error { @@ -477,7 +586,15 @@ func (c *connection) createTable(ctx context.Context, name, sql string, outputPr } else { fmt.Fprintf(&create, " %s ", outputProps.Columns) } - create.WriteString(outputProps.tblConfig()) + + tableConfig := outputProps.tblConfig() + create.WriteString(tableConfig) + + // validate incremental strategy + if outputProps.IncrementalStrategy == drivers.IncrementalStrategyPartitionOverwrite && + !strings.Contains(strings.ToUpper(tableConfig), "PARTITION BY") { + return fmt.Errorf("clickhouse: incremental strategy partition_overwrite requires a partition key") + } // create table err := c.Exec(ctx, &drivers.Statement{Query: create.String(), Priority: 100}) @@ -490,8 +607,12 @@ func (c *connection) createTable(ctx context.Context, name, sql string, outputPr } // create the distributed table var distributed strings.Builder + database := c.config.Database + if c.config.Database == "" { + database = "currentDatabase()" + } fmt.Fprintf(&distributed, "CREATE OR REPLACE TABLE %s %s AS %s", safeSQLName(name), onClusterClause, safelocalTableName(name)) - fmt.Fprintf(&distributed, " ENGINE = Distributed(%s, currentDatabase(), %s", safeSQLName(c.config.Cluster), safelocalTableName(name)) + fmt.Fprintf(&distributed, " ENGINE = Distributed(%s, %s, %s", safeSQLName(c.config.Cluster), database, safelocalTableName(name)) if outputProps.DistributedShardingKey != "" { fmt.Fprintf(&distributed, ", %s", outputProps.DistributedShardingKey) } else { @@ -555,9 +676,13 @@ func (c *connection) createDictionary(ctx context.Context, name, sql string, out func (c *connection) columnClause(ctx context.Context, table string) (string, error) { var columnClause strings.Builder + args := []any{c.config.Database, table} + if c.config.Database == "" { + args = []any{nil, table} + } res, err := c.Execute(ctx, &drivers.Statement{ - Query: "SELECT name, type FROM system.columns WHERE database = currentDatabase() AND table = ?", - Args: []any{table}, + Query: "SELECT name, type FROM system.columns WHERE database = coalesce(?, currentDatabase()) AND table = ?", + Args: args, Priority: 100, }) if err != nil { @@ -659,6 +784,51 @@ func (c *connection) acquireConn(ctx context.Context) (*sqlx.Conn, func() error, return conn, release, nil } +func (c *connection) getTableEngine(ctx context.Context, name string) (string, error) { + var engine string + args := []any{c.config.Database, name} + if c.config.Database == "" { + args = []any{nil, name} + } + res, err := c.Execute(ctx, &drivers.Statement{ + Query: "SELECT engine FROM system.tables WHERE database = coalesce(?, currentDatabase()) AND name = ?", + Args: args, + Priority: 1, + }) + if err != nil { + return "", err + } + defer res.Close() + if res.Next() { + if err := res.Scan(&engine); err != nil { + return "", err + } + } + return engine, nil +} + +func (c *connection) getTablePartitions(ctx context.Context, name string) ([]string, error) { + res, err := c.Execute(ctx, &drivers.Statement{ + Query: "SELECT DISTINCT partition FROM system.parts WHERE table = ?", + Args: []any{name}, + Priority: 1, + }) + if err != nil { + return nil, err + } + defer res.Close() + // collect partitions + var partitions []string + for res.Next() { + var part string + if err := res.Scan(&part); err != nil { + return nil, err + } + partitions = append(partitions, part) + } + return partitions, nil +} + func rowsToSchema(r *sqlx.Rows) (*runtimev1.StructType, error) { if r == nil { return nil, nil @@ -755,6 +925,8 @@ func databaseTypeToPB(dbt string, nullable bool) (*runtimev1.Type, error) { t.Code = runtimev1.Type_CODE_TIMESTAMP case "DATETIME64": t.Code = runtimev1.Type_CODE_TIMESTAMP + case "INTERVALNANOSECOND", "INTERVALMICROSECOND", "INTERVALMILLISECOND", "INTERVALSECOND", "INTERVALMINUTE", "INTERVALHOUR", "INTERVALDAY", "INTERVALWEEK", "INTERVALMONTH", "INTERVALQUARTER", "INTERVALYEAR": + t.Code = runtimev1.Type_CODE_INTERVAL case "JSON": t.Code = runtimev1.Type_CODE_JSON case "UUID": diff --git a/runtime/drivers/clickhouse/olap_test.go b/runtime/drivers/clickhouse/olap_test.go index d05e2767431..9a0fbf0cb8e 100644 --- a/runtime/drivers/clickhouse/olap_test.go +++ b/runtime/drivers/clickhouse/olap_test.go @@ -2,10 +2,13 @@ package clickhouse_test import ( "context" + "database/sql" "fmt" "testing" + runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" "github.com/rilldata/rill/runtime/drivers" + "github.com/rilldata/rill/runtime/drivers/clickhouse" "github.com/rilldata/rill/runtime/pkg/activity" "github.com/rilldata/rill/runtime/storage" "github.com/rilldata/rill/runtime/testruntime" @@ -13,35 +16,35 @@ import ( "go.uber.org/zap" ) -func TestClickhouseCrudOps(t *testing.T) { - // t.Setenv("TESTCONTAINERS_RYUK_DISABLED", "true") - if testing.Short() { - t.Skip("clickhouse: skipping test in short mode") - } +func TestClickhouseSingle(t *testing.T) { + cfg := testruntime.AcquireConnector(t, "clickhouse") - dsn, cluster := testruntime.ClickhouseCluster(t) - t.Run("SingleHost", func(t *testing.T) { testClickhouseSingleHost(t, dsn) }) - t.Run("Cluster", func(t *testing.T) { testClickhouseCluster(t, dsn, cluster) }) -} - -func testClickhouseSingleHost(t *testing.T, dsn string) { - conn, err := drivers.Open("clickhouse", "default", map[string]any{"dsn": dsn}, storage.MustNew(t.TempDir(), nil), activity.NewNoopClient(), zap.NewNop()) + conn, err := drivers.Open("clickhouse", "default", cfg, storage.MustNew(t.TempDir(), nil), activity.NewNoopClient(), zap.NewNop()) require.NoError(t, err) defer conn.Close() prepareConn(t, conn) olap, ok := conn.AsOLAP("default") require.True(t, ok) - t.Run("RenameView", func(t *testing.T) { - testRenameView(t, olap) - }) + t.Run("WithConnection", func(t *testing.T) { testWithConnection(t, olap) }) + t.Run("RenameView", func(t *testing.T) { testRenameView(t, olap) }) t.Run("RenameTable", func(t *testing.T) { testRenameTable(t, olap) }) t.Run("CreateTableAsSelect", func(t *testing.T) { testCreateTableAsSelect(t, olap) }) + t.Run("InsertTableAsSelect_WithAppend", func(t *testing.T) { testInsertTableAsSelect_WithAppend(t, olap) }) + t.Run("InsertTableAsSelect_WithMerge", func(t *testing.T) { testInsertTableAsSelect_WithMerge(t, olap) }) + t.Run("InsertTableAsSelect_WithPartitionOverwrite", func(t *testing.T) { testInsertTableAsSelect_WithPartitionOverwrite(t, olap) }) + t.Run("InsertTableAsSelect_WithPartitionOverwrite_DatePartition", func(t *testing.T) { testInsertTableAsSelect_WithPartitionOverwrite_DatePartition(t, olap) }) t.Run("TestDictionary", func(t *testing.T) { testDictionary(t, olap) }) - + t.Run("TestIntervalType", func(t *testing.T) { testIntervalType(t, olap) }) } -func testClickhouseCluster(t *testing.T, dsn, cluster string) { +func TestClickhouseCluster(t *testing.T) { + if testing.Short() { + t.Skip("clickhouse: skipping test in short mode") + } + + dsn, cluster := testruntime.ClickhouseCluster(t) + conn, err := drivers.Open("clickhouse", "default", map[string]any{"dsn": dsn, "cluster": cluster}, storage.MustNew(t.TempDir(), nil), activity.NewNoopClient(), zap.NewNop()) require.NoError(t, err) defer conn.Close() @@ -51,28 +54,62 @@ func testClickhouseCluster(t *testing.T, dsn, cluster string) { prepareClusterConn(t, olap, cluster) - t.Run("RenameView", func(t *testing.T) { - testRenameView(t, olap) - }) + t.Run("WithConnection", func(t *testing.T) { testWithConnection(t, olap) }) + t.Run("RenameView", func(t *testing.T) { testRenameView(t, olap) }) t.Run("RenameTable", func(t *testing.T) { testRenameTable(t, olap) }) t.Run("CreateTableAsSelect", func(t *testing.T) { testCreateTableAsSelect(t, olap) }) + t.Run("InsertTableAsSelect_WithAppend", func(t *testing.T) { testInsertTableAsSelect_WithAppend(t, olap) }) + t.Run("InsertTableAsSelect_WithMerge", func(t *testing.T) { testInsertTableAsSelect_WithMerge(t, olap) }) + t.Run("InsertTableAsSelect_WithPartitionOverwrite", func(t *testing.T) { testInsertTableAsSelect_WithPartitionOverwrite(t, olap) }) + t.Run("InsertTableAsSelect_WithPartitionOverwrite_DatePartition", func(t *testing.T) { testInsertTableAsSelect_WithPartitionOverwrite_DatePartition(t, olap) }) t.Run("TestDictionary", func(t *testing.T) { testDictionary(t, olap) }) } +func testWithConnection(t *testing.T, olap drivers.OLAPStore) { + err := olap.WithConnection(context.Background(), 1, false, func(ctx, ensuredCtx context.Context, conn *sql.Conn) error { + err := olap.Exec(ctx, &drivers.Statement{ + Query: "CREATE table tbl engine=Memory AS SELECT 1 AS id, 'Earth' AS planet", + }) + require.NoError(t, err) + + res, err := olap.Execute(ctx, &drivers.Statement{ + Query: "SELECT id, planet FROM tbl", + }) + require.NoError(t, err) + var ( + id int + planet string + ) + for res.Next() { + err = res.Scan(&id, &planet) + require.NoError(t, err) + require.Equal(t, 1, id) + } + require.NoError(t, res.Err()) + require.NoError(t, res.Close()) + return nil + }) + require.NoError(t, err) +} + func testRenameView(t *testing.T, olap drivers.OLAPStore) { ctx := context.Background() - err := olap.CreateTableAsSelect(ctx, "foo_view", true, "SELECT 1 AS id", map[string]any{"type": "VIEW"}) + opts := &drivers.CreateTableOptions{ + View: true, + TableOpts: map[string]any{"type": "VIEW"}, + } + err := olap.CreateTableAsSelect(ctx, "foo_view", "SELECT 1 AS id", opts) require.NoError(t, err) - err = olap.CreateTableAsSelect(ctx, "bar_view", true, "SELECT 'city' AS name", map[string]any{"type": "VIEW"}) + err = olap.CreateTableAsSelect(ctx, "bar_view", "SELECT 'city' AS name", opts) require.NoError(t, err) // rename to unknown view - err = olap.RenameTable(ctx, "foo_view", "foo_view1", true) + err = olap.RenameTable(ctx, "foo_view", "foo_view1") require.NoError(t, err) // rename to existing view - err = olap.RenameTable(ctx, "foo_view1", "bar_view", true) + err = olap.RenameTable(ctx, "foo_view1", "bar_view") require.NoError(t, err) // check that views no longer exist @@ -89,10 +126,10 @@ func testRenameView(t *testing.T, olap drivers.OLAPStore) { func testRenameTable(t *testing.T, olap drivers.OLAPStore) { ctx := context.Background() - err := olap.RenameTable(ctx, "foo", "foo1", false) + err := olap.RenameTable(ctx, "foo", "foo1") require.NoError(t, err) - err = olap.RenameTable(ctx, "foo1", "bar", false) + err = olap.RenameTable(ctx, "foo1", "bar") require.NoError(t, err) notExists(t, olap, "foo") @@ -111,19 +148,287 @@ func notExists(t *testing.T, olap drivers.OLAPStore, tbl string) { } func testCreateTableAsSelect(t *testing.T, olap drivers.OLAPStore) { - err := olap.CreateTableAsSelect(context.Background(), "tbl", false, "SELECT 1 AS id, 'Earth' AS planet", map[string]any{ - "engine": "MergeTree", - "table": "tbl", - "distributed.sharding_key": "rand()", - }) + opts := &drivers.CreateTableOptions{ + View: false, + TableOpts: map[string]any{"engine": "MergeTree", "table": "tbl", "distributed.sharding_key": "rand()"}, + } + err := olap.CreateTableAsSelect(context.Background(), "tbl", "SELECT 1 AS id, 'Earth' AS planet", opts) require.NoError(t, err) } +func testInsertTableAsSelect_WithAppend(t *testing.T, olap drivers.OLAPStore) { + opts := &drivers.CreateTableOptions{ + View: false, + TableOpts: map[string]any{ + "engine": "MergeTree", + "table": "tbl", + "distributed.sharding_key": "rand()", + "incremental_strategy": drivers.IncrementalStrategyAppend, + }, + } + err := olap.CreateTableAsSelect(context.Background(), "append_tbl", "SELECT 1 AS id, 'Earth' AS planet", opts) + require.NoError(t, err) + + insertOpts := &drivers.InsertTableOptions{ + ByName: false, + InPlace: true, + Strategy: drivers.IncrementalStrategyAppend, + UniqueKey: nil, + } + err = olap.InsertTableAsSelect(context.Background(), "append_tbl", "SELECT 2 AS id, 'Mars' AS planet", insertOpts) + require.NoError(t, err) + + res, err := olap.Execute(context.Background(), &drivers.Statement{Query: "SELECT id, planet FROM append_tbl ORDER BY id"}) + require.NoError(t, err) + + var result []struct { + ID int + Planet string + } + + for res.Next() { + var r struct { + ID int + Planet string + } + require.NoError(t, res.Scan(&r.ID, &r.Planet)) + result = append(result, r) + } + + expected := []struct { + ID int + Planet string + }{ + {1, "Earth"}, + {2, "Mars"}, + } + + // Convert the result set to a map to represent the set + resultSet := make(map[int]string) + for _, r := range result { + resultSet[r.ID] = r.Planet + } + + // Check if the expected values are present in the result set + for _, e := range expected { + value, exists := resultSet[e.ID] + require.True(t, exists, "Expected ID %d to be present in the result set", e.ID) + require.Equal(t, e.Planet, value, "Expected planet for ID %d to be %s, but got %s", e.ID, e.Planet, value) + } +} + +func testInsertTableAsSelect_WithMerge(t *testing.T, olap drivers.OLAPStore) { + opts := &drivers.CreateTableOptions{ + View: false, + TableOpts: map[string]any{ + "typs": "TABLE", + "engine": "ReplacingMergeTree", + "table": "tbl", + "distributed.sharding_key": "rand()", + "incremental_strategy": drivers.IncrementalStrategyMerge, + "order_by": "id", + }, + } + + err := olap.CreateTableAsSelect(context.Background(), "merge_tbl", "SELECT generate_series AS id, 'insert' AS value FROM generate_series(0, 4)", opts) + require.NoError(t, err) + + insertOpts := &drivers.InsertTableOptions{ + ByName: false, + InPlace: true, + Strategy: drivers.IncrementalStrategyMerge, + UniqueKey: []string{"id"}, + } + err = olap.InsertTableAsSelect(context.Background(), "merge_tbl", "SELECT generate_series AS id, 'merge' AS value FROM generate_series(2, 5)", insertOpts) + require.NoError(t, err) + + var result []struct { + ID int + Value string + } + + res, err := olap.Execute(context.Background(), &drivers.Statement{Query: "SELECT id, value FROM merge_tbl ORDER BY id"}) + require.NoError(t, err) + + for res.Next() { + var r struct { + ID int + Value string + } + require.NoError(t, res.Scan(&r.ID, &r.Value)) + result = append(result, r) + } + + expected := map[int]string{ + 0: "insert", + 1: "insert", + 2: "merge", + 3: "merge", + 4: "merge", + } + + // Convert the result set to a map to represent the set + resultSet := make(map[int]string) + for _, r := range result { + if v, ok := resultSet[r.ID]; !ok { + resultSet[r.ID] = r.Value + } else { + if v == "merge" { + resultSet[r.ID] = v + } + } + + } + + // Check if the expected values are present in the result set + for id, expected := range expected { + actual, exists := resultSet[id] + require.True(t, exists, "Expected ID %d to be present in the result set", id) + require.Equal(t, expected, actual, "Expected value for ID %d to be %s, but got %s", id, expected, actual) + } +} + +func testInsertTableAsSelect_WithPartitionOverwrite(t *testing.T, olap drivers.OLAPStore) { + opts := &drivers.CreateTableOptions{ + View: false, + TableOpts: map[string]any{ + "engine": "MergeTree", + "table": "tbl", + "distributed.sharding_key": "rand()", + "incremental_strategy": drivers.IncrementalStrategyPartitionOverwrite, + "partition_by": "id", + "order_by": "value", + "primary_key": "value", + }, + } + err := olap.CreateTableAsSelect(context.Background(), "replace_tbl", "SELECT generate_series AS id, 'insert' AS value FROM generate_series(0, 4)", opts) + require.NoError(t, err) + + insertOpts := &drivers.InsertTableOptions{ + ByName: false, + InPlace: true, + Strategy: drivers.IncrementalStrategyPartitionOverwrite, + } + err = olap.InsertTableAsSelect(context.Background(), "replace_tbl", "SELECT generate_series AS id, 'replace' AS value FROM generate_series(2, 5)", insertOpts) + require.NoError(t, err) + + res, err := olap.Execute(context.Background(), &drivers.Statement{Query: "SELECT id, value FROM replace_tbl ORDER BY id"}) + require.NoError(t, err) + + var result []struct { + ID int + Value string + } + + for res.Next() { + var r struct { + ID int + Value string + } + require.NoError(t, res.Scan(&r.ID, &r.Value)) + result = append(result, r) + } + + expected := []struct { + ID int + Value string + }{ + {0, "insert"}, + {1, "insert"}, + {2, "replace"}, + {3, "replace"}, + {4, "replace"}, + } + + // Convert the result set to a map to represent the set + resultSet := make(map[int]string) + for _, r := range result { + resultSet[r.ID] = r.Value + } + + // Check if the expected values are present in the result set + for _, e := range expected { + value, exists := resultSet[e.ID] + require.True(t, exists, "Expected ID %d to be present in the result set", e.ID) + require.Equal(t, e.Value, value, "Expected value for ID %d to be %s, but got %s", e.ID, e.Value, value) + } +} + +func testInsertTableAsSelect_WithPartitionOverwrite_DatePartition(t *testing.T, olap drivers.OLAPStore) { + opts := &drivers.CreateTableOptions{ + View: false, + TableOpts: map[string]any{ + "engine": "MergeTree", + "table": "tbl", + "distributed.sharding_key": "rand()", + "incremental_strategy": drivers.IncrementalStrategyPartitionOverwrite, + "partition_by": "dt", + "order_by": "value", + "primary_key": "value", + }, + } + err := olap.CreateTableAsSelect(context.Background(), "replace_tbl", "SELECT date_add(hour, generate_series, toDate('2024-12-01')) AS dt, 'insert' AS value FROM generate_series(0, 4)", opts) + require.NoError(t, err) + + insertOpts := &drivers.InsertTableOptions{ + ByName: false, + InPlace: true, + Strategy: drivers.IncrementalStrategyPartitionOverwrite, + } + err = olap.InsertTableAsSelect(context.Background(), "replace_tbl", "SELECT date_add(hour, generate_series, toDate('2024-12-01')) AS dt, 'replace' AS value FROM generate_series(2, 5)", insertOpts) + require.NoError(t, err) + + res, err := olap.Execute(context.Background(), &drivers.Statement{Query: "SELECT dt, value FROM replace_tbl ORDER BY dt"}) + require.NoError(t, err) + + var result []struct { + DT string + Value string + } + + for res.Next() { + var r struct { + DT string + Value string + } + require.NoError(t, res.Scan(&r.DT, &r.Value)) + result = append(result, r) + } + + expected := []struct { + DT string + Value string + }{ + {"2024-12-01T00:00:00Z", "insert"}, + {"2024-12-01T01:00:00Z", "insert"}, + {"2024-12-01T02:00:00Z", "replace"}, + {"2024-12-01T03:00:00Z", "replace"}, + {"2024-12-01T04:00:00Z", "replace"}, + } + + // Convert the result set to a map to represent the set + resultSet := make(map[string]string) + for _, r := range result { + resultSet[r.DT] = r.Value + } + + // Check if the expected values are present in the result set + for _, e := range expected { + value, exists := resultSet[e.DT] + require.True(t, exists, "Expected DateTime %s to be present in the result set", e.DT) + require.Equal(t, e.Value, value, "Expected value for DateTime %s to be %s, but got %s", e.DT, e.Value, value) + } +} + func testDictionary(t *testing.T, olap drivers.OLAPStore) { - err := olap.CreateTableAsSelect(context.Background(), "dict", false, "SELECT 1 AS id, 'Earth' AS planet", map[string]any{"table": "Dictionary", "primary_key": "id"}) + opts := &drivers.CreateTableOptions{ + View: false, + TableOpts: map[string]any{"table": "Dictionary", "primary_key": "id"}, + } + err := olap.CreateTableAsSelect(context.Background(), "dict", "SELECT 1 AS id, 'Earth' AS planet", opts) require.NoError(t, err) - err = olap.RenameTable(context.Background(), "dict", "dict1", false) + err = olap.RenameTable(context.Background(), "dict", "dict1") require.NoError(t, err) res, err := olap.Execute(context.Background(), &drivers.Statement{Query: "SELECT id, planet FROM dict1"}) @@ -136,7 +441,34 @@ func testDictionary(t *testing.T, olap drivers.OLAPStore) { require.Equal(t, 1, id) require.Equal(t, "Earth", planet) - require.NoError(t, olap.DropTable(context.Background(), "dict1", false)) + require.NoError(t, olap.DropTable(context.Background(), "dict1")) +} + +func testIntervalType(t *testing.T, olap drivers.OLAPStore) { + cases := []struct { + query string + ms int64 + }{ + {query: "SELECT INTERVAL '1' SECOND", ms: 1000}, + {query: "SELECT INTERVAL '2' MINUTES", ms: 2 * 60 * 1000}, + {query: "SELECT INTERVAL '3' HOURS", ms: 3 * 60 * 60 * 1000}, + {query: "SELECT INTERVAL '4' DAYS", ms: 4 * 24 * 60 * 60 * 1000}, + {query: "SELECT INTERVAL '5' MONTHS", ms: 5 * 30 * 24 * 60 * 60 * 1000}, + {query: "SELECT INTERVAL '6' YEAR", ms: 6 * 365 * 24 * 60 * 60 * 1000}, + } + for _, c := range cases { + rows, err := olap.Execute(context.Background(), &drivers.Statement{Query: c.query}) + require.NoError(t, err) + require.Equal(t, runtimev1.Type_CODE_INTERVAL, rows.Schema.Fields[0].Type.Code) + + require.True(t, rows.Next()) + var s string + require.NoError(t, rows.Scan(&s)) + ms, ok := clickhouse.ParseIntervalToMillis(s) + require.True(t, ok) + require.Equal(t, c.ms, ms) + require.NoError(t, rows.Close()) + } } func prepareClusterConn(t *testing.T, olap drivers.OLAPStore, cluster string) { diff --git a/runtime/drivers/clickhouse/utils.go b/runtime/drivers/clickhouse/utils.go index 86d49183ad7..2595ac12746 100644 --- a/runtime/drivers/clickhouse/utils.go +++ b/runtime/drivers/clickhouse/utils.go @@ -1,9 +1,52 @@ package clickhouse import ( + "strconv" + "strings" + "github.com/rilldata/rill/runtime/drivers" ) +// ParseIntervalToMillis parses a ClickHouse INTERVAL string into milliseconds. +// ClickHouse currently returns INTERVALs as strings in the format "1 Month", "2 Minutes", etc. +// This function follows our current policy of treating months as 30 days when converting to milliseconds. +func ParseIntervalToMillis(s string) (int64, bool) { + s1, s2, ok := strings.Cut(s, " ") + if !ok { + return 0, false + } + + units, err := strconv.ParseInt(s1, 10, 64) + if err != nil { + return 0, false + } + + switch s2 { + case "Nanosecond", "Nanoseconds": + return int64(float64(units) / 1_000_000), true + case "Microsecond", "Microseconds": + return int64(float64(units) / 1_000), true + case "Millisecond", "Milliseconds": + return units * 1, true + case "Second", "Seconds": + return units * 1000, true + case "Minute", "Minutes": + return units * 60 * 1000, true + case "Hour", "Hours": + return units * 60 * 60 * 1000, true + case "Day", "Days": + return units * 24 * 60 * 60 * 1000, true + case "Month", "Months": + return units * 30 * 24 * 60 * 60 * 1000, true + case "Quarter", "Quarters": + return units * 3 * 30 * 24 * 60 * 60 * 1000, true + case "Year", "Years": + return units * 365 * 24 * 60 * 60 * 1000, true + default: + return 0, false + } +} + func safeSQLName(name string) string { return drivers.DialectClickHouse.EscapeIdentifier(name) } diff --git a/runtime/drivers/drivers.go b/runtime/drivers/drivers.go index e1b4944c800..973d05b4b9a 100644 --- a/runtime/drivers/drivers.go +++ b/runtime/drivers/drivers.go @@ -107,10 +107,6 @@ type Handle interface { // An AI service enables an instance to request prompt-based text inference. AsAI(instanceID string) (AIService, bool) - // AsSQLStore returns a SQLStore if the driver can serve as such, otherwise returns false. - // A SQL store represents a service that can execute SQL statements and return the resulting rows. - AsSQLStore() (SQLStore, bool) - // AsOLAP returns an OLAPStore if the driver can serve as such, otherwise returns false. // An OLAP store is used to serve interactive, low-latency, analytical queries. // NOTE: We should consider merging the OLAPStore and SQLStore interfaces. diff --git a/runtime/drivers/druid/druid.go b/runtime/drivers/druid/druid.go index 54ed78d4323..ade0faf22df 100644 --- a/runtime/drivers/druid/druid.go +++ b/runtime/drivers/druid/druid.go @@ -254,12 +254,6 @@ func (c *connection) AsWarehouse() (drivers.Warehouse, bool) { return nil, false } -// AsSQLStore implements drivers.Connection. -// Use OLAPStore instead. -func (c *connection) AsSQLStore() (drivers.SQLStore, bool) { - return nil, false -} - // AsNotifier implements drivers.Connection. func (c *connection) AsNotifier(properties map[string]any) (drivers.Notifier, error) { return nil, drivers.ErrNotNotifier diff --git a/runtime/drivers/druid/druid_container_test.go b/runtime/drivers/druid/druid_container_test.go new file mode 100644 index 00000000000..a5705f7184a --- /dev/null +++ b/runtime/drivers/druid/druid_container_test.go @@ -0,0 +1,232 @@ +package druid + +import ( + "context" + "fmt" + "net/url" + "strings" + "testing" + "time" + + runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" + "github.com/rilldata/rill/runtime/drivers" + "github.com/rilldata/rill/runtime/pkg/activity" + "github.com/rilldata/rill/runtime/storage" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + "go.uber.org/zap" +) + +const testTable = "test_data" + +var testCSV = strings.TrimSpace(` +id,timestamp,publisher,domain,bid_price +5000,2022-03-18T12:25:58.074Z,Facebook,facebook.com,4.19 +9000,2022-03-15T11:17:23.530Z,Microsoft,msn.com,3.48 +10000,2022-03-02T04:00:56.643Z,Microsoft,msn.com,3.57 +11000,2022-01-16T00:26:44.770Z,,instagram.com,5.38 +12000,2022-01-17T08:55:09.270Z,,msn.com,1.34 +13000,2022-03-20T03:16:57.618Z,Yahoo,news.yahoo.com,1.05 +14000,2022-01-29T19:05:33.545Z,Google,news.google.com,4.54 +15000,2022-03-22T00:56:22.035Z,Yahoo,news.yahoo.com,1.13 +16000,2022-01-24T13:41:43.527Z,,instagram.com,1.78 +`) + +var testIngestSpec = fmt.Sprintf(`{ + "type": "index_parallel", + "spec": { + "ioConfig": { + "type": "index_parallel", + "inputSource": { + "type": "inline", + "data": "%s" + }, + "inputFormat": { + "type": "csv", + "findColumnsFromHeader": true + } + }, + "tuningConfig": { + "type": "index_parallel", + "partitionsSpec": { + "type": "dynamic" + } + }, + "dataSchema": { + "dataSource": "%s", + "timestampSpec": { + "column": "timestamp", + "format": "iso" + }, + "transformSpec": {}, + "dimensionsSpec": { + "dimensions": [ + {"type": "long", "name": "id"}, + "publisher", + "domain", + {"type": "double", "name": "bid_price"} + ] + }, + "granularitySpec": { + "queryGranularity": "none", + "rollup": false, + "segmentGranularity": "day" + } + } + } +}`, strings.ReplaceAll(testCSV, "\n", "\\n"), testTable) + +// TestContainer starts a Druid cluster using testcontainers, ingests data into it, then runs all other tests +// in this file as sub-tests (to prevent spawning many clusters). +// +// Unfortunately starting a Druid cluster with test containers is extremely slow. +// If you have access to our Druid test cluster, consider using the test_druid.go file instead. +func TestContainer(t *testing.T) { + if testing.Short() { + t.Skip("druid: skipping test in short mode") + } + + ctx := context.Background() + container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + Started: true, + ContainerRequest: testcontainers.ContainerRequest{ + WaitingFor: wait.ForHTTP("/status/health").WithPort("8081").WithStartupTimeout(time.Minute * 2), + Image: "gcr.io/rilldata/druid-micro:25.0.0", + ExposedPorts: []string{"8081/tcp", "8082/tcp"}, + Cmd: []string{"./bin/start-micro-quickstart"}, + }, + }) + require.NoError(t, err) + defer container.Terminate(ctx) + + coordinatorURL, err := container.PortEndpoint(ctx, "8081/tcp", "http") + require.NoError(t, err) + + t.Run("ingest", func(t *testing.T) { testIngest(t, coordinatorURL) }) + + brokerURL, err := container.PortEndpoint(ctx, "8082/tcp", "http") + require.NoError(t, err) + + druidAPIURL, err := url.JoinPath(brokerURL, "/druid/v2/sql") + require.NoError(t, err) + + dd := &driver{} + conn, err := dd.Open("default", map[string]any{"dsn": druidAPIURL}, storage.MustNew(t.TempDir(), nil), activity.NewNoopClient(), zap.NewNop()) + require.NoError(t, err) + + olap, ok := conn.AsOLAP("") + require.True(t, ok) + + t.Run("count", func(t *testing.T) { testCount(t, olap) }) + t.Run("max", func(t *testing.T) { testMax(t, olap) }) + t.Run("schema all", func(t *testing.T) { testSchemaAll(t, olap) }) + t.Run("schema all like", func(t *testing.T) { testSchemaAllLike(t, olap) }) + t.Run("schema lookup", func(t *testing.T) { testSchemaLookup(t, olap) }) + // Add new tests here + t.Run("time floor", func(t *testing.T) { testTimeFloor(t, olap) }) + + require.NoError(t, conn.Close()) +} + +func testIngest(t *testing.T, coordinatorURL string) { + timeout := 5 * time.Minute + err := Ingest(coordinatorURL, testIngestSpec, testTable, timeout) + require.NoError(t, err) +} + +func testCount(t *testing.T, olap drivers.OLAPStore) { + qry := fmt.Sprintf("SELECT count(*) FROM %s", testTable) + rows, err := olap.Execute(context.Background(), &drivers.Statement{Query: qry}) + require.NoError(t, err) + + var count int + rows.Next() + + require.NoError(t, rows.Scan(&count)) + require.Equal(t, 9, count) + require.NoError(t, rows.Close()) +} + +func testMax(t *testing.T, olap drivers.OLAPStore) { + qry := fmt.Sprintf("SELECT max(id) FROM %s", testTable) + expectedValue := 16000 + rows, err := olap.Execute(context.Background(), &drivers.Statement{Query: qry}) + require.NoError(t, err) + + var count int + rows.Next() + require.NoError(t, rows.Scan(&count)) + require.Equal(t, expectedValue, count) + require.NoError(t, rows.Close()) +} + +func testTimeFloor(t *testing.T, olap drivers.OLAPStore) { + qry := fmt.Sprintf("SELECT time_floor(__time, 'P1D', null, CAST(? AS VARCHAR)) FROM %s", testTable) + rows, err := olap.Execute(context.Background(), &drivers.Statement{ + Query: qry, + Args: []any{"Asia/Kathmandu"}, + }) + require.NoError(t, err) + defer rows.Close() + + var tmString string + count := 0 + for rows.Next() { + require.NoError(t, rows.Scan(&tmString)) + tm, err := time.Parse(time.RFC3339, tmString) + require.NoError(t, err) + require.Equal(t, 15, tm.Minute()) + count += 1 + } + require.Equal(t, 9, count) +} + +func testSchemaAll(t *testing.T, olap drivers.OLAPStore) { + tables, err := olap.InformationSchema().All(context.Background(), "") + require.NoError(t, err) + + require.Equal(t, 1, len(tables)) + require.Equal(t, testTable, tables[0].Name) + + require.Equal(t, 5, len(tables[0].Schema.Fields)) + + mp := make(map[string]*runtimev1.StructType_Field) + for _, f := range tables[0].Schema.Fields { + mp[f.Name] = f + } + + f := mp["__time"] + require.Equal(t, "__time", f.Name) + require.Equal(t, runtimev1.Type_CODE_TIMESTAMP, f.Type.Code) + require.Equal(t, false, f.Type.Nullable) + f = mp["bid_price"] + require.Equal(t, runtimev1.Type_CODE_FLOAT64, f.Type.Code) + require.Equal(t, false, f.Type.Nullable) + f = mp["domain"] + require.Equal(t, runtimev1.Type_CODE_STRING, f.Type.Code) + require.Equal(t, true, f.Type.Nullable) + f = mp["id"] + require.Equal(t, runtimev1.Type_CODE_INT64, f.Type.Code) + require.Equal(t, false, f.Type.Nullable) + f = mp["publisher"] + require.Equal(t, runtimev1.Type_CODE_STRING, f.Type.Code) + require.Equal(t, true, f.Type.Nullable) +} + +func testSchemaAllLike(t *testing.T, olap drivers.OLAPStore) { + tables, err := olap.InformationSchema().All(context.Background(), "%test%") + require.NoError(t, err) + require.Equal(t, 1, len(tables)) + require.Equal(t, testTable, tables[0].Name) +} + +func testSchemaLookup(t *testing.T, olap drivers.OLAPStore) { + ctx := context.Background() + table, err := olap.InformationSchema().Lookup(ctx, "", "", testTable) + require.NoError(t, err) + require.Equal(t, testTable, table.Name) + + _, err = olap.InformationSchema().Lookup(ctx, "", "", "foo") + require.Equal(t, drivers.ErrNotFound, err) +} diff --git a/runtime/drivers/druid/druid_test.go b/runtime/drivers/druid/druid_test.go index 635d4f2279e..b313bdcb6e6 100644 --- a/runtime/drivers/druid/druid_test.go +++ b/runtime/drivers/druid/druid_test.go @@ -1,229 +1,49 @@ -package druid +package druid_test import ( "context" - "fmt" - "net/url" - "strings" "testing" "time" - runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" "github.com/rilldata/rill/runtime/drivers" "github.com/rilldata/rill/runtime/pkg/activity" "github.com/rilldata/rill/runtime/storage" + "github.com/rilldata/rill/runtime/testruntime" "github.com/stretchr/testify/require" - "github.com/testcontainers/testcontainers-go" - "github.com/testcontainers/testcontainers-go/wait" "go.uber.org/zap" ) -const testTable = "test_data" +func TestScan(t *testing.T) { + _, olap := acquireTestDruid(t) -var testCSV = strings.TrimSpace(` -id,timestamp,publisher,domain,bid_price -5000,2022-03-18T12:25:58.074Z,Facebook,facebook.com,4.19 -9000,2022-03-15T11:17:23.530Z,Microsoft,msn.com,3.48 -10000,2022-03-02T04:00:56.643Z,Microsoft,msn.com,3.57 -11000,2022-01-16T00:26:44.770Z,,instagram.com,5.38 -12000,2022-01-17T08:55:09.270Z,,msn.com,1.34 -13000,2022-03-20T03:16:57.618Z,Yahoo,news.yahoo.com,1.05 -14000,2022-01-29T19:05:33.545Z,Google,news.google.com,4.54 -15000,2022-03-22T00:56:22.035Z,Yahoo,news.yahoo.com,1.13 -16000,2022-01-24T13:41:43.527Z,,instagram.com,1.78 -`) - -var testIngestSpec = fmt.Sprintf(`{ - "type": "index_parallel", - "spec": { - "ioConfig": { - "type": "index_parallel", - "inputSource": { - "type": "inline", - "data": "%s" - }, - "inputFormat": { - "type": "csv", - "findColumnsFromHeader": true - } - }, - "tuningConfig": { - "type": "index_parallel", - "partitionsSpec": { - "type": "dynamic" - } - }, - "dataSchema": { - "dataSource": "%s", - "timestampSpec": { - "column": "timestamp", - "format": "iso" - }, - "transformSpec": {}, - "dimensionsSpec": { - "dimensions": [ - {"type": "long", "name": "id"}, - "publisher", - "domain", - {"type": "double", "name": "bid_price"} - ] - }, - "granularitySpec": { - "queryGranularity": "none", - "rollup": false, - "segmentGranularity": "day" - } - } - } -}`, strings.ReplaceAll(testCSV, "\n", "\\n"), testTable) - -// TestDruid starts a Druid cluster using testcontainers, ingests data into it, then runs all other tests -// in this file as sub-tests (to prevent spawning many clusters). -func TestDruid(t *testing.T) { - if testing.Short() { - t.Skip("druid: skipping test in short mode") - } - - ctx := context.Background() - container, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ - Started: true, - ContainerRequest: testcontainers.ContainerRequest{ - WaitingFor: wait.ForHTTP("/status/health").WithPort("8081").WithStartupTimeout(time.Minute * 2), - Image: "gcr.io/rilldata/druid-micro:25.0.0", - ExposedPorts: []string{"8081/tcp", "8082/tcp"}, - Cmd: []string{"./bin/start-micro-quickstart"}, - }, - }) - require.NoError(t, err) - defer container.Terminate(ctx) - - coordinatorURL, err := container.PortEndpoint(ctx, "8081/tcp", "http") - require.NoError(t, err) - - t.Run("ingest", func(t *testing.T) { testIngest(t, coordinatorURL) }) - - brokerURL, err := container.PortEndpoint(ctx, "8082/tcp", "http") - require.NoError(t, err) - - druidAPIURL, err := url.JoinPath(brokerURL, "/druid/v2/sql") - require.NoError(t, err) - - dd := &driver{} - conn, err := dd.Open("default", map[string]any{"dsn": druidAPIURL}, storage.MustNew(t.TempDir(), nil), activity.NewNoopClient(), zap.NewNop()) - require.NoError(t, err) - - olap, ok := conn.AsOLAP("") - require.True(t, ok) - - t.Run("count", func(t *testing.T) { testCount(t, olap) }) - t.Run("max", func(t *testing.T) { testMax(t, olap) }) - t.Run("schema all", func(t *testing.T) { testSchemaAll(t, olap) }) - t.Run("schema all like", func(t *testing.T) { testSchemaAllLike(t, olap) }) - t.Run("schema lookup", func(t *testing.T) { testSchemaLookup(t, olap) }) - // Add new tests here - t.Run("time floor", func(t *testing.T) { testTimeFloor(t, olap) }) - - require.NoError(t, conn.Close()) -} - -func testIngest(t *testing.T, coordinatorURL string) { - timeout := 5 * time.Minute - err := Ingest(coordinatorURL, testIngestSpec, testTable, timeout) - require.NoError(t, err) -} - -func testCount(t *testing.T, olap drivers.OLAPStore) { - qry := fmt.Sprintf("SELECT count(*) FROM %s", testTable) - rows, err := olap.Execute(context.Background(), &drivers.Statement{Query: qry}) + rows, err := olap.Execute(context.Background(), &drivers.Statement{Query: "SELECT 1, 'hello world', true, null, CAST('2024-01-01T00:00:00Z' AS TIMESTAMP)"}) require.NoError(t, err) - var count int - rows.Next() - - require.NoError(t, rows.Scan(&count)) - require.Equal(t, 9, count) - require.NoError(t, rows.Close()) -} + var i int + var s string + var b bool + var n any + var t1 time.Time + require.True(t, rows.Next()) + require.NoError(t, rows.Scan(&i, &s, &b, &n, &t1)) -func testMax(t *testing.T, olap drivers.OLAPStore) { - qry := fmt.Sprintf("SELECT max(id) FROM %s", testTable) - expectedValue := 16000 - rows, err := olap.Execute(context.Background(), &drivers.Statement{Query: qry}) - require.NoError(t, err) + require.Equal(t, 1, i) + require.Equal(t, "hello world", s) + require.Equal(t, true, b) + require.Nil(t, n) + require.Equal(t, time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), t1) - var count int - rows.Next() - require.NoError(t, rows.Scan(&count)) - require.Equal(t, expectedValue, count) require.NoError(t, rows.Close()) } -func testTimeFloor(t *testing.T, olap drivers.OLAPStore) { - qry := fmt.Sprintf("SELECT time_floor(__time, 'P1D', null, CAST(? AS VARCHAR)) FROM %s", testTable) - rows, err := olap.Execute(context.Background(), &drivers.Statement{ - Query: qry, - Args: []any{"Asia/Kathmandu"}, - }) - require.NoError(t, err) - defer rows.Close() - - var tmString string - count := 0 - for rows.Next() { - require.NoError(t, rows.Scan(&tmString)) - tm, err := time.Parse(time.RFC3339, tmString) - require.NoError(t, err) - require.Equal(t, 15, tm.Minute()) - count += 1 - } - require.Equal(t, 9, count) -} - -func testSchemaAll(t *testing.T, olap drivers.OLAPStore) { - tables, err := olap.InformationSchema().All(context.Background(), "") +func acquireTestDruid(t *testing.T) (drivers.Handle, drivers.OLAPStore) { + cfg := testruntime.AcquireConnector(t, "druid") + conn, err := drivers.Open("druid", "default", cfg, storage.MustNew(t.TempDir(), nil), activity.NewNoopClient(), zap.NewNop()) require.NoError(t, err) + t.Cleanup(func() { conn.Close() }) - require.Equal(t, 1, len(tables)) - require.Equal(t, testTable, tables[0].Name) - - require.Equal(t, 5, len(tables[0].Schema.Fields)) - - mp := make(map[string]*runtimev1.StructType_Field) - for _, f := range tables[0].Schema.Fields { - mp[f.Name] = f - } - - f := mp["__time"] - require.Equal(t, "__time", f.Name) - require.Equal(t, runtimev1.Type_CODE_TIMESTAMP, f.Type.Code) - require.Equal(t, false, f.Type.Nullable) - f = mp["bid_price"] - require.Equal(t, runtimev1.Type_CODE_FLOAT64, f.Type.Code) - require.Equal(t, false, f.Type.Nullable) - f = mp["domain"] - require.Equal(t, runtimev1.Type_CODE_STRING, f.Type.Code) - require.Equal(t, true, f.Type.Nullable) - f = mp["id"] - require.Equal(t, runtimev1.Type_CODE_INT64, f.Type.Code) - require.Equal(t, false, f.Type.Nullable) - f = mp["publisher"] - require.Equal(t, runtimev1.Type_CODE_STRING, f.Type.Code) - require.Equal(t, true, f.Type.Nullable) -} - -func testSchemaAllLike(t *testing.T, olap drivers.OLAPStore) { - tables, err := olap.InformationSchema().All(context.Background(), "%test%") - require.NoError(t, err) - require.Equal(t, 1, len(tables)) - require.Equal(t, testTable, tables[0].Name) -} - -func testSchemaLookup(t *testing.T, olap drivers.OLAPStore) { - ctx := context.Background() - table, err := olap.InformationSchema().Lookup(ctx, "", "", testTable) - require.NoError(t, err) - require.Equal(t, testTable, table.Name) + olap, ok := conn.AsOLAP("default") + require.True(t, ok) - _, err = olap.InformationSchema().Lookup(ctx, "", "", "foo") - require.Equal(t, drivers.ErrNotFound, err) + return conn, olap } diff --git a/runtime/drivers/druid/olap.go b/runtime/drivers/druid/olap.go index 63e05ff2203..8ef02f99c5d 100644 --- a/runtime/drivers/druid/olap.go +++ b/runtime/drivers/druid/olap.go @@ -31,22 +31,22 @@ func (c *connection) AlterTableColumn(ctx context.Context, tableName, columnName } // CreateTableAsSelect implements drivers.OLAPStore. -func (c *connection) CreateTableAsSelect(ctx context.Context, name string, view bool, sql string, tableOpts map[string]any) error { +func (c *connection) CreateTableAsSelect(ctx context.Context, name, sql string, opts *drivers.CreateTableOptions) error { return fmt.Errorf("druid: data transformation not yet supported") } // InsertTableAsSelect implements drivers.OLAPStore. -func (c *connection) InsertTableAsSelect(ctx context.Context, name, sql string, byName, inPlace bool, strategy drivers.IncrementalStrategy, uniqueKey []string) error { +func (c *connection) InsertTableAsSelect(ctx context.Context, name, sql string, opts *drivers.InsertTableOptions) error { return fmt.Errorf("druid: data transformation not yet supported") } // DropTable implements drivers.OLAPStore. -func (c *connection) DropTable(ctx context.Context, name string, view bool) error { +func (c *connection) DropTable(ctx context.Context, name string) error { return fmt.Errorf("druid: data transformation not yet supported") } // RenameTable implements drivers.OLAPStore. -func (c *connection) RenameTable(ctx context.Context, name, newName string, view bool) error { +func (c *connection) RenameTable(ctx context.Context, name, newName string) error { return fmt.Errorf("druid: data transformation not yet supported") } @@ -54,7 +54,7 @@ func (c *connection) Dialect() drivers.Dialect { return drivers.DialectDruid } -func (c *connection) WithConnection(ctx context.Context, priority int, longRunning, tx bool, fn drivers.WithConnectionFunc) error { +func (c *connection) WithConnection(ctx context.Context, priority int, longRunning bool, fn drivers.WithConnectionFunc) error { return fmt.Errorf("druid: WithConnection not supported") } diff --git a/runtime/drivers/druid/sql_driver_test.go b/runtime/drivers/druid/sql_driver_test.go index ff9dbf57bd7..b526b77767b 100644 --- a/runtime/drivers/druid/sql_driver_test.go +++ b/runtime/drivers/druid/sql_driver_test.go @@ -6,11 +6,10 @@ import ( "testing" "github.com/rilldata/rill/runtime/drivers" - "github.com/rilldata/rill/runtime/storage" - "github.com/stretchr/testify/require" - "github.com/rilldata/rill/runtime/pkg/activity" "github.com/rilldata/rill/runtime/pkg/pbutil" + "github.com/rilldata/rill/runtime/storage" + "github.com/stretchr/testify/require" "go.uber.org/zap" "google.golang.org/protobuf/types/known/structpb" ) diff --git a/runtime/drivers/duckdb/config.go b/runtime/drivers/duckdb/config.go index 90f52b088a2..a952a410538 100644 --- a/runtime/drivers/duckdb/config.go +++ b/runtime/drivers/duckdb/config.go @@ -2,10 +2,7 @@ package duckdb import ( "fmt" - "net/url" - "path/filepath" "strconv" - "strings" "github.com/mitchellh/mapstructure" ) @@ -17,141 +14,63 @@ const ( // config represents the DuckDB driver config type config struct { - // DSN is the connection string. Also allows a special `:memory:` path to initialize an in-memory database. - DSN string `mapstructure:"dsn"` - // Path is a path to the database file. If set, it will take precedence over the path contained in DSN. - // This is a convenience option for setting the path in a more human-readable way. - Path string `mapstructure:"path"` - // DataDir is the path to directory where duckdb file named `main.db` will be created. In case of external table storage all the files will also be present in DataDir's subdirectories. - // If path is set then DataDir is ignored. - DataDir string `mapstructure:"data_dir"` // PoolSize is the number of concurrent connections and queries allowed PoolSize int `mapstructure:"pool_size"` // AllowHostAccess denotes whether to limit access to the local environment and file system AllowHostAccess bool `mapstructure:"allow_host_access"` - // ErrorOnIncompatibleVersion controls whether to return error or delete DBFile created with older duckdb version. - ErrorOnIncompatibleVersion bool `mapstructure:"error_on_incompatible_version"` - // ExtTableStorage controls if every table is stored in a different db file - ExtTableStorage bool `mapstructure:"external_table_storage"` - // CPU cores available for the DB + // CPU cores available for the read DB. If no CPUWrite is set then this is split evenly between read and write. CPU int `mapstructure:"cpu"` - // MemoryLimitGB is the amount of memory available for the DB + // MemoryLimitGB is the amount of memory available for the read DB. If no MemoryLimitGBWrite is set then this is split evenly between read and write. MemoryLimitGB int `mapstructure:"memory_limit_gb"` - // MaxMemoryOverride sets a hard override for the "max_memory" DuckDB setting - MaxMemoryGBOverride int `mapstructure:"max_memory_gb_override"` - // ThreadsOverride sets a hard override for the "threads" DuckDB setting. Set to -1 for unlimited threads. - ThreadsOverride int `mapstructure:"threads_override"` + // CPUWrite is CPU available for the DB when writing data. + CPUWrite int `mapstructure:"cpu_write"` + // MemoryLimitGBWrite is the amount of memory available for the DB when writing data. + MemoryLimitGBWrite int `mapstructure:"memory_limit_gb_write"` // BootQueries is SQL to execute when initializing a new connection. It runs before any extensions are loaded or default settings are set. BootQueries string `mapstructure:"boot_queries"` // InitSQL is SQL to execute when initializing a new connection. It runs after extensions are loaded and and default settings are set. InitSQL string `mapstructure:"init_sql"` - // DBFilePath is the path where the database is stored. It is inferred from the DSN (can't be provided by user). - DBFilePath string `mapstructure:"-"` - // DBStoragePath is the path where the database files are stored. It is inferred from the DSN (can't be provided by user). - DBStoragePath string `mapstructure:"-"` // LogQueries controls whether to log the raw SQL passed to OLAP.Execute. (Internal queries will not be logged.) LogQueries bool `mapstructure:"log_queries"` } -func newConfig(cfgMap map[string]any, dataDir string) (*config, error) { - cfg := &config{ - ExtTableStorage: true, - DataDir: dataDir, - } +func newConfig(cfgMap map[string]any) (*config, error) { + cfg := &config{} err := mapstructure.WeakDecode(cfgMap, cfg) if err != nil { return nil, fmt.Errorf("could not decode config: %w", err) } - inMemory := false - if strings.HasPrefix(cfg.DSN, ":memory:") { - inMemory = true - cfg.DSN = strings.Replace(cfg.DSN, ":memory:", "", 1) - cfg.ExtTableStorage = false - } - - // Parse DSN as URL - uri, err := url.Parse(cfg.DSN) - if err != nil { - return nil, fmt.Errorf("could not parse dsn: %w", err) - } - qry, err := url.ParseQuery(uri.RawQuery) - if err != nil { - return nil, fmt.Errorf("could not parse dsn: %w", err) - } - - if !inMemory { - // Override DSN.Path with config.Path - if cfg.Path != "" { // backward compatibility, cfg.Path takes precedence over cfg.DataDir - uri.Path = cfg.Path - } else if cfg.DataDir != "" && uri.Path == "" { // if some path is set in DSN, honour that path and ignore DataDir - uri.Path = filepath.Join(cfg.DataDir, "main.db") - } - - // Infer DBFilePath - cfg.DBFilePath = uri.Path - cfg.DBStoragePath = filepath.Dir(cfg.DBFilePath) - } - - // Set memory limit - maxMemory := cfg.MemoryLimitGB - if cfg.MaxMemoryGBOverride != 0 { - maxMemory = cfg.MaxMemoryGBOverride - } - if maxMemory > 0 { - qry.Add("max_memory", fmt.Sprintf("%dGB", maxMemory)) - } - - // Set threads limit - var threads int - if cfg.ThreadsOverride != 0 { - threads = cfg.ThreadsOverride - } else if cfg.CPU > 0 { - threads = cfg.CPU - } - if threads > 0 { // NOTE: threads=0 or threads=-1 means no limit - qry.Add("threads", strconv.Itoa(threads)) - } - // Set pool size poolSize := cfg.PoolSize - if qry.Has("rill_pool_size") { - // For backwards compatibility, we also support overriding the pool size via the DSN when "rill_pool_size" is a query argument. - - // Remove from query string (so not passed into DuckDB config) - val := qry.Get("rill_pool_size") - qry.Del("rill_pool_size") - - // Parse as integer - poolSize, err = strconv.Atoi(val) - if err != nil { - return nil, fmt.Errorf("could not parse dsn: 'rill_pool_size' is not an integer") - } - } - if poolSize == 0 && threads != 0 { - poolSize = threads - if cfg.CPU != 0 && cfg.CPU < poolSize { - poolSize = cfg.CPU - } - poolSize = min(poolSizeMax, poolSize) // Only enforce max pool size when inferred from threads/CPU + if poolSize == 0 && cfg.CPU != 0 { + poolSize = min(poolSizeMax, cfg.CPU) // Only enforce max pool size when inferred from CPU } poolSize = max(poolSizeMin, poolSize) // Always enforce min pool size cfg.PoolSize = poolSize + return cfg, nil +} - // useful for motherduck but safe to pass at initial connect - if !qry.Has("custom_user_agent") { - qry.Add("custom_user_agent", "rill") +func (c *config) readSettings() map[string]string { + readSettings := make(map[string]string) + if c.MemoryLimitGB > 0 { + readSettings["max_memory"] = fmt.Sprintf("%dGB", c.MemoryLimitGB) } - // Rebuild DuckDB DSN (which should be "path?key=val&...") - // this is required since spaces and other special characters are valid in db file path but invalid and hence encoded in URL - cfg.DSN = generateDSN(uri.Path, qry.Encode()) - - return cfg, nil + if c.CPU > 0 { + readSettings["threads"] = strconv.Itoa(c.CPU) + } + return readSettings } -func generateDSN(path, encodedQuery string) string { - if encodedQuery == "" { - return path +func (c *config) writeSettings() map[string]string { + writeSettings := make(map[string]string) + if c.MemoryLimitGBWrite > 0 { + writeSettings["max_memory"] = fmt.Sprintf("%dGB", c.MemoryLimitGBWrite) + } + if c.CPUWrite > 0 { + writeSettings["threads"] = strconv.Itoa(c.CPUWrite) } - return path + "?" + encodedQuery + // useful for motherduck but safe to pass at initial connect + writeSettings["custom_user_agent"] = "rill" + return writeSettings } diff --git a/runtime/drivers/duckdb/config_test.go b/runtime/drivers/duckdb/config_test.go index d3caa218107..f10fabe65f2 100644 --- a/runtime/drivers/duckdb/config_test.go +++ b/runtime/drivers/duckdb/config_test.go @@ -2,6 +2,7 @@ package duckdb import ( "context" + "fmt" "io/fs" "os" "path/filepath" @@ -15,76 +16,27 @@ import ( ) func TestConfig(t *testing.T) { - cfg, err := newConfig(map[string]any{}, "") + cfg, err := newConfig(map[string]any{}) require.NoError(t, err) - require.Equal(t, "?custom_user_agent=rill", cfg.DSN) require.Equal(t, 2, cfg.PoolSize) - cfg, err = newConfig(map[string]any{"dsn": ":memory:?memory_limit=2GB"}, "") + cfg, err = newConfig(map[string]any{"dsn": "", "cpu": 2}) require.NoError(t, err) - require.Equal(t, "?custom_user_agent=rill&memory_limit=2GB", cfg.DSN) + require.Equal(t, "2", cfg.readSettings()["threads"]) + require.Subset(t, cfg.writeSettings(), map[string]string{"custom_user_agent": "rill"}) require.Equal(t, 2, cfg.PoolSize) - cfg, err = newConfig(map[string]any{"dsn": "", "memory_limit_gb": "1", "cpu": 2}, "") - require.NoError(t, err) - require.Equal(t, "?custom_user_agent=rill&max_memory=1GB&threads=2", cfg.DSN) - require.Equal(t, 2, cfg.PoolSize) - require.Equal(t, true, cfg.ExtTableStorage) - - cfg, err = newConfig(map[string]any{}, "path/to") - require.NoError(t, err) - require.Equal(t, "path/to/main.db?custom_user_agent=rill", cfg.DSN) - require.Equal(t, "path/to/main.db", cfg.DBFilePath) - require.Equal(t, 2, cfg.PoolSize) - - cfg, err = newConfig(map[string]any{"pool_size": 10}, "path/to") - require.NoError(t, err) - require.Equal(t, "path/to/main.db?custom_user_agent=rill", cfg.DSN) - require.Equal(t, "path/to/main.db", cfg.DBFilePath) - require.Equal(t, 10, cfg.PoolSize) - - cfg, err = newConfig(map[string]any{"pool_size": "10"}, "path/to") + cfg, err = newConfig(map[string]any{"pool_size": 10}) require.NoError(t, err) require.Equal(t, 10, cfg.PoolSize) - cfg, err = newConfig(map[string]any{"dsn": "?rill_pool_size=4", "pool_size": "10"}, "path/to") + cfg, err = newConfig(map[string]any{"dsn": "duck.db", "memory_limit_gb": "8", "cpu": "2"}) require.NoError(t, err) - require.Equal(t, 4, cfg.PoolSize) - - cfg, err = newConfig(map[string]any{"dsn": "path/to/duck.db?rill_pool_size=10"}, "path/to") - require.NoError(t, err) - require.Equal(t, "path/to/duck.db?custom_user_agent=rill", cfg.DSN) - require.Equal(t, "path/to/duck.db", cfg.DBFilePath) - require.Equal(t, 10, cfg.PoolSize) - - cfg, err = newConfig(map[string]any{"dsn": "path/to/duck.db?max_memory=4GB&rill_pool_size=10"}, "path/to") - require.NoError(t, err) - require.Equal(t, "path/to/duck.db?custom_user_agent=rill&max_memory=4GB", cfg.DSN) - require.Equal(t, 10, cfg.PoolSize) - require.Equal(t, "path/to/duck.db", cfg.DBFilePath) - - _, err = newConfig(map[string]any{"dsn": "path/to/duck.db?max_memory=4GB", "pool_size": "abc"}, "path/to") - require.Error(t, err) - - cfg, err = newConfig(map[string]any{"dsn": "duck.db"}, "path/to") - require.NoError(t, err) - require.Equal(t, "duck.db", cfg.DBFilePath) - - cfg, err = newConfig(map[string]any{"dsn": "duck.db?rill_pool_size=10"}, "path/to") - require.NoError(t, err) - require.Equal(t, "duck.db", cfg.DBFilePath) - - cfg, err = newConfig(map[string]any{"dsn": "duck.db", "memory_limit_gb": "8", "cpu": "2"}, "path/to") - require.NoError(t, err) - require.Equal(t, "duck.db", cfg.DBFilePath) - require.Equal(t, "duck.db?custom_user_agent=rill&max_memory=8GB&threads=2", cfg.DSN) + require.Equal(t, "2", cfg.readSettings()["threads"]) + require.Equal(t, "", cfg.writeSettings()["threads"]) + require.Equal(t, "8GB", cfg.readSettings()["max_memory"]) + require.Equal(t, "", cfg.writeSettings()["max_memory"]) require.Equal(t, 2, cfg.PoolSize) - - cfg, err = newConfig(map[string]any{"dsn": "duck.db?max_memory=2GB&rill_pool_size=4"}, "path/to") - require.NoError(t, err) - require.Equal(t, "duck.db", cfg.DBFilePath) - require.Equal(t, "duck.db?custom_user_agent=rill&max_memory=2GB", cfg.DSN) - require.Equal(t, 4, cfg.PoolSize) } func Test_specialCharInPath(t *testing.T) { @@ -94,11 +46,8 @@ func Test_specialCharInPath(t *testing.T) { require.NoError(t, err) dbFile := filepath.Join(path, "st@g3's.db") - conn, err := Driver{}.Open("default", map[string]any{"path": dbFile, "memory_limit_gb": "4", "cpu": "1", "external_table_storage": false}, storage.MustNew(t.TempDir(), nil), activity.NewNoopClient(), zap.NewNop()) + conn, err := Driver{}.Open("default", map[string]any{"init_sql": fmt.Sprintf("ATTACH %s", safeSQLString(dbFile))}, storage.MustNew(t.TempDir(), nil), activity.NewNoopClient(), zap.NewNop()) require.NoError(t, err) - config := conn.(*connection).config - require.Equal(t, filepath.Join(path, "st@g3's.db?custom_user_agent=rill&max_memory=4GB&threads=1"), config.DSN) - require.Equal(t, 2, config.PoolSize) olap, ok := conn.AsOLAP("") require.True(t, ok) @@ -108,21 +57,3 @@ func Test_specialCharInPath(t *testing.T) { require.NoError(t, res.Close()) require.NoError(t, conn.Close()) } - -func TestOverrides(t *testing.T) { - cfgMap := map[string]any{"path": "duck.db", "memory_limit_gb": "4", "cpu": "2", "max_memory_gb_override": "2", "threads_override": "10", "external_table_storage": false} - handle, err := Driver{}.Open("default", cfgMap, storage.MustNew(t.TempDir(), nil), activity.NewNoopClient(), zap.NewNop()) - require.NoError(t, err) - - olap, ok := handle.AsOLAP("") - require.True(t, ok) - - res, err := olap.Execute(context.Background(), &drivers.Statement{Query: "SELECT value FROM duckdb_settings() WHERE name='max_memory'"}) - require.NoError(t, err) - require.True(t, res.Next()) - var mem string - require.NoError(t, res.Scan(&mem)) - require.NoError(t, res.Close()) - - require.Equal(t, "1.8 GiB", mem) -} diff --git a/runtime/drivers/duckdb/duckdb.go b/runtime/drivers/duckdb/duckdb.go index e402fa5c060..edb15421052 100644 --- a/runtime/drivers/duckdb/duckdb.go +++ b/runtime/drivers/duckdb/duckdb.go @@ -2,23 +2,16 @@ package duckdb import ( "context" - "database/sql/driver" "errors" "fmt" - "io/fs" + "log/slog" "net/url" - "os" "path/filepath" - "regexp" - "strconv" "strings" "sync" "time" - "github.com/XSAM/otelsql" - "github.com/c2h5oh/datasize" "github.com/jmoiron/sqlx" - "github.com/marcboeker/go-duckdb" "github.com/mitchellh/mapstructure" "github.com/rilldata/rill/runtime/drivers" "github.com/rilldata/rill/runtime/drivers/duckdb/extensions" @@ -27,10 +20,13 @@ import ( "github.com/rilldata/rill/runtime/pkg/duckdbsql" "github.com/rilldata/rill/runtime/pkg/observability" "github.com/rilldata/rill/runtime/pkg/priorityqueue" + "github.com/rilldata/rill/runtime/pkg/rduckdb" "github.com/rilldata/rill/runtime/storage" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" "go.uber.org/zap" + "go.uber.org/zap/exp/zapslog" + "gocloud.dev/blob" "golang.org/x/sync/semaphore" ) @@ -146,41 +142,11 @@ func (d Driver) Open(instanceID string, cfgMap map[string]any, st *storage.Clien logger.Warn("failed to install embedded DuckDB extensions, let DuckDB download them", zap.Error(err)) } - dataDir, err := st.DataDir() + cfg, err := newConfig(cfgMap) if err != nil { return nil, err } - cfg, err := newConfig(cfgMap, dataDir) - if err != nil { - return nil, err - } - logger.Debug("opening duckdb handle", zap.String("dsn", cfg.DSN)) - - // We've seen the DuckDB .wal and .tmp files grow to 100s of GBs in some cases. - // This prevents recovery after restarts since DuckDB hangs while trying to reprocess the files. - // This is a hacky solution that deletes the files (if they exist) before re-opening the DB. - // Generally, this should not lead to data loss since reconcile will bring the database back to the correct state. - if cfg.DBFilePath != "" { - // Always drop the .tmp directory - tmpPath := cfg.DBFilePath + ".tmp" - _ = os.RemoveAll(tmpPath) - - // Drop the .wal file if it's bigger than 100MB - walPath := cfg.DBFilePath + ".wal" - if stat, err := os.Stat(walPath); err == nil { - if stat.Size() >= 100*int64(datasize.MB) { - _ = os.Remove(walPath) - } - } - } - - if cfg.DBStoragePath != "" { - if err := os.MkdirAll(cfg.DBStoragePath, fs.ModePerm); err != nil && !errors.Is(err, fs.ErrExist) { - return nil, err - } - } - // See note in connection struct olapSemSize := cfg.PoolSize - 1 if olapSemSize < 1 { @@ -193,6 +159,7 @@ func (d Driver) Open(instanceID string, cfgMap map[string]any, st *storage.Clien config: cfg, logger: logger, activity: ac, + storage: st, metaSem: semaphore.NewWeighted(1), olapSem: priorityqueue.NewSemaphore(olapSemSize), longRunningSem: semaphore.NewWeighted(1), // Currently hard-coded to 1 @@ -203,43 +170,31 @@ func (d Driver) Open(instanceID string, cfgMap map[string]any, st *storage.Clien ctx: ctx, cancel: cancel, } + remote, ok, err := st.OpenBucket(context.Background()) + if err != nil { + return nil, err + } + if ok { + c.remote = remote + } // register a callback to add a gauge on number of connections in use per db - attrs := []attribute.KeyValue{attribute.String("db", c.config.DBFilePath)} + attrs := []attribute.KeyValue{attribute.String("instance_id", instanceID)} c.registration = observability.Must(meter.RegisterCallback(func(ctx context.Context, observer metric.Observer) error { observer.ObserveInt64(connectionsInUse, int64(c.dbConnCount), metric.WithAttributes(attrs...)) return nil }, connectionsInUse)) // Open the DB - err = c.reopenDB() + err = c.reopenDB(context.Background()) if err != nil { - if c.config.ErrorOnIncompatibleVersion || !strings.Contains(err.Error(), "created with an older, incompatible version of Rill") { - return nil, err + if remote != nil { + _ = remote.Close() } - - c.logger.Debug("Resetting .db file because it was created with an older, incompatible version of Rill") - - tmpPath := cfg.DBFilePath + ".tmp" - _ = os.RemoveAll(tmpPath) - walPath := cfg.DBFilePath + ".wal" - _ = os.Remove(walPath) - _ = os.Remove(cfg.DBFilePath) - - // reopen connection again - if err := c.reopenDB(); err != nil { - return nil, err + // Check for another process currently accessing the DB + if strings.Contains(err.Error(), "Could not set lock on file") { + return nil, fmt.Errorf("failed to open database (is Rill already running?): %w", err) } - } - - // Return nice error for old macOS versions - conn, err := c.db.Connx(context.Background()) - if err != nil && strings.Contains(err.Error(), "Symbol not found") { - fmt.Printf("Your version of macOS is not supported. Please upgrade to the latest major release of macOS. See this link for details: https://support.apple.com/en-in/macos/upgrade") - os.Exit(1) - } else if err == nil { - conn.Close() - } else { return nil, err } @@ -305,8 +260,8 @@ func (d Driver) TertiarySourceConnectors(ctx context.Context, src map[string]any type connection struct { instanceID string // do not use directly it can also be nil or closed - // use acquireOLAPConn/acquireMetaConn - db *sqlx.DB + // use acquireOLAPConn/acquireMetaConn for select and acquireDB for write queries + db rduckdb.DB // driverConfig is input config passed during Open driverConfig map[string]any driverName string @@ -314,6 +269,8 @@ type connection struct { config *config logger *zap.Logger activity *activity.Client + storage *storage.Client + remote *blob.Bucket // This driver may issue both OLAP and "meta" queries (like catalog info) against DuckDB. // Meta queries are usually fast, but OLAP queries may take a long time. To enable predictable parallel performance, // we gate queries with semaphores that limits the number of concurrent queries of each type. @@ -325,9 +282,6 @@ type connection struct { // The OLAP interface additionally provides an option to limit the number of long-running queries, as designated by the caller. // longRunningSem enforces this limitation. longRunningSem *semaphore.Weighted - // The OLAP interface also provides an option to acquire a connection "transactionally". - // We've run into issues with DuckDB freezing up on transactions, so we just use a lock for now to serialize them (inconsistency in case of crashes is acceptable). - txMu sync.RWMutex // If DuckDB encounters a fatal error, all queries will fail until the DB has been reopened. // When dbReopen is set to true, dbCond will be used to stop acquisition of new connections, // and then when dbConnCount becomes 0, the DB will be reopened and dbReopen set to false again. @@ -377,7 +331,13 @@ func (c *connection) Config() map[string]any { func (c *connection) Close() error { c.cancel() _ = c.registration.Unregister() - return c.db.Close() + if c.remote != nil { + _ = c.remote.Close() + } + if c.db != nil { + return c.db.Close() + } + return nil } // AsRegistry Registry implements drivers.Connection. @@ -415,12 +375,6 @@ func (c *connection) AsObjectStore() (drivers.ObjectStore, bool) { return nil, false } -// AsSQLStore implements drivers.Connection. -// Use OLAPStore instead. -func (c *connection) AsSQLStore() (drivers.SQLStore, bool) { - return nil, false -} - // AsModelExecutor implements drivers.Handle. func (c *connection) AsModelExecutor(instanceID string, opts *drivers.ModelExecutorOptions) (drivers.ModelExecutor, bool) { if opts.InputHandle == c && opts.OutputHandle == c { @@ -458,13 +412,15 @@ func (c *connection) AsTransporter(from, to drivers.Handle) (drivers.Transporter olap, _ := to.(*connection) if c == to { if from == to { - return NewDuckDBToDuckDB(olap, c.logger), true + return newDuckDBToDuckDB(c, "duckdb", c.logger), true } - if from.Driver() == "motherduck" { - return NewMotherduckToDuckDB(from, olap, c.logger), true - } - if store, ok := from.AsSQLStore(); ok { - return NewSQLStoreToDuckDB(store, olap, c.logger), true + switch from.Driver() { + case "motherduck": + return newMotherduckToDuckDB(from, olap, c.logger), true + case "postgres": + return newDuckDBToDuckDB(c, "postgres", c.logger), true + case "mysql": + return newDuckDBToDuckDB(c, "mysql", c.logger), true } if store, ok := from.AsWarehouse(); ok { return NewWarehouseToDuckDB(store, olap, c.logger), true @@ -494,7 +450,7 @@ func (c *connection) AsNotifier(properties map[string]any) (drivers.Notifier, er } // reopenDB opens the DuckDB handle anew. If c.db is already set, it closes the existing handle first. -func (c *connection) reopenDB() error { +func (c *connection) reopenDB(ctx context.Context) error { // If c.db is already open, close it first if c.db != nil { err := c.db.Close() @@ -529,10 +485,18 @@ func (c *connection) reopenDB() error { "SET old_implicit_casting = true", // Implicit Cast to VARCHAR ) + dataDir, err := c.storage.DataDir() + if err != nil { + return err + } + // We want to set preserve_insertion_order=false in hosted environments only (where source data is never viewed directly). Setting it reduces batch data ingestion time by ~40%. // Hack: Using AllowHostAccess as a proxy indicator for a hosted environment. if !c.config.AllowHostAccess { - bootQueries = append(bootQueries, "SET preserve_insertion_order TO false") + bootQueries = append(bootQueries, + "SET preserve_insertion_order TO false", + fmt.Sprintf("SET secret_directory = %s", safeSQLString(filepath.Join(dataDir, ".duckdb", "secrets"))), + ) } // Add init SQL if provided @@ -540,108 +504,20 @@ func (c *connection) reopenDB() error { bootQueries = append(bootQueries, c.config.InitSQL) } - // DuckDB extensions need to be loaded separately on each connection, but the built-in connection pool in database/sql doesn't enable that. - // So we use go-duckdb's custom connector to pass a callback that it invokes for each new connection. - connector, err := duckdb.NewConnector(c.config.DSN, func(execer driver.ExecerContext) error { - for _, qry := range bootQueries { - _, err := execer.ExecContext(context.Background(), qry, nil) - if err != nil && strings.Contains(err.Error(), "Failed to download extension") { - // Retry using another mirror. Based on: https://github.com/duckdb/duckdb/issues/9378 - _, err = execer.ExecContext(context.Background(), qry+" FROM 'http://nightly-extensions.duckdb.org'", nil) - } - if err != nil { - return err - } - } - return nil - }) - if err != nil { - // Check for using incompatible database files - if strings.Contains(err.Error(), "Trying to read a database file with version number") { - return fmt.Errorf("database file %q was created with an older, incompatible version of Rill (please remove it and try again)", c.config.DSN) - } - - // Check for another process currently accessing the DB - if strings.Contains(err.Error(), "Could not set lock on file") { - return fmt.Errorf("failed to open database (is Rill already running?): %w", err) - } - - return err - } - // Create new DB - sqlDB := otelsql.OpenDB(connector) - db := sqlx.NewDb(sqlDB, "duckdb") - db.SetMaxOpenConns(c.config.PoolSize) - c.db = db - - if !c.config.ExtTableStorage { - return nil - } - - conn, err := db.Connx(context.Background()) - if err != nil { - return err - } - defer conn.Close() - - c.logLimits(conn) - - // 2023-12-11: Hail mary for solving this issue: https://github.com/duckdblabs/rilldata/issues/6. - // Forces DuckDB to create catalog entries for the information schema up front (they are normally created lazily). - // Can be removed if the issue persists. - _, err = conn.ExecContext(context.Background(), ` - select - coalesce(t.table_catalog, current_database()) as "database", - t.table_schema as "schema", - t.table_name as "name", - t.table_type as "type", - array_agg(c.column_name order by c.ordinal_position) as "column_names", - array_agg(c.data_type order by c.ordinal_position) as "column_types", - array_agg(c.is_nullable = 'YES' order by c.ordinal_position) as "column_nullable" - from information_schema.tables t - join information_schema.columns c on t.table_schema = c.table_schema and t.table_name = c.table_name - group by 1, 2, 3, 4 - order by 1, 2, 3, 4 - `) - if err != nil { - return err - } - - // List the directories directly in the external storage directory - // Load the version.txt from each sub-directory - // If version.txt is found, attach only the .db file matching the version.txt. - // If attach fails, log the error and delete the version.txt and .db file (e.g. might be DuckDB version change) - entries, err := os.ReadDir(c.config.DBStoragePath) - if err != nil { - return err - } - for _, entry := range entries { - if !entry.IsDir() { - continue - } - path := filepath.Join(c.config.DBStoragePath, entry.Name()) - version, exist, err := c.tableVersion(entry.Name()) - if err != nil { - c.logger.Error("error in fetching db version", zap.String("table", entry.Name()), zap.Error(err)) - _ = os.RemoveAll(path) - continue - } - if !exist { - _ = os.RemoveAll(path) - continue - } - - dbFile := filepath.Join(path, fmt.Sprintf("%s.db", version)) - db := dbName(entry.Name(), version) - _, err = conn.ExecContext(context.Background(), fmt.Sprintf("ATTACH %s AS %s", safeSQLString(dbFile), safeSQLName(db))) - if err != nil { - c.logger.Error("attach failed clearing db file", zap.String("db", dbFile), zap.Error(err)) - _, _ = conn.ExecContext(context.Background(), fmt.Sprintf("DROP VIEW IF EXISTS %s", safeSQLName(entry.Name()))) - _ = os.RemoveAll(path) - } - } - return nil + logger := slog.New(zapslog.NewHandler(c.logger.Core(), &zapslog.HandlerOptions{ + AddSource: true, + })) + c.db, err = rduckdb.NewDB(ctx, &rduckdb.DBOptions{ + LocalPath: dataDir, + Remote: c.remote, + ReadSettings: c.config.readSettings(), + WriteSettings: c.config.writeSettings(), + InitQueries: bootQueries, + Logger: logger, + OtelAttributes: []attribute.KeyValue{attribute.String("instance_id", c.instanceID)}, + }) + return err } // acquireMetaConn gets a connection from the pool for "meta" queries like catalog and information schema (i.e. fast queries). @@ -660,7 +536,7 @@ func (c *connection) acquireMetaConn(ctx context.Context) (*sqlx.Conn, func() er } // Get new conn - conn, releaseConn, err := c.acquireConn(ctx, false) + conn, releaseConn, err := c.acquireReadConnection(ctx) if err != nil { c.metaSem.Release(1) return nil, nil, err @@ -678,7 +554,7 @@ func (c *connection) acquireMetaConn(ctx context.Context) (*sqlx.Conn, func() er // acquireOLAPConn gets a connection from the pool for OLAP queries (i.e. slow queries). // It returns a function that puts the connection back in the pool (if applicable). -func (c *connection) acquireOLAPConn(ctx context.Context, priority int, longRunning, tx bool) (*sqlx.Conn, func() error, error) { +func (c *connection) acquireOLAPConn(ctx context.Context, priority int, longRunning bool) (*sqlx.Conn, func() error, error) { // Try to get conn from context (means the call is wrapped in WithConnection) conn := connFromContext(ctx) if conn != nil { @@ -703,7 +579,7 @@ func (c *connection) acquireOLAPConn(ctx context.Context, priority int, longRunn } // Get new conn - conn, releaseConn, err := c.acquireConn(ctx, tx) + conn, releaseConn, err := c.acquireReadConnection(ctx) if err != nil { c.olapSem.Release() if longRunning { @@ -725,9 +601,32 @@ func (c *connection) acquireOLAPConn(ctx context.Context, priority int, longRunn return conn, release, nil } -// acquireConn returns a DuckDB connection. It should only be used internally in acquireMetaConn and acquireOLAPConn. -// acquireConn implements the connection tracking and DB reopening logic described in the struct definition for connection. -func (c *connection) acquireConn(ctx context.Context, tx bool) (*sqlx.Conn, func() error, error) { +// acquireReadConnection is a helper function to acquire a read connection from rduckdb. +// Do not use this function directly for OLAP queries. Use acquireOLAPConn, acquireMetaConn instead. +func (c *connection) acquireReadConnection(ctx context.Context) (*sqlx.Conn, func() error, error) { + db, releaseDB, err := c.acquireDB() + if err != nil { + return nil, nil, err + } + + conn, releaseConn, err := db.AcquireReadConnection(ctx) + if err != nil { + _ = releaseDB() + return nil, nil, err + } + + release := func() error { + err := releaseConn() + return errors.Join(err, releaseDB()) + } + return conn, release, nil +} + +// acquireDB returns rduckDB handle. +// acquireDB implements the connection tracking and DB reopening logic described in the struct definition for connection. +// It should not be used directly for select queries. For select queries use acquireOLAPConn and acquireMetaConn. +// It should only be used for write queries. +func (c *connection) acquireDB() (rduckdb.DB, func() error, error) { c.dbCond.L.Lock() for { if c.dbErr != nil { @@ -743,36 +642,6 @@ func (c *connection) acquireConn(ctx context.Context, tx bool) (*sqlx.Conn, func c.dbConnCount++ c.dbCond.L.Unlock() - // Poor man's transaction support – see struct docstring for details. - if tx { - c.txMu.Lock() - - // When tx is true, and the database is backed by a file, we reopen the database to ensure only one DuckDB connection is open. - // This avoids the following issue: https://github.com/duckdb/duckdb/issues/9150 - if c.config.DBFilePath != "" { - err := c.reopenDB() - if err != nil { - c.txMu.Unlock() - return nil, nil, err - } - } - } else { - c.txMu.RLock() - } - releaseTx := func() { - if tx { - c.txMu.Unlock() - } else { - c.txMu.RUnlock() - } - } - - conn, err := c.db.Connx(ctx) - if err != nil { - releaseTx() - return nil, nil, err - } - c.connTimesMu.Lock() connID := c.nextConnID c.nextConnID++ @@ -780,29 +649,38 @@ func (c *connection) acquireConn(ctx context.Context, tx bool) (*sqlx.Conn, func c.connTimesMu.Unlock() release := func() error { - err := conn.Close() c.connTimesMu.Lock() delete(c.connTimes, connID) c.connTimesMu.Unlock() - releaseTx() c.dbCond.L.Lock() c.dbConnCount-- if c.dbConnCount == 0 && c.dbReopen { - c.dbReopen = false - err = c.reopenDB() - if err == nil { - c.logger.Debug("reopened DuckDB successfully") - } else { - c.logger.Debug("reopen of DuckDB failed - the handle is now permanently locked", zap.Error(err)) - } - c.dbErr = err - c.dbCond.Broadcast() + c.triggerReopen() } c.dbCond.L.Unlock() - return err + return nil } + return c.db, release, nil +} - return conn, release, nil +func (c *connection) triggerReopen() { + go func() { + c.dbCond.L.Lock() + defer c.dbCond.L.Unlock() + if !c.dbReopen || c.dbConnCount == 0 { + c.logger.Error("triggerReopen called but should not reopen", zap.Bool("dbReopen", c.dbReopen), zap.Int("dbConnCount", c.dbConnCount)) + return + } + c.dbReopen = false + err := c.reopenDB(c.ctx) + if err != nil { + if !errors.Is(err, context.Canceled) { + c.logger.Error("reopen of DuckDB failed - the handle is now permanently locked", zap.Error(err)) + } + } + c.dbErr = err + c.dbCond.Broadcast() + }() } // checkErr marks the DB for reopening if the error is an internal DuckDB error. @@ -832,71 +710,8 @@ func (c *connection) periodicallyEmitStats(d time.Duration) { for { select { case <-statTicker.C: - estimatedDBSize := c.estimateSize(false) + estimatedDBSize := c.estimateSize() c.activity.RecordMetric(c.ctx, "duckdb_estimated_size_bytes", float64(estimatedDBSize)) - - // NOTE :: running CALL pragma_database_size() while duckdb is ingesting data is causing the WAL file to explode. - // Commenting the below code for now. Verify with next duckdb release - - // // Motherduck driver doesn't provide pragma stats - // if c.driverName == "motherduck" { - // continue - // } - - // var stat dbStat - // // Obtain a connection, query, release - // err := func() error { - // conn, release, err := c.acquireMetaConn(c.ctx) - // if err != nil { - // return err - // } - // defer func() { _ = release() }() - // err = conn.GetContext(c.ctx, &stat, "CALL pragma_database_size()") - // return err - // }() - // if err != nil { - // c.logger.Error("couldn't query DuckDB stats", zap.Error(err)) - // continue - // } - - // // Emit collected stats as activity events - // commonDims := []attribute.KeyValue{ - // attribute.String("duckdb.name", stat.DatabaseName), - // } - - // dbSize, err := humanReadableSizeToBytes(stat.DatabaseSize) - // if err != nil { - // c.logger.Error("couldn't convert duckdb size to bytes", zap.Error(err)) - // } else { - // c.activity.RecordMetric(c.ctx, "duckdb_size_bytes", dbSize, commonDims...) - // } - - // walSize, err := humanReadableSizeToBytes(stat.WalSize) - // if err != nil { - // c.logger.Error("couldn't convert duckdb wal size to bytes", zap.Error(err)) - // } else { - // c.activity.RecordMetric(c.ctx, "duckdb_wal_size_bytes", walSize, commonDims...) - // } - - // memoryUsage, err := humanReadableSizeToBytes(stat.MemoryUsage) - // if err != nil { - // c.logger.Error("couldn't convert duckdb memory usage to bytes", zap.Error(err)) - // } else { - // c.activity.RecordMetric(c.ctx, "duckdb_memory_usage_bytes", memoryUsage, commonDims...) - // } - - // memoryLimit, err := humanReadableSizeToBytes(stat.MemoryLimit) - // if err != nil { - // c.logger.Error("couldn't convert duckdb memory limit to bytes", zap.Error(err)) - // } else { - // c.activity.RecordMetric(c.ctx, "duckdb_memory_limit_bytes", memoryLimit, commonDims...) - // } - - // c.activity.RecordMetric(c.ctx, "duckdb_block_size_bytes", float64(stat.BlockSize), commonDims...) - // c.activity.RecordMetric(c.ctx, "duckdb_total_blocks", float64(stat.TotalBlocks), commonDims...) - // c.activity.RecordMetric(c.ctx, "duckdb_free_blocks", float64(stat.FreeBlocks), commonDims...) - // c.activity.RecordMetric(c.ctx, "duckdb_used_blocks", float64(stat.UsedBlocks), commonDims...) - case <-c.ctx.Done(): statTicker.Stop() return @@ -929,77 +744,3 @@ func (c *connection) periodicallyCheckConnDurations(d time.Duration) { } } } - -func (c *connection) logLimits(conn *sqlx.Conn) { - row := conn.QueryRowContext(context.Background(), "SELECT value FROM duckdb_settings() WHERE name='max_memory'") - var memory string - _ = row.Scan(&memory) - - row = conn.QueryRowContext(context.Background(), "SELECT value FROM duckdb_settings() WHERE name='threads'") - var threads string - _ = row.Scan(&threads) - - c.logger.Debug("duckdb limits", zap.String("memory", memory), zap.String("threads", threads)) -} - -// fatalInternalError logs a critical internal error and exits the process. -// This is used for errors that are completely unrecoverable. -// Ideally, we should refactor to cleanup/reopen/rebuild so that we don't need this. -func (c *connection) fatalInternalError(err error) { - c.logger.Fatal("duckdb: critical internal error", zap.Error(err)) -} - -// Regex to parse human-readable size returned by DuckDB -// nolint -var humanReadableSizeRegex = regexp.MustCompile(`^([\d.]+)\s*(\S+)$`) - -// Reversed logic of StringUtil::BytesToHumanReadableString -// see https://github.com/cran/duckdb/blob/master/src/duckdb/src/common/string_util.cpp#L157 -// Examples: 1 bytes, 2 bytes, 1KB, 1MB, 1TB, 1PB -// nolint -func humanReadableSizeToBytes(sizeStr string) (float64, error) { - var multiplier float64 - - match := humanReadableSizeRegex.FindStringSubmatch(sizeStr) - - if match == nil { - return 0, fmt.Errorf("invalid size format: '%s'", sizeStr) - } - - sizeFloat, err := strconv.ParseFloat(match[1], 64) - if err != nil { - return 0, err - } - - switch match[2] { - case "byte", "bytes": - multiplier = 1 - case "KB": - multiplier = 1000 - case "MB": - multiplier = 1000 * 1000 - case "GB": - multiplier = 1000 * 1000 * 1000 - case "TB": - multiplier = 1000 * 1000 * 1000 * 1000 - case "PB": - multiplier = 1000 * 1000 * 1000 * 1000 * 1000 - default: - return 0, fmt.Errorf("unknown size unit '%s' in '%s'", match[2], sizeStr) - } - - return sizeFloat * multiplier, nil -} - -// nolint -type dbStat struct { - DatabaseName string `db:"database_name"` - DatabaseSize string `db:"database_size"` - BlockSize int64 `db:"block_size"` - TotalBlocks int64 `db:"total_blocks"` - UsedBlocks int64 `db:"used_blocks"` - FreeBlocks int64 `db:"free_blocks"` - WalSize string `db:"wal_size"` - MemoryUsage string `db:"memory_usage"` - MemoryLimit string `db:"memory_limit"` -} diff --git a/runtime/drivers/duckdb/duckdb_test.go b/runtime/drivers/duckdb/duckdb_test.go index 6b8510b89f4..213f330baf7 100644 --- a/runtime/drivers/duckdb/duckdb_test.go +++ b/runtime/drivers/duckdb/duckdb_test.go @@ -3,7 +3,6 @@ package duckdb import ( "context" "database/sql" - "path/filepath" "sync" "testing" "time" @@ -18,9 +17,7 @@ import ( func TestNoFatalErr(t *testing.T) { // NOTE: Using this issue to create a fatal error: https://github.com/duckdb/duckdb/issues/7905 - dsn := filepath.Join(t.TempDir(), "tmp.db") - - handle, err := Driver{}.Open("default", map[string]any{"path": dsn, "pool_size": 2, "external_table_storage": false}, storage.MustNew(t.TempDir(), nil), activity.NewNoopClient(), zap.NewNop()) + handle, err := Driver{}.Open("default", map[string]any{"pool_size": 2}, storage.MustNew(t.TempDir(), nil), activity.NewNoopClient(), zap.NewNop()) require.NoError(t, err) olap, ok := handle.AsOLAP("") @@ -80,9 +77,7 @@ func TestNoFatalErr(t *testing.T) { func TestNoFatalErrConcurrent(t *testing.T) { // NOTE: Using this issue to create a fatal error: https://github.com/duckdb/duckdb/issues/7905 - dsn := filepath.Join(t.TempDir(), "tmp.db") - - handle, err := Driver{}.Open("default", map[string]any{"path": dsn, "pool_size": 3, "external_table_storage": false}, storage.MustNew(t.TempDir(), nil), activity.NewNoopClient(), zap.NewNop()) + handle, err := Driver{}.Open("default", map[string]any{"pool_size": 2}, storage.MustNew(t.TempDir(), nil), activity.NewNoopClient(), zap.NewNop()) require.NoError(t, err) olap, ok := handle.AsOLAP("") @@ -139,7 +134,7 @@ func TestNoFatalErrConcurrent(t *testing.T) { LEFT JOIN d ON b.b12 = d.d1 WHERE d.d2 IN (''); ` - err1 = olap.WithConnection(context.Background(), 0, false, false, func(ctx, ensuredCtx context.Context, _ *sql.Conn) error { + err1 = olap.WithConnection(context.Background(), 0, false, func(ctx, ensuredCtx context.Context, _ *sql.Conn) error { time.Sleep(500 * time.Millisecond) return olap.Exec(ctx, &drivers.Statement{Query: qry}) }) @@ -152,7 +147,7 @@ func TestNoFatalErrConcurrent(t *testing.T) { var err2 error go func() { qry := `SELECT * FROM a;` - err2 = olap.WithConnection(context.Background(), 0, false, false, func(ctx, ensuredCtx context.Context, _ *sql.Conn) error { + err2 = olap.WithConnection(context.Background(), 0, false, func(ctx, ensuredCtx context.Context, _ *sql.Conn) error { time.Sleep(1000 * time.Millisecond) return olap.Exec(ctx, &drivers.Statement{Query: qry}) }) @@ -166,7 +161,7 @@ func TestNoFatalErrConcurrent(t *testing.T) { go func() { time.Sleep(250 * time.Millisecond) qry := `SELECT * FROM a;` - err3 = olap.WithConnection(context.Background(), 0, false, false, func(ctx, ensuredCtx context.Context, _ *sql.Conn) error { + err3 = olap.WithConnection(context.Background(), 0, false, func(ctx, ensuredCtx context.Context, _ *sql.Conn) error { return olap.Exec(ctx, &drivers.Statement{Query: qry}) }) wg.Done() @@ -184,38 +179,3 @@ func TestNoFatalErrConcurrent(t *testing.T) { err = handle.Close() require.NoError(t, err) } - -func TestHumanReadableSizeToBytes(t *testing.T) { - tests := []struct { - input string - expected float64 - shouldErr bool - }{ - {"1 byte", 1, false}, - {"2 bytes", 2, false}, - {"1KB", 1000, false}, - {"1.5KB", 1500, false}, - {"1MB", 1000 * 1000, false}, - {"2.5MB", 2.5 * 1000 * 1000, false}, - {"1GB", 1000 * 1000 * 1000, false}, - {"1.5GB", 1.5 * 1000 * 1000 * 1000, false}, - {"1TB", 1000 * 1000 * 1000 * 1000, false}, - {"1.5TB", 1.5 * 1000 * 1000 * 1000 * 1000, false}, - {"1PB", 1000 * 1000 * 1000 * 1000 * 1000, false}, - {"1.5PB", 1.5 * 1000 * 1000 * 1000 * 1000 * 1000, false}, - {"invalid", 0, true}, - {"123invalid", 0, true}, - {"123 ZZ", 0, true}, - } - - for _, tt := range tests { - result, err := humanReadableSizeToBytes(tt.input) - if (err != nil) != tt.shouldErr { - t.Errorf("expected error: %v, got error: %v for input: %s", tt.shouldErr, err, tt.input) - } - - if !tt.shouldErr && result != tt.expected { - t.Errorf("expected: %v, got: %v for input: %s", tt.expected, result, tt.input) - } - } -} diff --git a/runtime/drivers/duckdb/information_schema.go b/runtime/drivers/duckdb/information_schema.go index 9e1168ec2e1..3b780f0a067 100644 --- a/runtime/drivers/duckdb/information_schema.go +++ b/runtime/drivers/duckdb/information_schema.go @@ -43,7 +43,7 @@ func (i informationSchema) All(ctx context.Context, like string) ([]*drivers.Tab array_agg(c.is_nullable = 'YES' order by c.ordinal_position) as "column_nullable" from information_schema.tables t join information_schema.columns c on t.table_schema = c.table_schema and t.table_name = c.table_name - where database = current_database() and t.table_schema = 'main' + where database = current_database() and t.table_schema = current_schema() %s group by 1, 2, 3, 4 order by 1, 2, 3, 4 @@ -81,7 +81,7 @@ func (i informationSchema) Lookup(ctx context.Context, db, schema, name string) array_agg(c.is_nullable = 'YES' order by c.ordinal_position) as "column_nullable" from information_schema.tables t join information_schema.columns c on t.table_schema = c.table_schema and t.table_name = c.table_name - where database = current_database() and t.table_schema = 'main' and lower(t.table_name) = lower(?) + where database = current_database() and t.table_schema = current_schema() and lower(t.table_name) = lower(?) group by 1, 2, 3, 4 order by 1, 2, 3, 4 ` @@ -199,7 +199,7 @@ func databaseTypeToPB(dbt string, nullable bool) (*runtimev1.Type, error) { case "TIME WITH TIME ZONE": t.Code = runtimev1.Type_CODE_TIME case "INTERVAL": - t.Code = runtimev1.Type_CODE_UNSPECIFIED // TODO - Consider adding interval type + t.Code = runtimev1.Type_CODE_INTERVAL case "HUGEINT": t.Code = runtimev1.Type_CODE_INT128 case "VARCHAR": diff --git a/runtime/drivers/duckdb/information_schema_test.go b/runtime/drivers/duckdb/information_schema_test.go index 42e6eaa1036..d590079a464 100644 --- a/runtime/drivers/duckdb/information_schema_test.go +++ b/runtime/drivers/duckdb/information_schema_test.go @@ -13,9 +13,10 @@ func TestInformationSchemaAll(t *testing.T) { conn := prepareConn(t) olap, _ := conn.AsOLAP("") - err := olap.Exec(context.Background(), &drivers.Statement{ - Query: "CREATE VIEW model as (select 1, 2, 3)", - }) + opts := &drivers.CreateTableOptions{ + View: true, + } + err := olap.CreateTableAsSelect(context.Background(), "model", "select 1, 2, 3", opts) require.NoError(t, err) tables, err := olap.InformationSchema().All(context.Background(), "") @@ -39,9 +40,8 @@ func TestInformationSchemaAllLike(t *testing.T) { conn := prepareConn(t) olap, _ := conn.AsOLAP("") - err := olap.Exec(context.Background(), &drivers.Statement{ - Query: "CREATE VIEW model as (select 1, 2, 3)", - }) + opts := &drivers.CreateTableOptions{View: true} + err := olap.CreateTableAsSelect(context.Background(), "model", "select 1, 2, 3", opts) require.NoError(t, err) tables, err := olap.InformationSchema().All(context.Background(), "%odel") @@ -49,7 +49,7 @@ func TestInformationSchemaAllLike(t *testing.T) { require.Equal(t, 1, len(tables)) require.Equal(t, "model", tables[0].Name) - tables, err = olap.InformationSchema().All(context.Background(), "%main.model%") + tables, err = olap.InformationSchema().All(context.Background(), "%model%") require.NoError(t, err) require.Equal(t, 1, len(tables)) require.Equal(t, "model", tables[0].Name) @@ -60,9 +60,8 @@ func TestInformationSchemaLookup(t *testing.T) { olap, _ := conn.AsOLAP("") ctx := context.Background() - err := olap.Exec(ctx, &drivers.Statement{ - Query: "CREATE VIEW model as (select 1, 2, 3)", - }) + opts := &drivers.CreateTableOptions{View: true} + err := olap.CreateTableAsSelect(context.Background(), "model", "select 1, 2, 3", opts) require.NoError(t, err) table, err := olap.InformationSchema().Lookup(ctx, "", "", "foo") diff --git a/runtime/drivers/duckdb/model_executor_localfile_self.go b/runtime/drivers/duckdb/model_executor_localfile_self.go index 3cad5d25251..17282ef8056 100644 --- a/runtime/drivers/duckdb/model_executor_localfile_self.go +++ b/runtime/drivers/duckdb/model_executor_localfile_self.go @@ -70,9 +70,7 @@ func (e *localFileToSelfExecutor) Execute(ctx context.Context, opts *drivers.Mod if opts.Env.StageChanges { stagingTableName = stagingTableNameFor(tableName) } - if t, err := e.c.InformationSchema().Lookup(ctx, "", "", stagingTableName); err == nil { - _ = e.c.DropTable(ctx, stagingTableName, t.View) - } + _ = e.c.DropTable(ctx, stagingTableName) // get the local file path localPaths, err := e.from.FilePaths(ctx, opts.InputProperties) @@ -93,9 +91,9 @@ func (e *localFileToSelfExecutor) Execute(ctx context.Context, opts *drivers.Mod } // create the table - err = e.c.CreateTableAsSelect(ctx, stagingTableName, asView, "SELECT * FROM "+from, nil) + err = e.c.CreateTableAsSelect(ctx, stagingTableName, "SELECT * FROM "+from, &drivers.CreateTableOptions{View: asView}) if err != nil { - _ = e.c.DropTable(ctx, stagingTableName, asView) + _ = e.c.DropTable(ctx, stagingTableName) return nil, fmt.Errorf("failed to create model: %w", err) } diff --git a/runtime/drivers/duckdb/model_executor_self.go b/runtime/drivers/duckdb/model_executor_self.go index f15232d7d74..84b0d93c427 100644 --- a/runtime/drivers/duckdb/model_executor_self.go +++ b/runtime/drivers/duckdb/model_executor_self.go @@ -64,14 +64,17 @@ func (e *selfToSelfExecutor) Execute(ctx context.Context, opts *drivers.ModelExe if opts.Env.StageChanges { stagingTableName = stagingTableNameFor(tableName) } - if t, err := olap.InformationSchema().Lookup(ctx, "", "", stagingTableName); err == nil { - _ = olap.DropTable(ctx, stagingTableName, t.View) - } + _ = olap.DropTable(ctx, stagingTableName) // Create the table - err := olap.CreateTableAsSelect(ctx, stagingTableName, asView, inputProps.SQL, nil) + createTableOpts := &drivers.CreateTableOptions{ + View: asView, + BeforeCreate: inputProps.PreExec, + AfterCreate: inputProps.PostExec, + } + err := olap.CreateTableAsSelect(ctx, stagingTableName, inputProps.SQL, createTableOpts) if err != nil { - _ = olap.DropTable(ctx, stagingTableName, asView) + _ = olap.DropTable(ctx, stagingTableName) return nil, fmt.Errorf("failed to create model: %w", err) } @@ -84,7 +87,15 @@ func (e *selfToSelfExecutor) Execute(ctx context.Context, opts *drivers.ModelExe } } else { // Insert into the table - err := olap.InsertTableAsSelect(ctx, tableName, inputProps.SQL, false, true, outputProps.IncrementalStrategy, outputProps.UniqueKey) + insertTableOpts := &drivers.InsertTableOptions{ + BeforeInsert: inputProps.PreExec, + AfterInsert: inputProps.PostExec, + ByName: false, + InPlace: true, + Strategy: outputProps.IncrementalStrategy, + UniqueKey: outputProps.UniqueKey, + } + err := olap.InsertTableAsSelect(ctx, tableName, inputProps.SQL, insertTableOpts) if err != nil { return nil, fmt.Errorf("failed to incrementally insert into table: %w", err) } diff --git a/runtime/drivers/duckdb/model_executor_warehouse_self.go b/runtime/drivers/duckdb/model_executor_warehouse_self.go index ecccbc78850..ccc3c216ca1 100644 --- a/runtime/drivers/duckdb/model_executor_warehouse_self.go +++ b/runtime/drivers/duckdb/model_executor_warehouse_self.go @@ -56,15 +56,13 @@ func (e *warehouseToSelfExecutor) Execute(ctx context.Context, opts *drivers.Mod } // NOTE: This intentionally drops the end table if not staging changes. - if t, err := olap.InformationSchema().Lookup(ctx, "", "", stagingTableName); err == nil { - _ = olap.DropTable(ctx, stagingTableName, t.View) - } + _ = olap.DropTable(ctx, stagingTableName) } err := e.queryAndInsert(ctx, opts, olap, stagingTableName, outputProps) if err != nil { if !opts.IncrementalRun { - _ = olap.DropTable(ctx, stagingTableName, false) + _ = olap.DropTable(ctx, stagingTableName) } return nil, err } @@ -113,8 +111,7 @@ func (e *warehouseToSelfExecutor) queryAndInsert(ctx context.Context, opts *driv for { files, err := iter.Next() if err != nil { - // TODO: Why is this not just one error? - if errors.Is(err, io.EOF) || errors.Is(err, drivers.ErrNoRows) || errors.Is(err, drivers.ErrIteratorDone) { + if errors.Is(err, io.EOF) || errors.Is(err, drivers.ErrNoRows) { break } return err @@ -132,7 +129,13 @@ func (e *warehouseToSelfExecutor) queryAndInsert(ctx context.Context, opts *driv qry := fmt.Sprintf("SELECT * FROM %s", from) if !create && opts.IncrementalRun { - err := olap.InsertTableAsSelect(ctx, outputTable, qry, false, true, outputProps.IncrementalStrategy, outputProps.UniqueKey) + insertOpts := &drivers.InsertTableOptions{ + ByName: false, + InPlace: true, + Strategy: outputProps.IncrementalStrategy, + UniqueKey: outputProps.UniqueKey, + } + err := olap.InsertTableAsSelect(ctx, outputTable, qry, insertOpts) if err != nil { return fmt.Errorf("failed to incrementally insert into table: %w", err) } @@ -140,14 +143,19 @@ func (e *warehouseToSelfExecutor) queryAndInsert(ctx context.Context, opts *driv } if !create { - err := olap.InsertTableAsSelect(ctx, outputTable, qry, false, true, drivers.IncrementalStrategyAppend, nil) + insertOpts := &drivers.InsertTableOptions{ + ByName: false, + InPlace: true, + Strategy: drivers.IncrementalStrategyAppend, + } + err := olap.InsertTableAsSelect(ctx, outputTable, qry, insertOpts) if err != nil { return fmt.Errorf("failed to insert into table: %w", err) } continue } - err = olap.CreateTableAsSelect(ctx, outputTable, false, qry, nil) + err = olap.CreateTableAsSelect(ctx, outputTable, qry, &drivers.CreateTableOptions{}) if err != nil { return fmt.Errorf("failed to create table: %w", err) } diff --git a/runtime/drivers/duckdb/model_manager.go b/runtime/drivers/duckdb/model_manager.go index 5b9fd051633..408c9e22863 100644 --- a/runtime/drivers/duckdb/model_manager.go +++ b/runtime/drivers/duckdb/model_manager.go @@ -10,8 +10,10 @@ import ( ) type ModelInputProperties struct { - SQL string `mapstructure:"sql"` - Args []any `mapstructure:"args"` + SQL string `mapstructure:"sql"` + Args []any `mapstructure:"args"` + PreExec string `mapstructure:"pre_exec"` + PostExec string `mapstructure:"post_exec"` } func (p *ModelInputProperties) Validate() error { @@ -120,17 +122,8 @@ func (c *connection) Delete(ctx context.Context, res *drivers.ModelResult) error return fmt.Errorf("connector is not an OLAP") } - stagingTable, err := olap.InformationSchema().Lookup(ctx, "", "", stagingTableNameFor(res.Table)) - if err == nil { - _ = olap.DropTable(ctx, stagingTable.Name, stagingTable.View) - } - - table, err := olap.InformationSchema().Lookup(ctx, "", "", res.Table) - if err != nil { - return err - } - - return olap.DropTable(ctx, table.Name, table.View) + _ = olap.DropTable(ctx, stagingTableNameFor(res.Table)) + return olap.DropTable(ctx, res.Table) } func (c *connection) MergePartitionResults(a, b *drivers.ModelResult) (*drivers.ModelResult, error) { @@ -168,7 +161,7 @@ func olapForceRenameTable(ctx context.Context, olap drivers.OLAPStore, fromName // Renaming a table to the same name with different casing is not supported. Workaround by renaming to a temporary name first. if strings.EqualFold(fromName, toName) { tmpName := fmt.Sprintf("__rill_tmp_rename_%s_%s", typ, toName) - err := olap.RenameTable(ctx, fromName, tmpName, fromIsView) + err := olap.RenameTable(ctx, fromName, tmpName) if err != nil { return err } @@ -176,7 +169,7 @@ func olapForceRenameTable(ctx context.Context, olap drivers.OLAPStore, fromName } // Do the rename - return olap.RenameTable(ctx, fromName, toName, fromIsView) + return olap.RenameTable(ctx, fromName, toName) } func boolPtr(b bool) *bool { diff --git a/runtime/drivers/duckdb/olap.go b/runtime/drivers/duckdb/olap.go index 57f8973ea7b..b73f6e3ca29 100644 --- a/runtime/drivers/duckdb/olap.go +++ b/runtime/drivers/duckdb/olap.go @@ -2,14 +2,8 @@ package duckdb import ( "context" - dbsql "database/sql" "errors" "fmt" - "io" - "io/fs" - "os" - "path/filepath" - "strings" "time" "github.com/google/uuid" @@ -17,6 +11,7 @@ import ( runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" "github.com/rilldata/rill/runtime/drivers" "github.com/rilldata/rill/runtime/pkg/observability" + "github.com/rilldata/rill/runtime/pkg/rduckdb" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" @@ -37,14 +32,14 @@ func (c *connection) Dialect() drivers.Dialect { return drivers.DialectDuckDB } -func (c *connection) WithConnection(ctx context.Context, priority int, longRunning, tx bool, fn drivers.WithConnectionFunc) error { +func (c *connection) WithConnection(ctx context.Context, priority int, longRunning bool, fn drivers.WithConnectionFunc) error { // Check not nested if connFromContext(ctx) != nil { panic("nested WithConnection") } // Acquire connection - conn, release, err := c.acquireOLAPConn(ctx, priority, longRunning, tx) + conn, release, err := c.acquireOLAPConn(ctx, priority, longRunning) if err != nil { return err } @@ -103,7 +98,6 @@ func (c *connection) Execute(ctx context.Context, stmt *drivers.Statement) (res queueLatency := acquiredTime.Sub(start).Milliseconds() attrs := []attribute.KeyValue{ - attribute.String("db", c.config.DBFilePath), attribute.Bool("cancelled", errors.Is(outErr, context.Canceled)), attribute.Bool("failed", outErr != nil), attribute.String("instance_id", c.instanceID), @@ -129,7 +123,7 @@ func (c *connection) Execute(ctx context.Context, stmt *drivers.Statement) (res }() // Acquire connection - conn, release, err := c.acquireOLAPConn(ctx, stmt.Priority, stmt.LongRunning, false) + conn, release, err := c.acquireOLAPConn(ctx, stmt.Priority, stmt.LongRunning) acquiredTime = time.Now() if err != nil { return nil, err @@ -181,748 +175,183 @@ func (c *connection) Execute(ctx context.Context, stmt *drivers.Statement) (res return res, nil } -func (c *connection) estimateSize(includeTemp bool) int64 { - path := c.config.DBFilePath - if path == "" { +func (c *connection) estimateSize() int64 { + db, release, err := c.acquireDB() + if err != nil { return 0 } - - paths := []string{path} - if includeTemp { - paths = append(paths, fmt.Sprintf("%s.wal", path)) - } - if c.config.ExtTableStorage { - entries, err := os.ReadDir(c.config.DBStoragePath) - if err == nil { // ignore error - for _, entry := range entries { - if !entry.IsDir() { - continue - } - // this is to avoid counting temp tables during source ingestion - // in certain cases we only want to compute the size of the serving db files - if strings.HasPrefix(entry.Name(), "__rill_tmp_") && !includeTemp { - continue - } - path := filepath.Join(c.config.DBStoragePath, entry.Name()) - version, exist, err := c.tableVersion(entry.Name()) - if err != nil || !exist { - continue - } - paths = append(paths, filepath.Join(path, fmt.Sprintf("%s.db", version))) - if includeTemp { - paths = append(paths, filepath.Join(path, fmt.Sprintf("%s.db.wal", version))) - } - } - } - } - return fileSize(paths) + size := db.Size() + _ = release() + return size } // AddTableColumn implements drivers.OLAPStore. func (c *connection) AddTableColumn(ctx context.Context, tableName, columnName, typ string) error { - c.logger.Debug("add table column", zap.String("tableName", tableName), zap.String("columnName", columnName), zap.String("typ", typ)) - if !c.config.ExtTableStorage { - return c.Exec(ctx, &drivers.Statement{ - Query: fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", safeSQLName(tableName), safeSQLName(columnName), typ), - Priority: 1, - LongRunning: true, - }) - } - - version, exist, err := c.tableVersion(tableName) + db, release, err := c.acquireDB() if err != nil { return err } + defer func() { + _ = release() + }() - if !exist { - return fmt.Errorf("table %q does not exist", tableName) - } - dbName := dbName(tableName, version) - return c.WithConnection(ctx, 1, true, false, func(ctx, ensuredCtx context.Context, conn *dbsql.Conn) error { - err = c.Exec(ctx, &drivers.Statement{Query: fmt.Sprintf("ALTER TABLE %s.default ADD COLUMN %s %s", safeSQLName(dbName), safeSQLName(columnName), typ)}) - if err != nil { - return err - } - // recreate view to propagate schema changes - return c.Exec(ensuredCtx, &drivers.Statement{Query: fmt.Sprintf("CREATE OR REPLACE VIEW %s AS SELECT * FROM %s.default", safeSQLName(tableName), safeSQLName(dbName))}) + err = db.MutateTable(ctx, tableName, func(ctx context.Context, conn *sqlx.Conn) error { + _, err := conn.ExecContext(ctx, fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", safeSQLName(tableName), safeSQLName(columnName), typ)) + return err }) + return c.checkErr(err) } // AlterTableColumn implements drivers.OLAPStore. func (c *connection) AlterTableColumn(ctx context.Context, tableName, columnName, newType string) error { - c.logger.Debug("alter table column", zap.String("tableName", tableName), zap.String("columnName", columnName), zap.String("newType", newType)) - if !c.config.ExtTableStorage { - return c.Exec(ctx, &drivers.Statement{ - Query: fmt.Sprintf("ALTER TABLE %s ALTER %s TYPE %s", safeSQLName(tableName), safeSQLName(columnName), newType), - Priority: 1, - LongRunning: true, - }) - } - - version, exist, err := c.tableVersion(tableName) + db, release, err := c.acquireDB() if err != nil { return err } + defer func() { + _ = release() + }() - if !exist { - return fmt.Errorf("table %q does not exist", tableName) - } - dbName := dbName(tableName, version) - return c.WithConnection(ctx, 1, true, false, func(ctx, ensuredCtx context.Context, conn *dbsql.Conn) error { - err = c.Exec(ctx, &drivers.Statement{Query: fmt.Sprintf("ALTER TABLE %s.default ALTER %s TYPE %s", safeSQLName(dbName), safeSQLName(columnName), newType)}) - if err != nil { - return err - } - - // recreate view to propagate schema changes - return c.Exec(ensuredCtx, &drivers.Statement{Query: fmt.Sprintf("CREATE OR REPLACE VIEW %s AS SELECT * FROM %s.default", safeSQLName(tableName), safeSQLName(dbName))}) + err = db.MutateTable(ctx, tableName, func(ctx context.Context, conn *sqlx.Conn) error { + _, err := conn.ExecContext(ctx, fmt.Sprintf("ALTER TABLE %s ALTER %s TYPE %s", safeSQLName(tableName), safeSQLName(columnName), newType)) + return err }) + return c.checkErr(err) } // CreateTableAsSelect implements drivers.OLAPStore. // We add a \n at the end of the any user query to ensure any comment at the end of model doesn't make the query incomplete. -func (c *connection) CreateTableAsSelect(ctx context.Context, name string, view bool, sql string, tableOpts map[string]any) error { - c.logger.Debug("create table", zap.String("name", name), zap.Bool("view", view)) - if view { - return c.Exec(ctx, &drivers.Statement{ - Query: fmt.Sprintf("CREATE OR REPLACE VIEW %s AS (%s\n)", safeSQLName(name), sql), - Priority: 1, - LongRunning: true, - }) - } - if !c.config.ExtTableStorage { - return c.Exec(ctx, &drivers.Statement{ - Query: fmt.Sprintf("CREATE OR REPLACE TABLE %s AS (%s\n)", safeSQLName(name), sql), - Priority: 1, - LongRunning: true, - }) - } - - var cleanupFunc func() - err := c.WithConnection(ctx, 1, true, false, func(ctx, ensuredCtx context.Context, _ *dbsql.Conn) error { - // NOTE: Running mkdir while holding the connection to avoid directory getting cleaned up when concurrent calls to RenameTable cause reopenDB to be called. - - // create a new db file in // directory - sourceDir := filepath.Join(c.config.DBStoragePath, name) - if err := os.Mkdir(sourceDir, fs.ModePerm); err != nil && !errors.Is(err, fs.ErrExist) { - return fmt.Errorf("create: unable to create dir %q: %w", sourceDir, err) - } - - // check if some older version existed previously to detach it later - oldVersion, oldVersionExists, _ := c.tableVersion(name) - - newVersion := fmt.Sprint(time.Now().UnixMilli()) - dbFile := filepath.Join(sourceDir, fmt.Sprintf("%s.db", newVersion)) - db := dbName(name, newVersion) - - // attach new db - err := c.Exec(ctx, &drivers.Statement{Query: fmt.Sprintf("ATTACH %s AS %s", safeSQLString(dbFile), safeSQLName(db))}) - if err != nil { - removeDBFile(dbFile) - return fmt.Errorf("create: attach %q db failed: %w", dbFile, err) - } - - // Enforce storage limits - if err := c.Exec(ctx, &drivers.Statement{Query: fmt.Sprintf("CREATE OR REPLACE TABLE %s.default AS (%s\n)", safeSQLName(db), sql)}); err != nil { - cleanupFunc = func() { c.detachAndRemoveFile(db, dbFile) } - return fmt.Errorf("create: create %q.default table failed: %w", db, err) - } - - // success update version - err = c.updateVersion(name, newVersion) - if err != nil { - // extreme bad luck - cleanupFunc = func() { c.detachAndRemoveFile(db, dbFile) } - return fmt.Errorf("create: update version %q failed: %w", newVersion, err) - } - - qry, err := c.generateSelectQuery(ctx, db) - if err != nil { - return err - } - - // create view query - err = c.Exec(ctx, &drivers.Statement{ - Query: fmt.Sprintf("CREATE OR REPLACE VIEW %s AS %s", safeSQLName(name), qry), - }) - if err != nil { - cleanupFunc = func() { c.detachAndRemoveFile(db, dbFile) } - return fmt.Errorf("create: create view %q failed: %w", name, err) - } - - if oldVersionExists { - oldDB := dbName(name, oldVersion) - // ignore these errors since source has been correctly ingested and attached - cleanupFunc = func() { c.detachAndRemoveFile(oldDB, filepath.Join(sourceDir, fmt.Sprintf("%s.db", oldVersion))) } - } - return nil - }) - if cleanupFunc != nil { - cleanupFunc() - } - return err -} - -// InsertTableAsSelect implements drivers.OLAPStore. -func (c *connection) InsertTableAsSelect(ctx context.Context, name, sql string, byName, inPlace bool, strategy drivers.IncrementalStrategy, uniqueKey []string) error { - c.logger.Debug("insert table", zap.String("name", name), zap.Bool("byName", byName), zap.String("strategy", string(strategy)), zap.Strings("uniqueKey", uniqueKey)) - - if !c.config.ExtTableStorage { - return c.WithConnection(ctx, 1, true, false, func(ctx, ensuredCtx context.Context, _ *dbsql.Conn) error { - return c.execIncrementalInsert(ctx, safeSQLName(name), sql, byName, strategy, uniqueKey) - }) - } - - if inPlace { - version, exist, err := c.tableVersion(name) - if err != nil { - return err - } - if !exist { - return fmt.Errorf("insert: table %q does not exist", name) - } - - db := dbName(name, version) - safeName := fmt.Sprintf("%s.default", safeSQLName(db)) - - return c.WithConnection(ctx, 1, true, false, func(ctx, ensuredCtx context.Context, _ *dbsql.Conn) error { - return c.execIncrementalInsert(ctx, safeName, sql, byName, strategy, uniqueKey) - }) - } - - var cleanupFunc func() - err := c.WithConnection(ctx, 1, true, false, func(ctx, ensuredCtx context.Context, _ *dbsql.Conn) error { - // Get current table version - oldVersion, oldVersionExists, _ := c.tableVersion(name) - if !oldVersionExists { - return fmt.Errorf("table %q does not exist", name) - } - - // Prepare a new version - newVersion := fmt.Sprint(time.Now().UnixMilli()) - - // Prepare paths - sourceDir := filepath.Join(c.config.DBStoragePath, name) - oldDBFile := filepath.Join(sourceDir, fmt.Sprintf("%s.db", oldVersion)) - newDBFile := filepath.Join(sourceDir, fmt.Sprintf("%s.db", newVersion)) - oldDB := dbName(name, oldVersion) - newDB := dbName(name, newVersion) - - // Copy the old version to the new version - if err := copyFile(oldDBFile, newDBFile); err != nil { - return fmt.Errorf("insert: copy file failed: %w", err) - } - - // Attach the new db - err := c.Exec(ctx, &drivers.Statement{Query: fmt.Sprintf("ATTACH %s AS %s", safeSQLString(newDBFile), safeSQLName(newDB))}) - if err != nil { - removeDBFile(newDBFile) - return fmt.Errorf("insert: attach %q db failed: %w", newDBFile, err) - } - - // Execute the insert - safeName := fmt.Sprintf("%s.default", safeSQLName(newDB)) - err = c.execIncrementalInsert(ctx, safeName, sql, byName, strategy, uniqueKey) - if err != nil { - cleanupFunc = func() { c.detachAndRemoveFile(newDB, newDBFile) } - return fmt.Errorf("insert: create %q.default table failed: %w", newDB, err) - } - - // Success: update version - err = c.updateVersion(name, newVersion) - if err != nil { - // extreme bad luck - cleanupFunc = func() { c.detachAndRemoveFile(newDB, newDBFile) } - return fmt.Errorf("insert: update version %q failed: %w", newVersion, err) - } - - // Update the view to the external table in the main DB handle - qry, err := c.generateSelectQuery(ctx, newDB) - if err != nil { - return err - } - err = c.Exec(ctx, &drivers.Statement{ - Query: fmt.Sprintf("CREATE OR REPLACE VIEW %s AS %s", safeSQLName(name), qry), - }) - if err != nil { - cleanupFunc = func() { c.detachAndRemoveFile(newDB, newDBFile) } - return fmt.Errorf("insert: create view %q failed: %w", name, err) - } - - // Delete the old version (ignoring errors since source the new data has already been correctly inserted and attached) - cleanupFunc = func() { c.detachAndRemoveFile(oldDB, oldDBFile) } - return nil - }) - if cleanupFunc != nil { - cleanupFunc() - } - return err -} - -// DropTable implements drivers.OLAPStore. -func (c *connection) DropTable(ctx context.Context, name string, view bool) error { - c.logger.Debug("drop table", zap.String("name", name), zap.Bool("view", view)) - if !c.config.ExtTableStorage { - var typ string - if view { - typ = "VIEW" - } else { - typ = "TABLE" - } - return c.Exec(ctx, &drivers.Statement{ - Query: fmt.Sprintf("DROP %s IF EXISTS %s", typ, safeSQLName(name)), - Priority: 100, - LongRunning: true, - }) - } - // determine if it is a true view or view on externally stored table - version, exist, err := c.tableVersion(name) +func (c *connection) CreateTableAsSelect(ctx context.Context, name, sql string, opts *drivers.CreateTableOptions) error { + db, release, err := c.acquireDB() if err != nil { return err } - - if !exist { - if !view { - return nil - } - return c.Exec(ctx, &drivers.Statement{ - Query: fmt.Sprintf("DROP VIEW IF EXISTS %s", safeSQLName(name)), - Priority: 100, - LongRunning: true, - }) - } - - err = c.WithConnection(ctx, 100, true, true, func(ctx, ensuredCtx context.Context, _ *dbsql.Conn) error { - // drop view - err = c.Exec(ctx, &drivers.Statement{Query: fmt.Sprintf("DROP VIEW IF EXISTS %s", safeSQLName(name))}) - if err != nil { + defer func() { + _ = release() + }() + var beforeCreateFn, afterCreateFn func(ctx context.Context, conn *sqlx.Conn) error + if opts.BeforeCreate != "" { + beforeCreateFn = func(ctx context.Context, conn *sqlx.Conn) error { + _, err := conn.ExecContext(ctx, opts.BeforeCreate) return err } - - oldDB := dbName(name, version) - err = c.Exec(ensuredCtx, &drivers.Statement{Query: fmt.Sprintf("DETACH %s", safeSQLName(oldDB))}) - if err != nil && !strings.Contains(err.Error(), "database not found") { // ignore database not found errors for idempotency + } + if opts.AfterCreate != "" { + afterCreateFn = func(ctx context.Context, conn *sqlx.Conn) error { + _, err := conn.ExecContext(ctx, opts.AfterCreate) return err } - // delete source directory - return os.RemoveAll(filepath.Join(c.config.DBStoragePath, name)) - }) - return err -} - -// RenameTable implements drivers.OLAPStore. -// For drop and replace (when running `RenameTable("__tmp_foo", "foo")`): -// `DROP VIEW __tmp_foo` -// `DETACH __tmp_foo__1` -// `mv __tmp_foo/1.db foo/2.db` -// `echo 2 > version.txt` -// `rm __tmp_foo` -// `ATTACH 'foo/2.db' AS foo__2` -// `CREATE OR REPLACE VIEW foo AS SELECT * FROM foo_2` -// `DETACH foo__1` -// `rm foo/1.db` -func (c *connection) RenameTable(ctx context.Context, oldName, newName string, view bool) error { - c.logger.Debug("rename table", zap.String("from", oldName), zap.String("to", newName), zap.Bool("view", view), zap.Bool("ext", c.config.ExtTableStorage)) - if strings.EqualFold(oldName, newName) { - return fmt.Errorf("rename: old and new name are same case insensitive strings") - } - if !c.config.ExtTableStorage { - return c.dropAndReplace(ctx, oldName, newName, view) - } - // determine if it is a true view or a view on externally stored table - oldVersion, exist, err := c.tableVersion(oldName) - if err != nil { - return err - } - if !exist { - return c.dropAndReplace(ctx, oldName, newName, view) } + err = db.CreateTableAsSelect(ctx, name, sql, &rduckdb.CreateTableOptions{View: opts.View, BeforeCreateFn: beforeCreateFn, AfterCreateFn: afterCreateFn}) + return c.checkErr(err) +} - oldVersionInNewDir, replaceInNewTable, err := c.tableVersion(newName) +// InsertTableAsSelect implements drivers.OLAPStore. +func (c *connection) InsertTableAsSelect(ctx context.Context, name, sql string, opts *drivers.InsertTableOptions) error { + db, release, err := c.acquireDB() if err != nil { return err } - - newSrcDir := filepath.Join(c.config.DBStoragePath, newName) - oldSrcDir := filepath.Join(c.config.DBStoragePath, oldName) - - // reopen duckdb connections which should delete any temporary files built up during ingestion - // need to do detach using tx=true to isolate it from other queries - err = c.WithConnection(ctx, 100, true, true, func(currentCtx, ctx context.Context, conn *dbsql.Conn) error { - err = os.Mkdir(newSrcDir, fs.ModePerm) - if err != nil && !errors.Is(err, fs.ErrExist) { - return err - } - - // drop old view - err = c.Exec(currentCtx, &drivers.Statement{Query: fmt.Sprintf("DROP VIEW IF EXISTS %s", safeSQLName(oldName))}) - if err != nil { - return fmt.Errorf("rename: drop %q view failed: %w", oldName, err) - } - - // detach old db - err = c.Exec(ctx, &drivers.Statement{Query: fmt.Sprintf("DETACH %s", safeSQLName(dbName(oldName, oldVersion)))}) - if err != nil { - return fmt.Errorf("rename: detach %q db failed: %w", dbName(oldName, oldVersion), err) - } - - // move old file as a new file in source directory - newVersion := fmt.Sprint(time.Now().UnixMilli()) - newFile := filepath.Join(newSrcDir, fmt.Sprintf("%s.db", newVersion)) - err = os.Rename(filepath.Join(oldSrcDir, fmt.Sprintf("%s.db", oldVersion)), newFile) - if err != nil { - return fmt.Errorf("rename: rename file failed: %w", err) - } - // also move .db.wal file in case checkpointing was not completed - _ = os.Rename(filepath.Join(oldSrcDir, fmt.Sprintf("%s.db.wal", oldVersion)), - filepath.Join(newSrcDir, fmt.Sprintf("%s.db.wal", newVersion))) - - err = c.updateVersion(newName, newVersion) - if err != nil { - return fmt.Errorf("rename: update version failed: %w", err) - } - err = os.RemoveAll(filepath.Join(c.config.DBStoragePath, oldName)) - if err != nil { - c.logger.Error("rename: unable to delete old path", zap.Error(err)) - } - - newDB := dbName(newName, newVersion) - // attach new db - err = c.Exec(ctx, &drivers.Statement{Query: fmt.Sprintf("ATTACH %s AS %s", safeSQLString(newFile), safeSQLName(newDB))}) - if err != nil { - return fmt.Errorf("rename: attach %q db failed: %w", newDB, err) - } - - qry, err := c.generateSelectQuery(ctx, newDB) - if err != nil { - return err - } - - // change view query - err = c.Exec(ctx, &drivers.Statement{Query: fmt.Sprintf("CREATE OR REPLACE VIEW %s AS %s", safeSQLName(newName), qry)}) - if err != nil { - return fmt.Errorf("rename: create %q view failed: %w", newName, err) - } - - if !replaceInNewTable { - return nil - } - // new table had some other file previously - if err := c.Exec(ctx, &drivers.Statement{Query: fmt.Sprintf("DETACH %s", safeSQLName(dbName(newName, oldVersionInNewDir)))}); err != nil { - return err - } - removeDBFile(filepath.Join(newSrcDir, fmt.Sprintf("%s.db", oldVersionInNewDir))) - return nil - }) - return err -} - -func (c *connection) MayBeScaledToZero(ctx context.Context) bool { - return false -} - -func (c *connection) execIncrementalInsert(ctx context.Context, safeName, sql string, byName bool, strategy drivers.IncrementalStrategy, uniqueKey []string) error { + defer func() { + _ = release() + }() var byNameClause string - if byName { + if opts.ByName { byNameClause = "BY NAME" } - if strategy == drivers.IncrementalStrategyAppend { - return c.Exec(ctx, &drivers.Statement{ - Query: fmt.Sprintf("INSERT INTO %s %s (%s\n)", safeName, byNameClause, sql), - Priority: 1, - LongRunning: true, + if opts.Strategy == drivers.IncrementalStrategyAppend { + err = db.MutateTable(ctx, name, func(ctx context.Context, conn *sqlx.Conn) error { + _, err := conn.ExecContext(ctx, fmt.Sprintf("INSERT INTO %s %s (%s\n)", safeSQLName(name), byNameClause, sql)) + return err }) + return c.checkErr(err) } - if strategy == drivers.IncrementalStrategyMerge { - // Create a temporary table with the new data - tmp := uuid.New().String() - err := c.Exec(ctx, &drivers.Statement{ - Query: fmt.Sprintf("CREATE TEMPORARY TABLE %s AS (%s\n)", safeSQLName(tmp), sql), - Priority: 1, - LongRunning: true, - }) - if err != nil { - return err - } - - // check the count of the new data - // skip if the count is 0 - // if there was no data in the empty file then the detected schema can be different from the current schema which leads to errors or performance issues - res, err := c.Execute(ctx, &drivers.Statement{ - Query: fmt.Sprintf("SELECT COUNT(*) == 0 FROM %s", safeSQLName(tmp)), - Priority: 1, - }) - if err != nil { - return err - } - var empty bool - for res.Next() { - if err := res.Scan(&empty); err != nil { - _ = res.Close() + if opts.Strategy == drivers.IncrementalStrategyMerge { + err = db.MutateTable(ctx, name, func(ctx context.Context, conn *sqlx.Conn) (mutate error) { + // Execute the pre-init SQL first + if opts.BeforeInsert != "" { + _, err := conn.ExecContext(ctx, opts.BeforeInsert) return err } - } - _ = res.Close() - if empty { - return nil - } - - // Drop the rows from the target table where the unique key is present in the temporary table - where := "" - for i, key := range uniqueKey { - key = safeSQLName(key) - if i != 0 { - where += " AND " + defer func() { + if opts.AfterInsert != "" { + _, err := conn.ExecContext(ctx, opts.AfterInsert) + mutate = errors.Join(mutate, err) + } + }() + // Create a temporary table with the new data + tmp := uuid.New().String() + _, err := conn.ExecContext(ctx, fmt.Sprintf("CREATE TEMPORARY TABLE %s AS (%s\n)", safeSQLName(tmp), sql)) + if err != nil { + return err } - where += fmt.Sprintf("base.%s IS NOT DISTINCT FROM tmp.%s", key, key) - } - err = c.Exec(ctx, &drivers.Statement{ - Query: fmt.Sprintf("DELETE FROM %s base WHERE EXISTS (SELECT 1 FROM %s tmp WHERE %s)", safeName, safeSQLName(tmp), where), - Priority: 1, - LongRunning: true, - }) - if err != nil { - return err - } - // Insert the new data into the target table - return c.Exec(ctx, &drivers.Statement{ - Query: fmt.Sprintf("INSERT INTO %s %s SELECT * FROM %s", safeName, byNameClause, safeSQLName(tmp)), - Priority: 1, - LongRunning: true, - }) - } - - return fmt.Errorf("incremental insert strategy %q not supported", strategy) -} + // check the count of the new data + // skip if the count is 0 + // if there was no data in the empty file then the detected schema can be different from the current schema which leads to errors or performance issues + var empty bool + err = conn.QueryRowxContext(ctx, fmt.Sprintf("SELECT COUNT(*) == 0 FROM %s", safeSQLName(tmp))).Scan(&empty) + if err != nil { + return err + } + if empty { + return nil + } -func (c *connection) dropAndReplace(ctx context.Context, oldName, newName string, view bool) error { - var typ string - if view { - typ = "VIEW" - } else { - typ = "TABLE" - } + // Drop the rows from the target table where the unique key is present in the temporary table + where := "" + for i, key := range opts.UniqueKey { + key = safeSQLName(key) + if i != 0 { + where += " AND " + } + where += fmt.Sprintf("base.%s IS NOT DISTINCT FROM tmp.%s", key, key) + } + _, err = conn.ExecContext(ctx, fmt.Sprintf("DELETE FROM %s base WHERE EXISTS (SELECT 1 FROM %s tmp WHERE %s)", safeSQLName(name), safeSQLName(tmp), where)) + if err != nil { + return err + } - existing, err := c.InformationSchema().Lookup(ctx, "", "", newName) - if err != nil { - if !errors.Is(err, drivers.ErrNotFound) { + // Insert the new data into the target table + _, err = conn.ExecContext(ctx, fmt.Sprintf("INSERT INTO %s %s SELECT * FROM %s", safeSQLName(name), byNameClause, safeSQLName(tmp))) return err - } - return c.Exec(ctx, &drivers.Statement{ - Query: fmt.Sprintf("ALTER %s %s RENAME TO %s", typ, safeSQLName(oldName), safeSQLName(newName)), - Priority: 100, - LongRunning: true, }) + return c.checkErr(err) } - return c.WithConnection(ctx, 100, true, true, func(ctx, ensuredCtx context.Context, conn *dbsql.Conn) error { - // The newName may currently be occupied by a name of another type than oldName. - var existingTyp string - if existing.View { - existingTyp = "VIEW" - } else { - existingTyp = "TABLE" - } - - err := c.Exec(ctx, &drivers.Statement{Query: fmt.Sprintf("DROP %s IF EXISTS %s", existingTyp, safeSQLName(newName))}) - if err != nil { - return err - } - - return c.Exec(ctx, &drivers.Statement{Query: fmt.Sprintf("ALTER %s %s RENAME TO %s", typ, safeSQLName(oldName), safeSQLName(newName))}) - }) -} - -func (c *connection) detachAndRemoveFile(db, dbFile string) { - err := c.WithConnection(context.Background(), 100, false, true, func(ctx, ensuredCtx context.Context, conn *dbsql.Conn) error { - err := c.Exec(ctx, &drivers.Statement{Query: fmt.Sprintf("DETACH %s", safeSQLName(db)), Priority: 100}) - removeDBFile(dbFile) - return err - }) - if err != nil { - c.logger.Debug("detach failed", zap.String("db", db), zap.Error(err)) - } -} - -func (c *connection) tableVersion(name string) (string, bool, error) { - pathToFile := filepath.Join(c.config.DBStoragePath, name, "version.txt") - contents, err := os.ReadFile(pathToFile) - if err != nil { - if errors.Is(err, fs.ErrNotExist) { - return "", false, nil - } - return "", false, err - } - return strings.TrimSpace(string(contents)), true, nil + return fmt.Errorf("incremental insert strategy %q not supported", opts.Strategy) } -func (c *connection) updateVersion(name, version string) error { - pathToFile := filepath.Join(c.config.DBStoragePath, name, "version.txt") - file, err := os.Create(pathToFile) +// DropTable implements drivers.OLAPStore. +func (c *connection) DropTable(ctx context.Context, name string) error { + db, release, err := c.acquireDB() if err != nil { return err } - defer file.Close() - - _, err = file.WriteString(version) - return err + defer func() { + _ = release() + }() + err = db.DropTable(ctx, name) + return c.checkErr(err) } -// convertToEnum converts a varchar col in table to an enum type. -// Generally to be used for low cardinality varchar columns although not enforced here. -func (c *connection) convertToEnum(ctx context.Context, table string, cols []string) error { - if len(cols) == 0 { - return fmt.Errorf("empty list") - } - if !c.config.ExtTableStorage { - return fmt.Errorf("`cast_to_enum` is only supported when `external_table_storage` is enabled") - } - c.logger.Debug("convert column to enum", zap.String("table", table), zap.Strings("col", cols)) - - oldVersion, exist, err := c.tableVersion(table) - if err != nil { - return err - } - - if !exist { - return fmt.Errorf("table %q does not exist", table) - } - - // scan main db and main schema - res, err := c.Execute(ctx, &drivers.Statement{ - Query: "SELECT current_database(), current_schema()", - Priority: 100, - }) +// RenameTable implements drivers.OLAPStore. +func (c *connection) RenameTable(ctx context.Context, oldName, newName string) error { + db, release, err := c.acquireDB() if err != nil { return err } - - var mainDB, mainSchema string - if res.Next() { - if err := res.Scan(&mainDB, &mainSchema); err != nil { - _ = res.Close() - return err - } - } - _ = res.Close() - - sourceDir := filepath.Join(c.config.DBStoragePath, table) - newVersion := fmt.Sprint(time.Now().UnixMilli()) - newDBFile := filepath.Join(sourceDir, fmt.Sprintf("%s.db", newVersion)) - newDB := dbName(table, newVersion) - var cleanupFunc func() - err = c.WithConnection(ctx, 100, true, false, func(ctx, ensuredCtx context.Context, _ *dbsql.Conn) error { - // attach new db - err = c.Exec(ctx, &drivers.Statement{Query: fmt.Sprintf("ATTACH %s AS %s", safeSQLString(newDBFile), safeSQLName(newDB))}) - if err != nil { - removeDBFile(newDBFile) - return fmt.Errorf("create: attach %q db failed: %w", newDBFile, err) - } - - // switch to new db - // this is only required since duckdb has bugs around db scoped custom types - // TODO: remove this when https://github.com/duckdb/duckdb/pull/9622 is released - err = c.Exec(ctx, &drivers.Statement{Query: fmt.Sprintf("USE %s", safeSQLName(newDB))}) - if err != nil { - cleanupFunc = func() { c.detachAndRemoveFile(newDB, newDBFile) } - return fmt.Errorf("failed switch db %q: %w", newDB, err) - } - defer func() { - // switch to original db, notice `db.schema` just doing USE db switches context to `main` schema in the current db if doing `USE main` - // we want to switch to original db and schema - err = c.Exec(ensuredCtx, &drivers.Statement{Query: fmt.Sprintf("USE %s.%s", safeSQLName(mainDB), safeSQLName(mainSchema))}) - if err != nil { - cleanupFunc = func() { c.detachAndRemoveFile(newDB, newDBFile) } - // This should NEVER happen - c.fatalInternalError(fmt.Errorf("failed to switch back from db %q: %w", mainDB, err)) - } - }() - - oldDB := dbName(table, oldVersion) - for _, col := range cols { - enum := fmt.Sprintf("%s_enum", col) - if err = c.Exec(ensuredCtx, &drivers.Statement{Query: fmt.Sprintf("CREATE TYPE %s AS ENUM (SELECT DISTINCT %s FROM %s.default WHERE %s IS NOT NULL)", safeSQLName(enum), safeSQLName(col), safeSQLName(oldDB), safeSQLName(col))}); err != nil { - cleanupFunc = func() { c.detachAndRemoveFile(newDB, newDBFile) } - return fmt.Errorf("failed to create enum %q: %w", enum, err) - } - } - - var selectQry string - for _, col := range cols { - enum := fmt.Sprintf("%s_enum", col) - selectQry += fmt.Sprintf("CAST(%s AS %s) AS %s,", safeSQLName(col), safeSQLName(enum), safeSQLName(col)) - } - selectQry += fmt.Sprintf("* EXCLUDE(%s)", strings.Join(cols, ",")) - - if err := c.Exec(ensuredCtx, &drivers.Statement{Query: fmt.Sprintf("CREATE OR REPLACE TABLE \"default\" AS SELECT %s FROM %s.default", selectQry, safeSQLName(oldDB))}); err != nil { - cleanupFunc = func() { c.detachAndRemoveFile(newDB, newDBFile) } - return fmt.Errorf("failed to create table with enum values: %w", err) - } - - // recreate view to propagate schema changes - selectQry, err := c.generateSelectQuery(ctx, newDB) - if err != nil { - return err - } - - // NOTE :: db name need to be appened in the view query else query fails when switching to main db - if err := c.Exec(ensuredCtx, &drivers.Statement{Query: fmt.Sprintf("CREATE OR REPLACE VIEW %s.%s.%s AS %s", safeSQLName(mainDB), safeSQLName(mainSchema), safeSQLName(table), selectQry)}); err != nil { - cleanupFunc = func() { c.detachAndRemoveFile(newDB, newDBFile) } - return fmt.Errorf("failed to create view %q: %w", table, err) - } - - // update version and detach old db - if err := c.updateVersion(table, newVersion); err != nil { - cleanupFunc = func() { c.detachAndRemoveFile(newDB, newDBFile) } - return fmt.Errorf("failed to update version: %w", err) - } - - cleanupFunc = func() { - c.detachAndRemoveFile(oldDB, filepath.Join(sourceDir, fmt.Sprintf("%s.db", oldVersion))) - } - return nil - }) - if cleanupFunc != nil { - cleanupFunc() - } - return err + defer func() { + _ = release() + }() + err = db.RenameTable(ctx, oldName, newName) + return c.checkErr(err) } -// duckDB raises Contents of view were altered: types don't match! error even when number of columns are same but sequence of column changes in underlying table. -// This causes temporary query failures till the model view is not updated to reflect the new column sequence. -// We ensure that view for external table storage is always generated using a stable order of columns of underlying table. -// Additionally we want to keep the same order as the underlying table locally so that we can show columns in the same order as they appear in source data. -// Using `AllowHostAccess` as proxy to check if we are running in local/cloud mode. -func (c *connection) generateSelectQuery(ctx context.Context, db string) (string, error) { - if c.config.AllowHostAccess { - return fmt.Sprintf("SELECT * FROM %s.default", safeSQLName(db)), nil - } - - rows, err := c.Execute(ctx, &drivers.Statement{ - Query: fmt.Sprintf(` - SELECT column_name AS name - FROM information_schema.columns - WHERE table_catalog = %s AND table_name = 'default' - ORDER BY name ASC`, safeSQLString(db)), - }) - if err != nil { - return "", err - } - defer rows.Close() - - cols := make([]string, 0) - var col string - for rows.Next() { - if err := rows.Scan(&col); err != nil { - return "", err - } - cols = append(cols, safeName(col)) - } - - return fmt.Sprintf("SELECT %s FROM %s.default", strings.Join(cols, ", "), safeSQLName(db)), nil +func (c *connection) MayBeScaledToZero(ctx context.Context) bool { + return false } func RowsToSchema(r *sqlx.Rows) (*runtimev1.StructType, error) { @@ -956,17 +385,6 @@ func RowsToSchema(r *sqlx.Rows) (*runtimev1.StructType, error) { return &runtimev1.StructType{Fields: fields}, nil } -func dbName(name, version string) string { - return fmt.Sprintf("%s_%s", name, version) -} - -func removeDBFile(dbFile string) { - _ = os.Remove(dbFile) - // Hacky approach to remove the wal and tmp file - _ = os.Remove(dbFile + ".wal") - _ = os.RemoveAll(dbFile + ".tmp") -} - // safeSQLName returns a quoted SQL identifier. func safeSQLName(name string) string { return safeName(name) @@ -975,20 +393,3 @@ func safeSQLName(name string) string { func safeSQLString(name string) string { return drivers.DialectDuckDB.EscapeStringValue(name) } - -func copyFile(src, dst string) error { - srcFile, err := os.Open(src) - if err != nil { - return err - } - defer srcFile.Close() - - dstFile, err := os.Create(dst) - if err != nil { - return err - } - defer dstFile.Close() - - _, err = io.Copy(dstFile, srcFile) - return err -} diff --git a/runtime/drivers/duckdb/olap_crud_test.go b/runtime/drivers/duckdb/olap_crud_test.go index 7ebf25827f0..64b6288e154 100644 --- a/runtime/drivers/duckdb/olap_crud_test.go +++ b/runtime/drivers/duckdb/olap_crud_test.go @@ -6,7 +6,6 @@ import ( "io/fs" "os" "path/filepath" - "strconv" "testing" "time" @@ -19,16 +18,8 @@ import ( func Test_connection_CreateTableAsSelect(t *testing.T) { temp := t.TempDir() - require.NoError(t, os.Mkdir(filepath.Join(temp, "default"), fs.ModePerm)) - dbPath := filepath.Join(temp, "default", "normal.db") - handle, err := Driver{}.Open("default", map[string]any{"path": dbPath, "external_table_storage": false}, storage.MustNew(temp, nil), activity.NewNoopClient(), zap.NewNop()) - require.NoError(t, err) - normalConn := handle.(*connection) - normalConn.AsOLAP("default") - require.NoError(t, normalConn.Migrate(context.Background())) - dbPath = filepath.Join(temp, "default", "view.db") - handle, err = Driver{}.Open("default", map[string]any{"path": dbPath, "external_table_storage": true}, storage.MustNew(temp, nil), activity.NewNoopClient(), zap.NewNop()) + handle, err := Driver{}.Open("default", map[string]any{}, storage.MustNew(temp, nil), activity.NewNoopClient(), zap.NewNop()) require.NoError(t, err) viewConnection := handle.(*connection) require.NoError(t, viewConnection.Migrate(context.Background())) @@ -41,17 +32,6 @@ func Test_connection_CreateTableAsSelect(t *testing.T) { tableAsView bool c *connection }{ - { - testName: "select from view", - name: "my-view", - view: true, - c: normalConn, - }, - { - testName: "select from table", - name: "my-table", - c: normalConn, - }, { testName: "select from view with external_table_storage", name: "my-view", @@ -69,7 +49,7 @@ func Test_connection_CreateTableAsSelect(t *testing.T) { sql := "SELECT 1" for _, tt := range tests { t.Run(tt.testName, func(t *testing.T) { - err := tt.c.CreateTableAsSelect(ctx, tt.name, tt.view, sql, nil) + err := tt.c.CreateTableAsSelect(ctx, tt.name, sql, &drivers.CreateTableOptions{View: tt.view}) require.NoError(t, err) res, err := tt.c.Execute(ctx, &drivers.Statement{Query: fmt.Sprintf("SELECT count(*) FROM %q", tt.name)}) require.NoError(t, err) @@ -87,11 +67,6 @@ func Test_connection_CreateTableAsSelect(t *testing.T) { require.NoError(t, res.Scan(&count)) require.Equal(t, 1, count) require.NoError(t, res.Close()) - contents, err := os.ReadFile(filepath.Join(temp, "default", tt.name, "version.txt")) - require.NoError(t, err) - version, err := strconv.ParseInt(string(contents), 10, 64) - require.NoError(t, err) - require.True(t, time.Since(time.UnixMilli(version)).Seconds() < 1) } }) } @@ -100,38 +75,21 @@ func Test_connection_CreateTableAsSelect(t *testing.T) { func Test_connection_CreateTableAsSelectMultipleTimes(t *testing.T) { temp := t.TempDir() - dbPath := filepath.Join(temp, "view.db") - handle, err := Driver{}.Open("default", map[string]any{"path": dbPath, "external_table_storage": true}, storage.MustNew(temp, nil), activity.NewNoopClient(), zap.NewNop()) + handle, err := Driver{}.Open("default", map[string]any{}, storage.MustNew(temp, nil), activity.NewNoopClient(), zap.NewNop()) require.NoError(t, err) c := handle.(*connection) require.NoError(t, c.Migrate(context.Background())) c.AsOLAP("default") - err = c.CreateTableAsSelect(context.Background(), "test-select-multiple", false, "select 1", nil) + err = c.CreateTableAsSelect(context.Background(), "test-select-multiple", "select 1", &drivers.CreateTableOptions{}) require.NoError(t, err) time.Sleep(2 * time.Millisecond) - err = c.CreateTableAsSelect(context.Background(), "test-select-multiple", false, "select 'hello'", nil) + err = c.CreateTableAsSelect(context.Background(), "test-select-multiple", "select 'hello'", &drivers.CreateTableOptions{}) require.NoError(t, err) - dirs, err := os.ReadDir(filepath.Join(temp, "test-select-multiple")) - require.NoError(t, err) - names := make([]string, 0) - for _, dir := range dirs { - names = append(names, dir.Name()) - } - - err = c.CreateTableAsSelect(context.Background(), "test-select-multiple", false, "select fail query", nil) + err = c.CreateTableAsSelect(context.Background(), "test-select-multiple", "select fail query", &drivers.CreateTableOptions{}) require.Error(t, err) - dirs, err = os.ReadDir(filepath.Join(temp, "test-select-multiple")) - require.NoError(t, err) - newNames := make([]string, 0) - for _, dir := range dirs { - newNames = append(newNames, dir.Name()) - } - - require.Equal(t, names, newNames) - res, err := c.Execute(context.Background(), &drivers.Statement{Query: fmt.Sprintf("SELECT * FROM %q", "test-select-multiple")}) require.NoError(t, err) require.True(t, res.Next()) @@ -145,23 +103,18 @@ func Test_connection_CreateTableAsSelectMultipleTimes(t *testing.T) { func Test_connection_DropTable(t *testing.T) { temp := t.TempDir() - dbPath := filepath.Join(temp, "view.db") - handle, err := Driver{}.Open("default", map[string]any{"path": dbPath, "external_table_storage": true}, storage.MustNew(temp, nil), activity.NewNoopClient(), zap.NewNop()) + handle, err := Driver{}.Open("default", map[string]any{}, storage.MustNew(temp, nil), activity.NewNoopClient(), zap.NewNop()) require.NoError(t, err) c := handle.(*connection) require.NoError(t, c.Migrate(context.Background())) c.AsOLAP("default") - err = c.CreateTableAsSelect(context.Background(), "test-drop", false, "select 1", nil) + err = c.CreateTableAsSelect(context.Background(), "test-drop", "select 1", &drivers.CreateTableOptions{}) require.NoError(t, err) - // Note: true since at lot of places we look at information_schema lookup on main db to determine whether tbl is a view or table - err = c.DropTable(context.Background(), "test-drop", true) + err = c.DropTable(context.Background(), "test-drop") require.NoError(t, err) - _, err = os.ReadDir(filepath.Join(temp, "test-drop")) - require.True(t, os.IsNotExist(err)) - res, err := c.Execute(context.Background(), &drivers.Statement{Query: "SELECT count(*) FROM information_schema.tables WHERE table_name='test-drop' AND table_type='VIEW'"}) require.NoError(t, err) require.True(t, res.Next()) @@ -171,23 +124,32 @@ func Test_connection_DropTable(t *testing.T) { require.NoError(t, res.Close()) } -func Test_connection_InsertTableAsSelect(t *testing.T) { +func Test_connection_InsertTableAsSelect_WithAppendStrategy(t *testing.T) { temp := t.TempDir() - dbPath := filepath.Join(temp, "view.db") - handle, err := Driver{}.Open("default", map[string]any{"path": dbPath, "external_table_storage": true}, storage.MustNew(temp, nil), activity.NewNoopClient(), zap.NewNop()) + handle, err := Driver{}.Open("default", map[string]any{}, storage.MustNew(temp, nil), activity.NewNoopClient(), zap.NewNop()) require.NoError(t, err) c := handle.(*connection) require.NoError(t, c.Migrate(context.Background())) c.AsOLAP("default") - err = c.CreateTableAsSelect(context.Background(), "test-insert", false, "select 1", nil) + err = c.CreateTableAsSelect(context.Background(), "test-insert", "select 1", &drivers.CreateTableOptions{}) require.NoError(t, err) - err = c.InsertTableAsSelect(context.Background(), "test-insert", "select 2", false, true, drivers.IncrementalStrategyAppend, nil) + opts := &drivers.InsertTableOptions{ + ByName: false, + InPlace: true, + Strategy: drivers.IncrementalStrategyAppend, + } + err = c.InsertTableAsSelect(context.Background(), "test-insert", "select 2", opts) require.NoError(t, err) - err = c.InsertTableAsSelect(context.Background(), "test-insert", "select 3", true, true, drivers.IncrementalStrategyAppend, nil) + opts = &drivers.InsertTableOptions{ + ByName: true, + InPlace: true, + Strategy: drivers.IncrementalStrategyAppend, + } + err = c.InsertTableAsSelect(context.Background(), "test-insert", "select 3", opts) require.Error(t, err) res, err := c.Execute(context.Background(), &drivers.Statement{Query: "SELECT count(*) FROM 'test-insert'"}) @@ -199,9 +161,8 @@ func Test_connection_InsertTableAsSelect(t *testing.T) { require.NoError(t, res.Close()) } -func Test_connection_RenameTable(t *testing.T) { +func Test_connection_InsertTableAsSelect_WithMergeStrategy(t *testing.T) { temp := t.TempDir() - os.Mkdir(temp, fs.ModePerm) dbPath := filepath.Join(temp, "view.db") handle, err := Driver{}.Open("default", map[string]any{"path": dbPath, "external_table_storage": true}, storage.MustNew(temp, nil), activity.NewNoopClient(), zap.NewNop()) @@ -210,10 +171,60 @@ func Test_connection_RenameTable(t *testing.T) { require.NoError(t, c.Migrate(context.Background())) c.AsOLAP("default") - err = c.CreateTableAsSelect(context.Background(), "test-rename", false, "select 1", nil) + err = c.CreateTableAsSelect(context.Background(), "test-merge", "SELECT range, 'insert' AS strategy FROM range(0, 4)", &drivers.CreateTableOptions{}) require.NoError(t, err) - err = c.RenameTable(context.Background(), "test-rename", "rename-test", false) + opts := &drivers.InsertTableOptions{ + ByName: false, + InPlace: true, + Strategy: drivers.IncrementalStrategyMerge, + UniqueKey: []string{"range"}, + } + err = c.InsertTableAsSelect(context.Background(), "test-merge", "SELECT range, 'merge' AS strategy FROM range(2, 4)", opts) + require.NoError(t, err) + + res, err := c.Execute(context.Background(), &drivers.Statement{Query: "SELECT range, strategy FROM 'test-merge' ORDER BY range"}) + require.NoError(t, err) + + var results []struct { + Range int + Strategy string + } + for res.Next() { + var r struct { + Range int + Strategy string + } + require.NoError(t, res.Scan(&r.Range, &r.Strategy)) + results = append(results, r) + } + require.NoError(t, res.Close()) + + exptected := []struct { + Range int + Strategy string + }{ + {0, "insert"}, + {1, "insert"}, + {2, "merge"}, + {3, "merge"}, + } + require.Equal(t, exptected, results) +} + +func Test_connection_RenameTable(t *testing.T) { + temp := t.TempDir() + + handle, err := Driver{}.Open("default", map[string]any{}, storage.MustNew(temp, nil), activity.NewNoopClient(), zap.NewNop()) + require.NoError(t, err) + c := handle.(*connection) + require.NoError(t, c.Migrate(context.Background())) + c.AsOLAP("default") + + err = c.CreateTableAsSelect(context.Background(), "test-rename", "select 1", &drivers.CreateTableOptions{}) + require.NoError(t, err) + + err = c.RenameTable(context.Background(), "test-rename", "rename-test") require.NoError(t, err) res, err := c.Execute(context.Background(), &drivers.Statement{Query: "SELECT count(*) FROM 'rename-test'"}) @@ -227,22 +238,19 @@ func Test_connection_RenameTable(t *testing.T) { func Test_connection_RenameToExistingTable(t *testing.T) { temp := t.TempDir() - os.Mkdir(temp, fs.ModePerm) - - dbPath := filepath.Join(temp, "default", "view.db") - handle, err := Driver{}.Open("default", map[string]any{"path": dbPath, "external_table_storage": true}, storage.MustNew(temp, nil), activity.NewNoopClient(), zap.NewNop()) + handle, err := Driver{}.Open("default", map[string]any{}, storage.MustNew(temp, nil), activity.NewNoopClient(), zap.NewNop()) require.NoError(t, err) c := handle.(*connection) require.NoError(t, c.Migrate(context.Background())) c.AsOLAP("default") - err = c.CreateTableAsSelect(context.Background(), "source", false, "SELECT 1 AS data", nil) + err = c.CreateTableAsSelect(context.Background(), "source", "SELECT 1 AS data", &drivers.CreateTableOptions{}) require.NoError(t, err) - err = c.CreateTableAsSelect(context.Background(), "_tmp_source", false, "SELECT 2 AS DATA", nil) + err = c.CreateTableAsSelect(context.Background(), "_tmp_source", "SELECT 2 AS DATA", &drivers.CreateTableOptions{}) require.NoError(t, err) - err = c.RenameTable(context.Background(), "_tmp_source", "source", false) + err = c.RenameTable(context.Background(), "_tmp_source", "source") require.NoError(t, err) res, err := c.Execute(context.Background(), &drivers.Statement{Query: "SELECT * FROM 'source'"}) @@ -258,17 +266,16 @@ func Test_connection_AddTableColumn(t *testing.T) { temp := t.TempDir() os.Mkdir(temp, fs.ModePerm) - dbPath := filepath.Join(temp, "view.db") - handle, err := Driver{}.Open("default", map[string]any{"path": dbPath, "external_table_storage": true}, storage.MustNew(temp, nil), activity.NewNoopClient(), zap.NewNop()) + handle, err := Driver{}.Open("default", map[string]any{}, storage.MustNew(temp, nil), activity.NewNoopClient(), zap.NewNop()) require.NoError(t, err) c := handle.(*connection) require.NoError(t, c.Migrate(context.Background())) c.AsOLAP("default") - err = c.CreateTableAsSelect(context.Background(), "test alter column", false, "select 1 as data", nil) + err = c.CreateTableAsSelect(context.Background(), "test alter column", "select 1 as data", &drivers.CreateTableOptions{}) require.NoError(t, err) - res, err := c.Execute(context.Background(), &drivers.Statement{Query: "SELECT data_type FROM information_schema.columns WHERE table_name='test alter column' AND table_catalog = 'view'"}) + res, err := c.Execute(context.Background(), &drivers.Statement{Query: "SELECT data_type FROM information_schema.columns WHERE table_name='test alter column'"}) require.NoError(t, err) require.True(t, res.Next()) var typ string @@ -279,7 +286,7 @@ func Test_connection_AddTableColumn(t *testing.T) { err = c.AlterTableColumn(context.Background(), "test alter column", "data", "VARCHAR") require.NoError(t, err) - res, err = c.Execute(context.Background(), &drivers.Statement{Query: "SELECT data_type FROM information_schema.columns WHERE table_name='test alter column' AND table_catalog = 'view'"}) + res, err = c.Execute(context.Background(), &drivers.Statement{Query: "SELECT data_type FROM information_schema.columns WHERE table_name='test alter column' AND table_schema=current_schema()"}) require.NoError(t, err) require.True(t, res.Next()) require.NoError(t, res.Scan(&typ)) @@ -288,19 +295,19 @@ func Test_connection_AddTableColumn(t *testing.T) { } func Test_connection_RenameToExistingTableOld(t *testing.T) { - handle, err := Driver{}.Open("default", map[string]any{"dsn": ":memory:", "external_table_storage": false}, storage.MustNew(t.TempDir(), nil), activity.NewNoopClient(), zap.NewNop()) + handle, err := Driver{}.Open("default", map[string]any{}, storage.MustNew(t.TempDir(), nil), activity.NewNoopClient(), zap.NewNop()) require.NoError(t, err) c := handle.(*connection) require.NoError(t, c.Migrate(context.Background())) c.AsOLAP("default") - err = c.CreateTableAsSelect(context.Background(), "source", false, "SELECT 1 AS data", nil) + err = c.CreateTableAsSelect(context.Background(), "source", "SELECT 1 AS data", &drivers.CreateTableOptions{}) require.NoError(t, err) - err = c.CreateTableAsSelect(context.Background(), "_tmp_source", false, "SELECT 2 AS DATA", nil) + err = c.CreateTableAsSelect(context.Background(), "_tmp_source", "SELECT 2 AS DATA", &drivers.CreateTableOptions{}) require.NoError(t, err) - err = c.RenameTable(context.Background(), "_tmp_source", "source", false) + err = c.RenameTable(context.Background(), "_tmp_source", "source") require.NoError(t, err) res, err := c.Execute(context.Background(), &drivers.Statement{Query: "SELECT * FROM 'source'"}) @@ -312,57 +319,10 @@ func Test_connection_RenameToExistingTableOld(t *testing.T) { require.NoError(t, res.Close()) } -func Test_connection_CastEnum(t *testing.T) { - temp := t.TempDir() - os.Mkdir(temp, fs.ModePerm) - - dbPath := filepath.Join(temp, "view.db") - handle, err := Driver{}.Open("default", map[string]any{"path": dbPath, "external_table_storage": true}, storage.MustNew(temp, nil), activity.NewNoopClient(), zap.NewNop()) - require.NoError(t, err) - c := handle.(*connection) - require.NoError(t, c.Migrate(context.Background())) - c.AsOLAP("default") - - err = c.CreateTableAsSelect(context.Background(), "test", false, "SELECT 1 AS id, 'bglr' AS city, 'IND' AS country", nil) - require.NoError(t, err) - - err = c.InsertTableAsSelect(context.Background(), "test", "SELECT 2, 'mUm', 'IND'", false, true, drivers.IncrementalStrategyAppend, nil) - require.NoError(t, err) - - err = c.InsertTableAsSelect(context.Background(), "test", "SELECT 3, 'Perth', 'Aus'", false, true, drivers.IncrementalStrategyAppend, nil) - require.NoError(t, err) - - err = c.InsertTableAsSelect(context.Background(), "test", "SELECT 3, null, 'Aus'", false, true, drivers.IncrementalStrategyAppend, nil) - require.NoError(t, err) - - err = c.InsertTableAsSelect(context.Background(), "test", "SELECT 3, 'bglr', null", false, true, drivers.IncrementalStrategyAppend, nil) - require.NoError(t, err) - - err = c.convertToEnum(context.Background(), "test", []string{"city", "country"}) - require.NoError(t, err) - - res, err := c.Execute(context.Background(), &drivers.Statement{Query: "SELECT data_type FROM information_schema.columns WHERE column_name='city' AND table_name='test' AND table_catalog = 'view'"}) - require.NoError(t, err) - - var typ string - require.True(t, res.Next()) - require.NoError(t, res.Scan(&typ)) - require.Equal(t, "ENUM('bglr', 'Perth', 'mUm')", typ) - require.NoError(t, res.Close()) - - res, err = c.Execute(context.Background(), &drivers.Statement{Query: "SELECT data_type FROM information_schema.columns WHERE column_name='country' AND table_name='test' AND table_catalog = 'view'"}) - require.NoError(t, err) - require.True(t, res.Next()) - require.NoError(t, res.Scan(&typ)) - require.Equal(t, "ENUM('Aus', 'IND')", typ) - require.NoError(t, res.Close()) -} - func Test_connection_CreateTableAsSelectWithComments(t *testing.T) { temp := t.TempDir() require.NoError(t, os.Mkdir(filepath.Join(temp, "default"), fs.ModePerm)) - dbPath := filepath.Join(temp, "default", "normal.db") - handle, err := Driver{}.Open("default", map[string]any{"path": dbPath, "external_table_storage": false}, storage.MustNew(temp, nil), activity.NewNoopClient(), zap.NewNop()) + handle, err := Driver{}.Open("default", map[string]any{}, storage.MustNew(temp, nil), activity.NewNoopClient(), zap.NewNop()) require.NoError(t, err) normalConn := handle.(*connection) normalConn.AsOLAP("default") @@ -376,75 +336,21 @@ func Test_connection_CreateTableAsSelectWithComments(t *testing.T) { -- that was a stupid query -- I hope to write not so stupid query ` - err = normalConn.CreateTableAsSelect(ctx, "test", false, sql, nil) + err = normalConn.CreateTableAsSelect(ctx, "test", sql, &drivers.CreateTableOptions{}) require.NoError(t, err) - err = normalConn.CreateTableAsSelect(ctx, "test_view", true, sql, nil) + err = normalConn.CreateTableAsSelect(ctx, "test_view", sql, &drivers.CreateTableOptions{View: true}) require.NoError(t, err) sql = ` with r as (select 1 as id ) select * from r ` - err = normalConn.CreateTableAsSelect(ctx, "test", false, sql, nil) - require.NoError(t, err) - - err = normalConn.CreateTableAsSelect(ctx, "test_view", true, sql, nil) - require.NoError(t, err) -} - -func Test_connection_ChangingOrder(t *testing.T) { - temp := t.TempDir() - os.Mkdir(temp, fs.ModePerm) - - // on cloud - dbPath := filepath.Join(temp, "view.db") - handle, err := Driver{}.Open("default", map[string]any{"path": dbPath, "external_table_storage": true, "allow_host_access": false}, storage.MustNew(temp, nil), activity.NewNoopClient(), zap.NewNop()) - require.NoError(t, err) - c := handle.(*connection) - require.NoError(t, c.Migrate(context.Background())) - c.AsOLAP("default") - - // create table - err = c.CreateTableAsSelect(context.Background(), "test", false, "SELECT 1 AS id, 'India' AS 'coun\"try'", nil) - require.NoError(t, err) - - // create view - err = c.CreateTableAsSelect(context.Background(), "test_view", true, "SELECT * FROM test", nil) - require.NoError(t, err) - verifyCount(t, c, "test_view", 1) - - // change sequence - err = c.CreateTableAsSelect(context.Background(), "test", false, "SELECT 'India' AS 'coun\"try', 1 AS id", nil) + err = normalConn.CreateTableAsSelect(ctx, "test", sql, &drivers.CreateTableOptions{}) require.NoError(t, err) - // view should still work - verifyCount(t, c, "test_view", 1) - // on local - dbPath = filepath.Join(temp, "local.db") - handle, err = Driver{}.Open("default", map[string]any{"path": dbPath, "external_table_storage": true, "allow_host_access": true}, storage.MustNew(temp, nil), activity.NewNoopClient(), zap.NewNop()) + err = normalConn.CreateTableAsSelect(ctx, "test_view", sql, &drivers.CreateTableOptions{View: true}) require.NoError(t, err) - c = handle.(*connection) - require.NoError(t, c.Migrate(context.Background())) - c.AsOLAP("default") - - // create table - err = c.CreateTableAsSelect(context.Background(), "test", false, "SELECT 1 AS id, 'India' AS 'coun\"try'", nil) - require.NoError(t, err) - - // create view - err = c.CreateTableAsSelect(context.Background(), "test_view", true, "SELECT * FROM test", nil) - require.NoError(t, err) - verifyCount(t, c, "test_view", 1) - - // change sequence - err = c.CreateTableAsSelect(context.Background(), "test", false, "SELECT 'India' AS 'coun\"try', 1 AS id", nil) - require.NoError(t, err) - - // view no longer works - _, err = c.Execute(context.Background(), &drivers.Statement{Query: "SELECT count(*) from test_view"}) - require.Error(t, err) - require.Contains(t, err.Error(), "Binder Error: Contents of view were altered: types don't match!") } func verifyCount(t *testing.T, c *connection, table string, expected int) { diff --git a/runtime/drivers/duckdb/olap_test.go b/runtime/drivers/duckdb/olap_test.go index 38d7fa3a3af..37edaacc6ac 100644 --- a/runtime/drivers/duckdb/olap_test.go +++ b/runtime/drivers/duckdb/olap_test.go @@ -2,7 +2,6 @@ package duckdb import ( "context" - "fmt" "io/fs" "os" "path/filepath" @@ -213,30 +212,16 @@ func TestClose(t *testing.T) { } func prepareConn(t *testing.T) drivers.Handle { - conn, err := Driver{}.Open("default", map[string]any{"dsn": ":memory:?access_mode=read_write", "pool_size": 4, "external_table_storage": false}, storage.MustNew(t.TempDir(), nil), activity.NewNoopClient(), zap.NewNop()) + conn, err := Driver{}.Open("default", map[string]any{}, storage.MustNew(t.TempDir(), nil), activity.NewNoopClient(), zap.NewNop()) require.NoError(t, err) olap, ok := conn.AsOLAP("") require.True(t, ok) - err = olap.Exec(context.Background(), &drivers.Statement{ - Query: "CREATE TABLE foo(bar VARCHAR, baz INTEGER)", - }) + err = olap.CreateTableAsSelect(context.Background(), "foo", "SELECT * FROM (VALUES ('a', 1), ('a', 2), ('b', 3), ('c', 4)) AS t(bar, baz)", &drivers.CreateTableOptions{}) require.NoError(t, err) - err = olap.Exec(context.Background(), &drivers.Statement{ - Query: "INSERT INTO foo VALUES ('a', 1), ('a', 2), ('b', 3), ('c', 4)", - }) - require.NoError(t, err) - - err = olap.Exec(context.Background(), &drivers.Statement{ - Query: "CREATE TABLE bar(bar VARCHAR, baz INTEGER)", - }) - require.NoError(t, err) - - err = olap.Exec(context.Background(), &drivers.Statement{ - Query: "INSERT INTO bar VALUES ('a', 1), ('a', 2), ('b', 3), ('c', 4)", - }) + err = olap.CreateTableAsSelect(context.Background(), "bar", "SELECT * FROM (VALUES ('a', 1), ('a', 2), ('b', 3), ('c', 4)) AS t(bar, baz)", &drivers.CreateTableOptions{}) require.NoError(t, err) return conn @@ -248,20 +233,8 @@ func Test_safeSQLString(t *testing.T) { err := os.Mkdir(path, fs.ModePerm) require.NoError(t, err) - dbFile := filepath.Join(path, "st@g3's.db") - conn, err := Driver{}.Open("default", map[string]any{"path": dbFile, "external_table_storage": false}, storage.MustNew(tempDir, nil), activity.NewNoopClient(), zap.NewNop()) + conn, err := Driver{}.Open("default", map[string]any{"data_dir": path}, storage.MustNew(tempDir, nil), activity.NewNoopClient(), zap.NewNop()) require.NoError(t, err) + require.NotNil(t, conn) require.NoError(t, conn.Close()) - - conn, err = Driver{}.Open("default", map[string]any{"external_table_storage": false}, storage.MustNew(tempDir, nil), activity.NewNoopClient(), zap.NewNop()) - require.NoError(t, err) - - olap, ok := conn.AsOLAP("") - require.True(t, ok) - - err = olap.Exec(context.Background(), &drivers.Statement{Query: fmt.Sprintf("ATTACH '%s'", dbFile)}) - require.Error(t, err) - - err = olap.Exec(context.Background(), &drivers.Statement{Query: fmt.Sprintf("ATTACH %s", safeSQLString(dbFile))}) - require.NoError(t, err) } diff --git a/runtime/drivers/duckdb/transporter_duckDB_to_duckDB.go b/runtime/drivers/duckdb/transporter_duckDB_to_duckDB.go index e70a7baaa79..872f535d676 100644 --- a/runtime/drivers/duckdb/transporter_duckDB_to_duckDB.go +++ b/runtime/drivers/duckdb/transporter_duckDB_to_duckDB.go @@ -2,28 +2,32 @@ package duckdb import ( "context" - "database/sql" "errors" "fmt" + "net" "net/url" - "path/filepath" "strings" + "github.com/go-sql-driver/mysql" + "github.com/jmoiron/sqlx" "github.com/rilldata/rill/runtime/drivers" "github.com/rilldata/rill/runtime/pkg/duckdbsql" "github.com/rilldata/rill/runtime/pkg/fileutil" + "github.com/rilldata/rill/runtime/pkg/rduckdb" "go.uber.org/zap" ) type duckDBToDuckDB struct { - to drivers.OLAPStore - logger *zap.Logger + to *connection + logger *zap.Logger + externalDBType string // mysql, postgres, duckdb } -func NewDuckDBToDuckDB(to drivers.OLAPStore, logger *zap.Logger) drivers.Transporter { +func newDuckDBToDuckDB(c *connection, db string, logger *zap.Logger) drivers.Transporter { return &duckDBToDuckDB{ - to: to, - logger: logger, + to: c, + logger: logger, + externalDBType: db, } } @@ -43,7 +47,7 @@ func (t *duckDBToDuckDB) Transfer(ctx context.Context, srcProps, sinkProps map[s t.logger = t.logger.With(zap.String("source", sinkCfg.Table)) if srcCfg.Database != "" { // query to be run against an external DB - if !strings.HasPrefix(srcCfg.Database, "md:") { + if t.externalDBType == "duckdb" { srcCfg.Database, err = fileutil.ResolveLocalPath(srcCfg.Database, opts.RepoRoot, opts.AllowHostAccess) if err != nil { return err @@ -112,67 +116,70 @@ func (t *duckDBToDuckDB) Transfer(ctx context.Context, srcProps, sinkProps map[s srcCfg.SQL = rewrittenSQL } - return t.to.CreateTableAsSelect(ctx, sinkCfg.Table, false, srcCfg.SQL, nil) + return t.to.CreateTableAsSelect(ctx, sinkCfg.Table, srcCfg.SQL, &drivers.CreateTableOptions{}) } func (t *duckDBToDuckDB) transferFromExternalDB(ctx context.Context, srcProps *dbSourceProperties, sinkProps *sinkProperties) error { - var cleanupFunc func() - err := t.to.WithConnection(ctx, 1, true, false, func(ctx, ensuredCtx context.Context, _ *sql.Conn) error { - res, err := t.to.Execute(ctx, &drivers.Statement{Query: "SELECT current_database(),current_schema();"}) - if err != nil { - return err - } - - var localDB, localSchema string - for res.Next() { - if err := res.Scan(&localDB, &localSchema); err != nil { - _ = res.Close() + var initSQL []string + safeDBName := safeName(sinkProps.Table + "_external_db_") + safeTempTable := safeName(sinkProps.Table + "__temp__") + switch t.externalDBType { + case "mysql": + dsn := rewriteMySQLDSN(srcProps.Database) + initSQL = append(initSQL, "INSTALL 'MYSQL'; LOAD 'MYSQL';", fmt.Sprintf("ATTACH %s AS %s (TYPE mysql, READ_ONLY)", safeSQLString(dsn), safeDBName)) + case "postgres": + initSQL = append(initSQL, "INSTALL 'POSTGRES'; LOAD 'POSTGRES';", fmt.Sprintf("ATTACH %s AS %s (TYPE postgres, READ_ONLY)", safeSQLString(srcProps.Database), safeDBName)) + case "duckdb": + initSQL = append(initSQL, fmt.Sprintf("ATTACH %s AS %s (READ_ONLY)", safeSQLString(srcProps.Database), safeDBName)) + default: + return fmt.Errorf("internal error: unsupported external database: %s", t.externalDBType) + } + beforeCreateFn := func(ctx context.Context, conn *sqlx.Conn) error { + for _, sql := range initSQL { + _, err := conn.ExecContext(ctx, sql) + if err != nil { return err } } - _ = res.Close() - - // duckdb considers everything before first . as db name - // alternative solution can be to query `show databases()` before and after to identify db name - dbName, _, _ := strings.Cut(filepath.Base(srcProps.Database), ".") - if dbName == "main" { - return fmt.Errorf("`main` is a reserved db name") - } - - if err = t.to.Exec(ctx, &drivers.Statement{Query: fmt.Sprintf("ATTACH %s AS %s", safeSQLString(srcProps.Database), safeSQLName(dbName))}); err != nil { - return fmt.Errorf("failed to attach db %q: %w", srcProps.Database, err) - } - cleanupFunc = func() { - // we don't want to run any detach db without `tx` lock - // tx=true will reopen duckdb handle(except in case of in-memory duckdb handle) which will detach the attached external db as well - err := t.to.WithConnection(context.Background(), 100, false, true, func(wrappedCtx, ensuredCtx context.Context, conn *sql.Conn) error { - return nil - }) - if err != nil { - t.logger.Debug("failed to detach db", zap.Error(err)) - } + var localDB, localSchema string + err := conn.QueryRowxContext(ctx, "SELECT current_database(),current_schema();").Scan(&localDB, &localSchema) + if err != nil { + return err } - if err := t.to.Exec(ctx, &drivers.Statement{Query: fmt.Sprintf("USE %s;", safeName(dbName))}); err != nil { + _, err = conn.ExecContext(ctx, fmt.Sprintf("USE %s;", safeDBName)) + if err != nil { return err } - defer func() { // revert back to localdb - if err = t.to.Exec(ensuredCtx, &drivers.Statement{Query: fmt.Sprintf("USE %s.%s;", safeName(localDB), safeName(localSchema))}); err != nil { - t.logger.Error("failed to switch to local database", zap.Error(err)) - } - }() - userQuery := strings.TrimSpace(srcProps.SQL) userQuery, _ = strings.CutSuffix(userQuery, ";") // trim trailing semi colon - query := fmt.Sprintf("CREATE OR REPLACE TABLE %s.%s.%s AS (%s\n);", safeName(localDB), safeName(localSchema), safeName(sinkProps.Table), userQuery) - return t.to.Exec(ctx, &drivers.Statement{Query: query}) - }) - if cleanupFunc != nil { - cleanupFunc() + query := fmt.Sprintf("CREATE OR REPLACE TABLE %s.%s.%s AS (%s\n);", safeName(localDB), safeName(localSchema), safeTempTable, userQuery) + _, err = conn.ExecContext(ctx, query) + // first revert back to localdb + if err != nil { + return err + } + // revert to localdb and schema before returning + _, err = conn.ExecContext(ctx, fmt.Sprintf("USE %s.%s;", safeName(localDB), safeName(localSchema))) + return err + } + afterCreateFn := func(ctx context.Context, conn *sqlx.Conn) error { + _, err := conn.ExecContext(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %s", safeTempTable)) + return err } - return err + db, release, err := t.to.acquireDB() + if err != nil { + return err + } + defer func() { + _ = release() + }() + return db.CreateTableAsSelect(ctx, sinkProps.Table, fmt.Sprintf("SELECT * FROM %s", safeTempTable), &rduckdb.CreateTableOptions{ + BeforeCreateFn: beforeCreateFn, + AfterCreateFn: afterCreateFn, + }) } // rewriteLocalPaths rewrites a DuckDB SQL statement such that relative paths become absolute paths relative to the basePath, @@ -205,3 +212,44 @@ func rewriteLocalPaths(ast *duckdbsql.AST, basePath string, allowHostAccess bool return ast.Format() } + +// rewriteMySQLDSN rewrites a MySQL DSN to a format that DuckDB expects. +// DuckDB does not support the URI based DSN format yet. It expects the DSN to be in the form of key=value pairs. +// This function parses the MySQL URI based DSN and converts it to the key=value format. It only converts the common parameters. +// For more advanced parameters like SSL configs, the user should manually convert the DSN to the key=value format. +// If there is an error parsing the DSN, it returns the DSN as is. +func rewriteMySQLDSN(dsn string) string { + cfg, err := mysql.ParseDSN(dsn) + if err != nil { + // If we can't parse the DSN, just return it as is. May be it is already in the form duckdb expects. + return dsn + } + + var sb strings.Builder + + if cfg.User != "" { + sb.WriteString(fmt.Sprintf("user=%s ", cfg.User)) + } + if cfg.Passwd != "" { + sb.WriteString(fmt.Sprintf("password=%s ", cfg.Passwd)) + } + if cfg.DBName != "" { + sb.WriteString(fmt.Sprintf("database=%s ", cfg.DBName)) + } + switch cfg.Net { + case "unix": + sb.WriteString(fmt.Sprintf("socket=%s ", cfg.Addr)) + case "tcp", "tcp6": + host, port, err := net.SplitHostPort(cfg.Addr) + if err != nil { + return dsn + } + sb.WriteString(fmt.Sprintf("host=%s ", host)) + if port != "" { + sb.WriteString(fmt.Sprintf("port=%s ", port)) + } + default: + return dsn + } + return sb.String() +} diff --git a/runtime/drivers/duckdb/transporter_duckDB_to_duckDB_test.go b/runtime/drivers/duckdb/transporter_duckDB_to_duckDB_test.go index 27fc71672d8..1da46f1cdd5 100644 --- a/runtime/drivers/duckdb/transporter_duckDB_to_duckDB_test.go +++ b/runtime/drivers/duckdb/transporter_duckDB_to_duckDB_test.go @@ -2,10 +2,11 @@ package duckdb import ( "context" - "fmt" + "database/sql" "path/filepath" "testing" + _ "github.com/marcboeker/go-duckdb" "github.com/rilldata/rill/runtime/drivers" activity "github.com/rilldata/rill/runtime/pkg/activity" "github.com/rilldata/rill/runtime/storage" @@ -15,35 +16,27 @@ import ( func TestDuckDBToDuckDBTransfer(t *testing.T) { tempDir := t.TempDir() - conn, err := Driver{}.Open("default", map[string]any{"path": fmt.Sprintf("%s.db", filepath.Join(tempDir, "tranfser")), "external_table_storage": false}, storage.MustNew(tempDir, nil), activity.NewNoopClient(), zap.NewNop()) + dbFile := filepath.Join(tempDir, "transfer.db") + db, err := sql.Open("duckdb", dbFile) require.NoError(t, err) - olap, ok := conn.AsOLAP("") - require.True(t, ok) - - err = olap.Exec(context.Background(), &drivers.Statement{ - Query: "CREATE TABLE foo(bar VARCHAR, baz INTEGER)", - }) + _, err = db.ExecContext(context.Background(), "CREATE TABLE foo(bar VARCHAR, baz INTEGER)") require.NoError(t, err) - err = olap.Exec(context.Background(), &drivers.Statement{ - Query: "INSERT INTO foo VALUES ('a', 1), ('a', 2), ('b', 3), ('c', 4)", - }) + _, err = db.ExecContext(context.Background(), "INSERT INTO foo VALUES ('a', 1), ('a', 2), ('b', 3), ('c', 4)") require.NoError(t, err) - require.NoError(t, conn.Close()) + require.NoError(t, db.Close()) - to, err := Driver{}.Open("default", map[string]any{"path": filepath.Join(tempDir, "main.db"), "external_table_storage": false}, storage.MustNew(tempDir, nil), activity.NewNoopClient(), zap.NewNop()) + to, err := Driver{}.Open("default", map[string]any{}, storage.MustNew(tempDir, nil), activity.NewNoopClient(), zap.NewNop()) require.NoError(t, err) - olap, _ = to.AsOLAP("") - - tr := NewDuckDBToDuckDB(olap, zap.NewNop()) + tr := newDuckDBToDuckDB(to.(*connection), "duckdb", zap.NewNop()) // transfer once - err = tr.Transfer(context.Background(), map[string]any{"sql": "SELECT * FROM foo", "db": filepath.Join(tempDir, "tranfser.db")}, map[string]any{"table": "test"}, &drivers.TransferOptions{}) + err = tr.Transfer(context.Background(), map[string]any{"sql": "SELECT * FROM foo", "db": dbFile}, map[string]any{"table": "test"}, &drivers.TransferOptions{}) require.NoError(t, err) - rows, err := olap.Execute(context.Background(), &drivers.Statement{Query: "SELECT COUNT(*) FROM test"}) + rows, err := to.(*connection).Execute(context.Background(), &drivers.Statement{Query: "SELECT COUNT(*) FROM test"}) require.NoError(t, err) var count int @@ -53,10 +46,10 @@ func TestDuckDBToDuckDBTransfer(t *testing.T) { require.NoError(t, rows.Close()) // transfer again - err = tr.Transfer(context.Background(), map[string]any{"sql": "SELECT * FROM foo", "db": filepath.Join(tempDir, "tranfser.db")}, map[string]any{"table": "test"}, &drivers.TransferOptions{}) + err = tr.Transfer(context.Background(), map[string]any{"sql": "SELECT * FROM foo", "db": dbFile}, map[string]any{"table": "test"}, &drivers.TransferOptions{}) require.NoError(t, err) - rows, err = olap.Execute(context.Background(), &drivers.Statement{Query: "SELECT COUNT(*) FROM test"}) + rows, err = to.(*connection).Execute(context.Background(), &drivers.Statement{Query: "SELECT COUNT(*) FROM test"}) require.NoError(t, err) rows.Next() diff --git a/runtime/drivers/duckdb/transporter_filestore_to_duckDB.go b/runtime/drivers/duckdb/transporter_filestore_to_duckDB.go index 49e73d0598e..6a53a75bc0e 100644 --- a/runtime/drivers/duckdb/transporter_filestore_to_duckDB.go +++ b/runtime/drivers/duckdb/transporter_filestore_to_duckDB.go @@ -60,7 +60,7 @@ func (t *fileStoreToDuckDB) Transfer(ctx context.Context, srcProps, sinkProps ma return err } - err = t.to.CreateTableAsSelect(ctx, sinkCfg.Table, false, fmt.Sprintf("SELECT * FROM %s", from), nil) + err = t.to.CreateTableAsSelect(ctx, sinkCfg.Table, fmt.Sprintf("SELECT * FROM %s", from), &drivers.CreateTableOptions{}) if err != nil { return err } diff --git a/runtime/drivers/duckdb/transporter_motherduck_to_duckDB.go b/runtime/drivers/duckdb/transporter_motherduck_to_duckDB.go index 6a7cb2b54f0..cd8560af805 100644 --- a/runtime/drivers/duckdb/transporter_motherduck_to_duckDB.go +++ b/runtime/drivers/duckdb/transporter_motherduck_to_duckDB.go @@ -2,18 +2,19 @@ package duckdb import ( "context" - "database/sql" "fmt" "os" "strings" + "github.com/jmoiron/sqlx" "github.com/mitchellh/mapstructure" "github.com/rilldata/rill/runtime/drivers" + "github.com/rilldata/rill/runtime/pkg/rduckdb" "go.uber.org/zap" ) type motherduckToDuckDB struct { - to drivers.OLAPStore + to *connection from drivers.Handle logger *zap.Logger } @@ -31,7 +32,7 @@ type mdConfigProps struct { var _ drivers.Transporter = &motherduckToDuckDB{} -func NewMotherduckToDuckDB(from drivers.Handle, to drivers.OLAPStore, logger *zap.Logger) drivers.Transporter { +func newMotherduckToDuckDB(from drivers.Handle, to *connection, logger *zap.Logger) drivers.Transporter { return &motherduckToDuckDB{ to: to, from: from, @@ -73,48 +74,33 @@ func (t *motherduckToDuckDB) Transfer(ctx context.Context, srcProps, sinkProps m return fmt.Errorf("no motherduck token found. Refer to this documentation for instructions: https://docs.rilldata.com/reference/connectors/motherduck") } - t.logger = t.logger.With(zap.String("source", sinkCfg.Table)) - - // we first ingest data in a temporary table in the main db - // and then copy it to the final table to ensure that the final table is always created using CRUD APIs which takes care - // whether table goes in main db or in separate table specific db - tmpTable := fmt.Sprintf("__%s_tmp_motherduck", sinkCfg.Table) - defer func() { - // ensure temporary table is cleaned - err := t.to.Exec(context.Background(), &drivers.Statement{ - Query: fmt.Sprintf("DROP TABLE IF EXISTS %s", tmpTable), - Priority: 100, - LongRunning: true, - }) + beforeCreateFn := func(ctx context.Context, conn *sqlx.Conn) error { + _, err := conn.ExecContext(ctx, "INSTALL 'motherduck'; LOAD 'motherduck';") if err != nil { - t.logger.Error("failed to drop temp table", zap.String("table", tmpTable), zap.Error(err)) + return fmt.Errorf("failed to load motherduck extension %w", err) } - }() - err = t.to.WithConnection(ctx, 1, true, false, func(ctx, ensuredCtx context.Context, _ *sql.Conn) error { - // load motherduck extension; connect to motherduck service - err = t.to.Exec(ctx, &drivers.Statement{Query: "INSTALL 'motherduck'; LOAD 'motherduck';"}) + _, err = conn.ExecContext(ctx, fmt.Sprintf("SET motherduck_token='%s'", token)) if err != nil { - return fmt.Errorf("failed to load motherduck extension %w", err) + return fmt.Errorf("failed to set motherduck token %w", err) } - if err = t.to.Exec(ctx, &drivers.Statement{Query: fmt.Sprintf("SET motherduck_token='%s'", token)}); err != nil { - if !strings.Contains(err.Error(), "can only be set during initialization") { - return fmt.Errorf("failed to set motherduck token %w", err) - } + _, err = conn.ExecContext(ctx, fmt.Sprintf("ATTACH '%s'", srcConfig.DSN)) + if err != nil { + return fmt.Errorf("failed to attach motherduck DSN: %w", err) } - - // ignore attach error since it might be already attached - _ = t.to.Exec(ctx, &drivers.Statement{Query: fmt.Sprintf("ATTACH '%s'", srcConfig.DSN)}) - userQuery := strings.TrimSpace(srcConfig.SQL) - userQuery, _ = strings.CutSuffix(userQuery, ";") // trim trailing semi colon - query := fmt.Sprintf("CREATE OR REPLACE TABLE %s AS (%s\n);", safeName(tmpTable), userQuery) - return t.to.Exec(ctx, &drivers.Statement{Query: query}) - }) + return err + } + userQuery := strings.TrimSpace(srcConfig.SQL) + userQuery, _ = strings.CutSuffix(userQuery, ";") // trim trailing semi colon + db, release, err := t.to.acquireDB() if err != nil { return err } - - // copy data from temp table to target table - return t.to.CreateTableAsSelect(ctx, sinkCfg.Table, false, fmt.Sprintf("SELECT * FROM %s", tmpTable), nil) + defer func() { + _ = release() + }() + return db.CreateTableAsSelect(ctx, sinkCfg.Table, userQuery, &rduckdb.CreateTableOptions{ + BeforeCreateFn: beforeCreateFn, + }) } diff --git a/runtime/drivers/duckdb/transporter_mysql_to_duckDB_test.go b/runtime/drivers/duckdb/transporter_mysql_to_duckDB_test.go index e011b17d3f7..db8f3575e9a 100644 --- a/runtime/drivers/duckdb/transporter_mysql_to_duckDB_test.go +++ b/runtime/drivers/duckdb/transporter_mysql_to_duckDB_test.go @@ -14,7 +14,7 @@ import ( "fmt" "time" - _ "github.com/rilldata/rill/runtime/drivers/mysql" + _ "github.com/go-sql-driver/mysql" "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" ) @@ -58,10 +58,10 @@ CREATE TABLE all_data_types_table ( sample_json JSON ); -INSERT INTO all_data_types_table (sample_char, sample_varchar, sample_tinytext, sample_text, sample_mediumtext, sample_longtext, sample_binary, sample_varbinary, sample_tinyblob, sample_blob, sample_mediumblob, sample_longblob, sample_enum, sample_set, sample_bit, sample_tinyint, sample_tinyint_unsigned, sample_smallint, sample_smallint_unsigned, sample_mediumint, sample_mediumint_unsigned, sample_int, sample_int_unsigned, sample_bigint, sample_bigint_unsigned, sample_float, sample_double, sample_decimal, sample_date, sample_datetime, sample_timestamp, sample_time, sample_year, sample_json) +INSERT INTO all_data_types_table (sample_char, sample_varchar, sample_tinytext, sample_text, sample_mediumtext, sample_longtext, sample_binary, sample_varbinary, sample_tinyblob, sample_blob, sample_mediumblob, sample_longblob, sample_enum, sample_set, sample_bit, sample_tinyint, sample_tinyint_unsigned, sample_smallint, sample_smallint_unsigned, sample_mediumint, sample_mediumint_unsigned, sample_int, sample_int_unsigned, sample_bigint, sample_bigint_unsigned, sample_float, sample_double, sample_decimal, sample_date, sample_datetime, sample_timestamp, sample_time, sample_year, sample_json) VALUES ('A', 'Sample Text', 'Tiny Text', 'Some Longer Text.', 'Medium Length Text', 'This is an example of really long text for the LONGTEXT column.', BINARY '1', 'Sample Binary', 'Tiny Blob Data', 'Sample Blob Data', 'Medium Blob Data', 'Long Blob Data', 'value1', 'value1,value2', b'10101010', -128, 255, -32768, 65535, -8388608, 16777215, -2147483648, 4294967295, -9223372036854775808, 18446744073709551615, 123.45, 1234567890.123, 12345.67, '2023-01-01', '2023-01-01 12:00:00', CURRENT_TIMESTAMP, '12:00:00', 2023, JSON_OBJECT('key', 'value')); -INSERT INTO all_data_types_table (sample_char, sample_varchar, sample_tinytext, sample_text, sample_mediumtext, sample_longtext, sample_binary, sample_varbinary, sample_tinyblob, sample_blob, sample_mediumblob, sample_longblob, sample_enum, sample_set, sample_bit, sample_tinyint, sample_tinyint_unsigned, sample_smallint, sample_smallint_unsigned, sample_mediumint, sample_mediumint_unsigned, sample_int, sample_int_unsigned, sample_bigint, sample_bigint_unsigned, sample_float, sample_double, sample_decimal, sample_date, sample_datetime, sample_timestamp, sample_time, sample_year, sample_json) +INSERT INTO all_data_types_table (sample_char, sample_varchar, sample_tinytext, sample_text, sample_mediumtext, sample_longtext, sample_binary, sample_varbinary, sample_tinyblob, sample_blob, sample_mediumblob, sample_longblob, sample_enum, sample_set, sample_bit, sample_tinyint, sample_tinyint_unsigned, sample_smallint, sample_smallint_unsigned, sample_mediumint, sample_mediumint_unsigned, sample_int, sample_int_unsigned, sample_bigint, sample_bigint_unsigned, sample_float, sample_double, sample_decimal, sample_date, sample_datetime, sample_timestamp, sample_time, sample_year, sample_json) VALUES (NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, 0, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); ` @@ -95,7 +95,9 @@ func TestMySQLToDuckDBTransfer(t *testing.T) { require.NoError(t, err) defer db.Close() - t.Run("AllDataTypes", func(t *testing.T) { allMySQLDataTypesTest(t, db, dsn) }) + t.Run("AllDataTypes", func(t *testing.T) { + allMySQLDataTypesTest(t, db, dsn) + }) } func allMySQLDataTypesTest(t *testing.T, db *sql.DB, dsn string) { @@ -103,17 +105,12 @@ func allMySQLDataTypesTest(t *testing.T, db *sql.DB, dsn string) { _, err := db.ExecContext(ctx, mysqlInitStmt) require.NoError(t, err) - handle, err := drivers.Open("mysql", "default", map[string]any{"dsn": dsn}, storage.MustNew(t.TempDir(), nil), activity.NewNoopClient(), zap.NewNop()) - require.NoError(t, err) - require.NotNil(t, handle) - - sqlStore, _ := handle.AsSQLStore() - to, err := drivers.Open("duckdb", "default", map[string]any{"dsn": ":memory:"}, storage.MustNew(t.TempDir(), nil), activity.NewNoopClient(), zap.NewNop()) + to, err := drivers.Open("duckdb", "default", map[string]any{}, storage.MustNew(t.TempDir(), nil), activity.NewNoopClient(), zap.NewNop()) require.NoError(t, err) olap, _ := to.AsOLAP("") - tr := NewSQLStoreToDuckDB(sqlStore, olap, zap.NewNop()) - err = tr.Transfer(ctx, map[string]any{"sql": "select * from all_data_types_table;"}, map[string]any{"table": "sink"}, &drivers.TransferOptions{}) + tr := newDuckDBToDuckDB(to.(*connection), "mysql", zap.NewNop()) + err = tr.Transfer(ctx, map[string]any{"sql": "select * from all_data_types_table;", "db": dsn}, map[string]any{"table": "sink"}, &drivers.TransferOptions{}) require.NoError(t, err) res, err := olap.Execute(context.Background(), &drivers.Statement{Query: "select count(*) from sink"}) require.NoError(t, err) diff --git a/runtime/drivers/duckdb/transporter_objectStore_to_duckDB.go b/runtime/drivers/duckdb/transporter_objectStore_to_duckDB.go index 24f86e9ffba..35a6ebe8d99 100644 --- a/runtime/drivers/duckdb/transporter_objectStore_to_duckDB.go +++ b/runtime/drivers/duckdb/transporter_objectStore_to_duckDB.go @@ -100,7 +100,7 @@ func (t *objectStoreToDuckDB) Transfer(ctx context.Context, srcProps, sinkProps return err } - err = t.to.CreateTableAsSelect(ctx, sinkCfg.Table, false, fmt.Sprintf("SELECT * FROM %s", from), nil) + err = t.to.CreateTableAsSelect(ctx, sinkCfg.Table, fmt.Sprintf("SELECT * FROM %s", from), &drivers.CreateTableOptions{}) if err != nil { return err } @@ -112,8 +112,7 @@ func (t *objectStoreToDuckDB) Transfer(ctx context.Context, srcProps, sinkProps } // convert to enum if len(srcCfg.CastToENUM) > 0 { - conn, _ := t.to.(*connection) - return conn.convertToEnum(ctx, sinkCfg.Table, srcCfg.CastToENUM) + return fmt.Errorf("`cast_to_enum` is not implemented") } return nil } @@ -163,7 +162,7 @@ func (t *objectStoreToDuckDB) ingestDuckDBSQL(ctx context.Context, originalSQL s return err } - err = t.to.CreateTableAsSelect(ctx, dbSink.Table, false, sql, nil) + err = t.to.CreateTableAsSelect(ctx, dbSink.Table, sql, &drivers.CreateTableOptions{}) if err != nil { return err } @@ -175,8 +174,7 @@ func (t *objectStoreToDuckDB) ingestDuckDBSQL(ctx context.Context, originalSQL s } // convert to enum if len(srcCfg.CastToENUM) > 0 { - conn, _ := t.to.(*connection) - return conn.convertToEnum(ctx, dbSink.Table, srcCfg.CastToENUM) + return fmt.Errorf("`cast_to_enum` is not implemented") } return nil } @@ -207,7 +205,12 @@ func (a *appender) appendData(ctx context.Context, files []string) error { return err } - err = a.to.InsertTableAsSelect(ctx, a.sink.Table, sql, a.allowSchemaRelaxation, true, drivers.IncrementalStrategyAppend, nil) + opts := &drivers.InsertTableOptions{ + ByName: a.allowSchemaRelaxation, + InPlace: true, + Strategy: drivers.IncrementalStrategyAppend, + } + err = a.to.InsertTableAsSelect(ctx, a.sink.Table, sql, opts) if err == nil || !a.allowSchemaRelaxation || !containsAny(err.Error(), []string{"binder error", "conversion error"}) { return err } @@ -218,7 +221,12 @@ func (a *appender) appendData(ctx context.Context, files []string) error { if err != nil { return fmt.Errorf("failed to update schema %w", err) } - return a.to.InsertTableAsSelect(ctx, a.sink.Table, sql, true, true, drivers.IncrementalStrategyAppend, nil) + opts = &drivers.InsertTableOptions{ + ByName: true, + InPlace: true, + Strategy: drivers.IncrementalStrategyAppend, + } + return a.to.InsertTableAsSelect(ctx, a.sink.Table, sql, opts) } // updateSchema updates the schema of the table in case new file adds a new column or diff --git a/runtime/drivers/duckdb/transporter_postgres_to_duckDB_test.go b/runtime/drivers/duckdb/transporter_postgres_to_duckDB_test.go index e59934d7e58..07615f425e9 100644 --- a/runtime/drivers/duckdb/transporter_postgres_to_duckDB_test.go +++ b/runtime/drivers/duckdb/transporter_postgres_to_duckDB_test.go @@ -68,17 +68,12 @@ func allDataTypesTest(t *testing.T, db *sql.DB, dbURL string) { _, err := db.ExecContext(ctx, sqlStmt) require.NoError(t, err) - handle, err := drivers.Open("postgres", "default", map[string]any{"database_url": dbURL}, storage.MustNew(t.TempDir(), nil), activity.NewNoopClient(), zap.NewNop()) - require.NoError(t, err) - require.NotNil(t, handle) - - sqlStore, _ := handle.AsSQLStore() - to, err := drivers.Open("duckdb", "default", map[string]any{"dsn": ":memory:"}, storage.MustNew(t.TempDir(), nil), activity.NewNoopClient(), zap.NewNop()) + to, err := drivers.Open("duckdb", "default", map[string]any{}, storage.MustNew(t.TempDir(), nil), activity.NewNoopClient(), zap.NewNop()) require.NoError(t, err) olap, _ := to.AsOLAP("") - tr := NewSQLStoreToDuckDB(sqlStore, olap, zap.NewNop()) - err = tr.Transfer(ctx, map[string]any{"sql": "select * from all_datatypes;"}, map[string]any{"table": "sink"}, &drivers.TransferOptions{}) + tr := newDuckDBToDuckDB(to.(*connection), "postgres", zap.NewNop()) + err = tr.Transfer(ctx, map[string]any{"sql": "select * from all_datatypes;", "db": dbURL}, map[string]any{"table": "sink"}, &drivers.TransferOptions{}) require.NoError(t, err) res, err := olap.Execute(context.Background(), &drivers.Statement{Query: "select count(*) from sink"}) require.NoError(t, err) diff --git a/runtime/drivers/duckdb/transporter_sqlite_to_duckDB_test.go b/runtime/drivers/duckdb/transporter_sqlite_to_duckDB_test.go index b99a382a965..cf0c342aeba 100644 --- a/runtime/drivers/duckdb/transporter_sqlite_to_duckDB_test.go +++ b/runtime/drivers/duckdb/transporter_sqlite_to_duckDB_test.go @@ -35,7 +35,7 @@ func Test_sqliteToDuckDB_Transfer(t *testing.T) { olap, _ := to.AsOLAP("") tr := &duckDBToDuckDB{ - to: olap, + to: to.(*connection), logger: zap.NewNop(), } query := fmt.Sprintf("SELECT * FROM sqlite_scan('%s', 't');", dbPath) diff --git a/runtime/drivers/duckdb/transporter_sqlstore_to_duckDB.go b/runtime/drivers/duckdb/transporter_sqlstore_to_duckDB.go deleted file mode 100644 index 862a4d131cf..00000000000 --- a/runtime/drivers/duckdb/transporter_sqlstore_to_duckDB.go +++ /dev/null @@ -1,235 +0,0 @@ -package duckdb - -import ( - "context" - "database/sql" - "database/sql/driver" - "errors" - "fmt" - - "github.com/marcboeker/go-duckdb" - runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" - "github.com/rilldata/rill/runtime/drivers" - "go.uber.org/zap" -) - -type sqlStoreToDuckDB struct { - to drivers.OLAPStore - from drivers.SQLStore - logger *zap.Logger -} - -var _ drivers.Transporter = &sqlStoreToDuckDB{} - -func NewSQLStoreToDuckDB(from drivers.SQLStore, to drivers.OLAPStore, logger *zap.Logger) drivers.Transporter { - return &sqlStoreToDuckDB{ - to: to, - from: from, - logger: logger, - } -} - -func (s *sqlStoreToDuckDB) Transfer(ctx context.Context, srcProps, sinkProps map[string]any, opts *drivers.TransferOptions) (transferErr error) { - sinkCfg, err := parseSinkProperties(sinkProps) - if err != nil { - return err - } - - s.logger = s.logger.With(zap.String("source", sinkCfg.Table)) - - rowIter, err := s.from.Query(ctx, srcProps) - if err != nil { - return err - } - defer func() { - err := rowIter.Close() - if err != nil && !errors.Is(err, ctx.Err()) { - s.logger.Error("error in closing row iterator", zap.Error(err)) - } - }() - return s.transferFromRowIterator(ctx, rowIter, sinkCfg.Table) -} - -func (s *sqlStoreToDuckDB) transferFromRowIterator(ctx context.Context, iter drivers.RowIterator, table string) error { - schema, err := iter.Schema(ctx) - if err != nil { - if errors.Is(err, drivers.ErrIteratorDone) { - return drivers.ErrNoRows - } - return err - } - - if total, ok := iter.Size(drivers.ProgressUnitRecord); ok { - s.logger.Debug("records to be ingested", zap.Uint64("rows", total)) - } - // we first ingest data in a temporary table in the main db - // and then copy it to the final table to ensure that the final table is always created using CRUD APIs which takes care - // whether table goes in main db or in separate table specific db - tmpTable := fmt.Sprintf("__%s_tmp_sqlstore", table) - // generate create table query - qry, err := createTableQuery(schema, tmpTable) - if err != nil { - return err - } - - // create table - err = s.to.Exec(ctx, &drivers.Statement{Query: qry, Priority: 1, LongRunning: true}) - if err != nil { - return err - } - - defer func() { - // ensure temporary table is cleaned - err := s.to.Exec(context.Background(), &drivers.Statement{ - Query: fmt.Sprintf("DROP TABLE IF EXISTS %s", tmpTable), - Priority: 100, - LongRunning: true, - }) - if err != nil { - s.logger.Error("failed to drop temp table", zap.String("table", tmpTable), zap.Error(err)) - } - }() - - err = s.to.WithConnection(ctx, 1, true, false, func(ctx, ensuredCtx context.Context, conn *sql.Conn) error { - // append data using appender API - return rawConn(conn, func(conn driver.Conn) error { - a, err := duckdb.NewAppenderFromConn(conn, "", tmpTable) - if err != nil { - return err - } - defer func() { - err = a.Close() - if err != nil { - s.logger.Error("appender closed failed", zap.Error(err)) - } - }() - - for num := 0; ; num++ { - select { - case <-ctx.Done(): - return ctx.Err() - default: - if num == 10000 { - num = 0 - if err := a.Flush(); err != nil { - return err - } - } - - row, err := iter.Next(ctx) - if err != nil { - if errors.Is(err, drivers.ErrIteratorDone) { - return nil - } - return err - } - if err := convert(row, schema); err != nil { // duckdb specific datatype conversion - return err - } - - if err := a.AppendRow(row...); err != nil { - return err - } - } - } - }) - }) - if err != nil { - return err - } - - // copy data from temp table to target table - return s.to.CreateTableAsSelect(ctx, table, false, fmt.Sprintf("SELECT * FROM %s", tmpTable), nil) -} - -func createTableQuery(schema *runtimev1.StructType, name string) (string, error) { - query := fmt.Sprintf("CREATE OR REPLACE TABLE %s(", safeName(name)) - for i, s := range schema.Fields { - i++ - duckDBType, err := pbTypeToDuckDB(s.Type) - if err != nil { - return "", err - } - query += fmt.Sprintf("%s %s", safeName(s.Name), duckDBType) - if i != len(schema.Fields) { - query += "," - } - } - query += ")" - return query, nil -} - -func convert(row []driver.Value, schema *runtimev1.StructType) error { - for i, v := range row { - if v == nil { - continue - } - if schema.Fields[i].Type.Code == runtimev1.Type_CODE_UUID { - val, ok := v.([16]byte) - if !ok { - return fmt.Errorf("unknown type for UUID field %s: %T", schema.Fields[i].Name, v) - } - var uuid duckdb.UUID - copy(uuid[:], val[:]) - row[i] = uuid - } - } - return nil -} - -func pbTypeToDuckDB(t *runtimev1.Type) (string, error) { - code := t.Code - switch code { - case runtimev1.Type_CODE_UNSPECIFIED: - return "", fmt.Errorf("unspecified code") - case runtimev1.Type_CODE_BOOL: - return "BOOLEAN", nil - case runtimev1.Type_CODE_INT8: - return "TINYINT", nil - case runtimev1.Type_CODE_INT16: - return "SMALLINT", nil - case runtimev1.Type_CODE_INT32: - return "INTEGER", nil - case runtimev1.Type_CODE_INT64: - return "BIGINT", nil - case runtimev1.Type_CODE_INT128: - return "HUGEINT", nil - case runtimev1.Type_CODE_UINT8: - return "UTINYINT", nil - case runtimev1.Type_CODE_UINT16: - return "USMALLINT", nil - case runtimev1.Type_CODE_UINT32: - return "UINTEGER", nil - case runtimev1.Type_CODE_UINT64: - return "UBIGINT", nil - case runtimev1.Type_CODE_FLOAT32: - return "FLOAT", nil - case runtimev1.Type_CODE_FLOAT64: - return "DOUBLE", nil - case runtimev1.Type_CODE_TIMESTAMP: - return "TIMESTAMP", nil - case runtimev1.Type_CODE_DATE: - return "DATE", nil - case runtimev1.Type_CODE_TIME: - return "TIME", nil - case runtimev1.Type_CODE_STRING: - return "VARCHAR", nil - case runtimev1.Type_CODE_BYTES: - return "BLOB", nil - case runtimev1.Type_CODE_ARRAY: - return "", fmt.Errorf("array is not supported") - case runtimev1.Type_CODE_STRUCT: - return "", fmt.Errorf("struct is not supported") - case runtimev1.Type_CODE_MAP: - return "", fmt.Errorf("map is not supported") - case runtimev1.Type_CODE_DECIMAL: - return "DECIMAL", nil - case runtimev1.Type_CODE_JSON: - // keeping type as json but appending varchar using the appender API causes duckdb invalid vector error intermittently - return "VARCHAR", nil - case runtimev1.Type_CODE_UUID: - return "UUID", nil - default: - return "", fmt.Errorf("unknown type_code %s", code) - } -} diff --git a/runtime/drivers/duckdb/transporter_warehouse_to_duckDB.go b/runtime/drivers/duckdb/transporter_warehouse_to_duckDB.go index 24d9f70d518..a03e98f7a14 100644 --- a/runtime/drivers/duckdb/transporter_warehouse_to_duckDB.go +++ b/runtime/drivers/duckdb/transporter_warehouse_to_duckDB.go @@ -19,7 +19,7 @@ type warehouseToDuckDB struct { logger *zap.Logger } -var _ drivers.Transporter = &sqlStoreToDuckDB{} +var _ drivers.Transporter = &warehouseToDuckDB{} func NewWarehouseToDuckDB(from drivers.Warehouse, to drivers.OLAPStore, logger *zap.Logger) drivers.Transporter { return &warehouseToDuckDB{ @@ -75,10 +75,15 @@ func (w *warehouseToDuckDB) Transfer(ctx context.Context, srcProps, sinkProps ma } if create { - err = w.to.CreateTableAsSelect(ctx, sinkCfg.Table, false, fmt.Sprintf("SELECT * FROM %s", from), nil) + err = w.to.CreateTableAsSelect(ctx, sinkCfg.Table, fmt.Sprintf("SELECT * FROM %s", from), &drivers.CreateTableOptions{}) create = false } else { - err = w.to.InsertTableAsSelect(ctx, sinkCfg.Table, fmt.Sprintf("SELECT * FROM %s", from), false, true, drivers.IncrementalStrategyAppend, nil) + insertOpts := &drivers.InsertTableOptions{ + ByName: false, + InPlace: true, + Strategy: drivers.IncrementalStrategyAppend, + } + err = w.to.InsertTableAsSelect(ctx, sinkCfg.Table, fmt.Sprintf("SELECT * FROM %s", from), insertOpts) } if err != nil { return err diff --git a/runtime/drivers/duckdb/utils.go b/runtime/drivers/duckdb/utils.go index 7377e58faa0..480109ea28f 100644 --- a/runtime/drivers/duckdb/utils.go +++ b/runtime/drivers/duckdb/utils.go @@ -1,8 +1,6 @@ package duckdb import ( - "database/sql" - "database/sql/driver" "fmt" "os" "path/filepath" @@ -12,24 +10,6 @@ import ( "github.com/rilldata/rill/runtime/drivers" ) -// rawConn is similar to *sql.Conn.Raw, but additionally unwraps otelsql (which we use for instrumentation). -func rawConn(conn *sql.Conn, f func(driver.Conn) error) error { - return conn.Raw(func(raw any) error { - // For details, see: https://github.com/XSAM/otelsql/issues/98 - if c, ok := raw.(interface{ Raw() driver.Conn }); ok { - raw = c.Raw() - } - - // This is currently guaranteed, but adding check to be safe - driverConn, ok := raw.(driver.Conn) - if !ok { - return fmt.Errorf("internal: did not obtain a driver.Conn") - } - - return f(driverConn) - }) -} - type sinkProperties struct { Table string `mapstructure:"table"` } @@ -44,6 +24,7 @@ func parseSinkProperties(props map[string]any) (*sinkProperties, error) { type dbSourceProperties struct { Database string `mapstructure:"db"` + DSN string `mapstructure:"dsn"` SQL string `mapstructure:"sql"` } @@ -52,6 +33,9 @@ func parseDBSourceProperties(props map[string]any) (*dbSourceProperties, error) if err := mapstructure.Decode(props, cfg); err != nil { return nil, fmt.Errorf("failed to parse source properties: %w", err) } + if cfg.DSN != "" { // For mysql, postgres the property is called as dsn and not db + cfg.Database = cfg.DSN + } if cfg.SQL == "" { return nil, fmt.Errorf("property 'sql' is mandatory") } diff --git a/runtime/drivers/file/file.go b/runtime/drivers/file/file.go index c1c7c9a4ee6..3ee55720767 100644 --- a/runtime/drivers/file/file.go +++ b/runtime/drivers/file/file.go @@ -238,11 +238,6 @@ func (c *connection) AsWarehouse() (drivers.Warehouse, bool) { return nil, false } -// AsSQLStore implements drivers.Connection. -func (c *connection) AsSQLStore() (drivers.SQLStore, bool) { - return nil, false -} - // AsNotifier implements drivers.Connection. func (c *connection) AsNotifier(properties map[string]any) (drivers.Notifier, error) { return nil, drivers.ErrNotNotifier diff --git a/runtime/drivers/file/model_executor_olap_self.go b/runtime/drivers/file/model_executor_olap_self.go index 5b30aea8fed..bb4804b1624 100644 --- a/runtime/drivers/file/model_executor_olap_self.go +++ b/runtime/drivers/file/model_executor_olap_self.go @@ -268,7 +268,7 @@ func writeParquet(res *drivers.Result, fw io.Writer) error { arrowField.Type = arrow.PrimitiveTypes.Float64 case runtimev1.Type_CODE_TIMESTAMP, runtimev1.Type_CODE_TIME: arrowField.Type = arrow.FixedWidthTypes.Timestamp_us - case runtimev1.Type_CODE_STRING, runtimev1.Type_CODE_DATE, runtimev1.Type_CODE_ARRAY, runtimev1.Type_CODE_STRUCT, runtimev1.Type_CODE_MAP, runtimev1.Type_CODE_JSON, runtimev1.Type_CODE_UUID: + case runtimev1.Type_CODE_STRING, runtimev1.Type_CODE_INTERVAL, runtimev1.Type_CODE_DATE, runtimev1.Type_CODE_ARRAY, runtimev1.Type_CODE_STRUCT, runtimev1.Type_CODE_MAP, runtimev1.Type_CODE_JSON, runtimev1.Type_CODE_UUID: arrowField.Type = arrow.BinaryTypes.String case runtimev1.Type_CODE_BYTES: arrowField.Type = arrow.BinaryTypes.Binary @@ -334,7 +334,7 @@ func writeParquet(res *drivers.Result, fw io.Writer) error { return err } recordBuilder.Field(i).(*array.TimestampBuilder).Append(tmp) - case runtimev1.Type_CODE_STRING, runtimev1.Type_CODE_DATE, runtimev1.Type_CODE_ARRAY, runtimev1.Type_CODE_STRUCT, runtimev1.Type_CODE_MAP, runtimev1.Type_CODE_JSON, runtimev1.Type_CODE_UUID: + case runtimev1.Type_CODE_STRING, runtimev1.Type_CODE_INTERVAL, runtimev1.Type_CODE_DATE, runtimev1.Type_CODE_ARRAY, runtimev1.Type_CODE_STRUCT, runtimev1.Type_CODE_MAP, runtimev1.Type_CODE_JSON, runtimev1.Type_CODE_UUID: res, err := json.Marshal(v) if err != nil { return fmt.Errorf("failed to convert to JSON value: %w", err) diff --git a/runtime/drivers/gcs/gcs.go b/runtime/drivers/gcs/gcs.go index cf1e5cfd8ae..80ad89cd91b 100644 --- a/runtime/drivers/gcs/gcs.go +++ b/runtime/drivers/gcs/gcs.go @@ -265,11 +265,6 @@ func (c *Connection) AsWarehouse() (drivers.Warehouse, bool) { return nil, false } -// AsSQLStore implements drivers.Connection. -func (c *Connection) AsSQLStore() (drivers.SQLStore, bool) { - return nil, false -} - // AsNotifier implements drivers.Connection. func (c *Connection) AsNotifier(properties map[string]any) (drivers.Notifier, error) { return nil, drivers.ErrNotNotifier diff --git a/runtime/drivers/https/https.go b/runtime/drivers/https/https.go index 24775d1de57..f6bfac06442 100644 --- a/runtime/drivers/https/https.go +++ b/runtime/drivers/https/https.go @@ -187,11 +187,6 @@ func (c *connection) AsWarehouse() (drivers.Warehouse, bool) { return nil, false } -// AsSQLStore implements drivers.Connection. -func (c *connection) AsSQLStore() (drivers.SQLStore, bool) { - return nil, false -} - // AsNotifier implements drivers.Connection. func (c *connection) AsNotifier(properties map[string]any) (drivers.Notifier, error) { return nil, drivers.ErrNotNotifier diff --git a/runtime/drivers/mock/object_store/object_store.go b/runtime/drivers/mock/object_store/object_store.go index 2a4ca00c91d..d3482473c94 100644 --- a/runtime/drivers/mock/object_store/object_store.go +++ b/runtime/drivers/mock/object_store/object_store.go @@ -161,11 +161,6 @@ func (h *handle) AsFileStore() (drivers.FileStore, bool) { return nil, false } -// AsSQLStore implements drivers.Connection. -func (h *handle) AsSQLStore() (drivers.SQLStore, bool) { - return nil, false -} - // AsWarehouse implements drivers.Handle. func (h *handle) AsWarehouse() (drivers.Warehouse, bool) { return nil, false diff --git a/runtime/drivers/mysql/mysql.go b/runtime/drivers/mysql/mysql.go index 5dc2a0a627b..1e0a6d22947 100644 --- a/runtime/drivers/mysql/mysql.go +++ b/runtime/drivers/mysql/mysql.go @@ -174,11 +174,6 @@ func (c *connection) AsWarehouse() (drivers.Warehouse, bool) { return nil, false } -// AsSQLStore implements drivers.Connection. -func (c *connection) AsSQLStore() (drivers.SQLStore, bool) { - return c, true -} - // AsNotifier implements drivers.Connection. func (c *connection) AsNotifier(properties map[string]any) (drivers.Notifier, error) { return nil, drivers.ErrNotNotifier diff --git a/runtime/drivers/mysql/parser.go b/runtime/drivers/mysql/parser.go deleted file mode 100644 index 32922c4a7f8..00000000000 --- a/runtime/drivers/mysql/parser.go +++ /dev/null @@ -1,314 +0,0 @@ -package mysql - -import ( - "database/sql" - "fmt" - "reflect" - "strings" - - runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" -) - -type mapper interface { - runtimeType(st reflect.Type) (*runtimev1.Type, error) - // dest returns a pointer to a destination value that can be used in Rows.Scan - dest(st reflect.Type) (any, error) - // value dereferences a pointer created by dest - value(p any) (any, error) -} - -// refer https://github.com/go-sql-driver/mysql/blob/master/fields.go for base types -func getDBTypeNameToMapperMap() map[string]mapper { - m := make(map[string]mapper) - - bit := bitMapper{} - numeric := numericMapper{} - char := charMapper{} - bytes := byteMapper{} - date := dateMapper{} - json := jsonMapper{} - - // bit - m["BIT"] = bit - - // numeric - m["TINYINT"] = numeric - m["SMALLINT"] = numeric - m["MEDIUMINT"] = numeric - m["INT"] = numeric - m["UNSIGNED TINYINT"] = numeric - m["UNSIGNED SMALLINT"] = numeric - m["UNSIGNED INT"] = numeric - m["UNSIGNED BIGINT"] = numeric - m["BIGINT"] = numeric - m["DOUBLE"] = numeric - m["FLOAT"] = numeric - // MySQL stores DECIMAL value in binary format - // It might be stored as string without losing precision - m["DECIMAL"] = char - - // string - m["CHAR"] = char - m["LONGTEXT"] = char - m["MEDIUMTEXT"] = char - m["TEXT"] = char - m["TINYTEXT"] = char - m["VARCHAR"] = char - - // binary - m["BINARY"] = bytes - m["TINYBLOB"] = bytes - m["BLOB"] = bytes - m["LONGBLOB"] = bytes - m["MEDIUMBLOB"] = bytes - m["VARBINARY"] = bytes - - // date and time - m["DATE"] = date - m["DATETIME"] = date - m["TIMESTAMP"] = date - m["YEAR"] = numeric - // TIME is scanned as bytes and can be converted to string - m["TIME"] = char - - // json - m["JSON"] = json - - return m -} - -var ( - scanTypeFloat32 = reflect.TypeOf(float32(0)) - scanTypeFloat64 = reflect.TypeOf(float64(0)) - scanTypeInt8 = reflect.TypeOf(int8(0)) - scanTypeInt16 = reflect.TypeOf(int16(0)) - scanTypeInt32 = reflect.TypeOf(int32(0)) - scanTypeInt64 = reflect.TypeOf(int64(0)) - scanTypeNullFloat = reflect.TypeOf(sql.NullFloat64{}) - scanTypeNullInt = reflect.TypeOf(sql.NullInt64{}) - scanTypeUint8 = reflect.TypeOf(uint8(0)) - scanTypeUint16 = reflect.TypeOf(uint16(0)) - scanTypeUint32 = reflect.TypeOf(uint32(0)) - scanTypeUint64 = reflect.TypeOf(uint64(0)) -) - -type numericMapper struct{} - -func (m numericMapper) runtimeType(st reflect.Type) (*runtimev1.Type, error) { - switch st { - case scanTypeInt8: - return &runtimev1.Type{Code: runtimev1.Type_CODE_INT8}, nil - case scanTypeInt16: - return &runtimev1.Type{Code: runtimev1.Type_CODE_INT16}, nil - case scanTypeInt32: - return &runtimev1.Type{Code: runtimev1.Type_CODE_INT32}, nil - case scanTypeInt64: - return &runtimev1.Type{Code: runtimev1.Type_CODE_INT64}, nil - case scanTypeUint8: - return &runtimev1.Type{Code: runtimev1.Type_CODE_UINT8}, nil - case scanTypeUint16: - return &runtimev1.Type{Code: runtimev1.Type_CODE_UINT16}, nil - case scanTypeUint32: - return &runtimev1.Type{Code: runtimev1.Type_CODE_UINT32}, nil - case scanTypeUint64: - return &runtimev1.Type{Code: runtimev1.Type_CODE_UINT64}, nil - case scanTypeNullInt: - return &runtimev1.Type{Code: runtimev1.Type_CODE_INT64}, nil - case scanTypeFloat32: - return &runtimev1.Type{Code: runtimev1.Type_CODE_FLOAT32}, nil - case scanTypeFloat64: - return &runtimev1.Type{Code: runtimev1.Type_CODE_FLOAT64}, nil - case scanTypeNullFloat: - return &runtimev1.Type{Code: runtimev1.Type_CODE_FLOAT64}, nil - default: - return nil, fmt.Errorf("numericMapper: unsupported scan type %v", st.Name()) - } -} - -func (m numericMapper) dest(st reflect.Type) (any, error) { - switch st { - case scanTypeInt8: - return new(int8), nil - case scanTypeInt16: - return new(int16), nil - case scanTypeInt32: - return new(int32), nil - case scanTypeInt64: - return new(int64), nil - case scanTypeUint8: - return new(uint8), nil - case scanTypeUint16: - return new(uint16), nil - case scanTypeUint32: - return new(uint32), nil - case scanTypeUint64: - return new(uint64), nil - case scanTypeNullInt: - return new(sql.NullInt64), nil - case scanTypeFloat32: - return new(float32), nil - case scanTypeFloat64: - return new(float64), nil - case scanTypeNullFloat: - return new(sql.NullFloat64), nil - default: - return nil, fmt.Errorf("numericMapper: unsupported scan type %v", st.Name()) - } -} - -func (m numericMapper) value(p any) (any, error) { - switch v := p.(type) { - case *int8: - return *v, nil - case *int16: - return *v, nil - case *int32: - return *v, nil - case *int64: - return *v, nil - case *uint8: - return *v, nil - case *uint16: - return *v, nil - case *uint32: - return *v, nil - case *uint64: - return *v, nil - case *sql.NullInt64: - vl, err := v.Value() - if err != nil { - return nil, err - } - return vl, nil - case *float32: - return *v, nil - case *float64: - return *v, nil - case *sql.NullFloat64: - vl, err := v.Value() - if err != nil { - return nil, err - } - return vl, nil - default: - return nil, fmt.Errorf("numericMapper: unsupported value type %v", p) - } -} - -type bitMapper struct{} - -func (m bitMapper) runtimeType(reflect.Type) (*runtimev1.Type, error) { - return &runtimev1.Type{Code: runtimev1.Type_CODE_STRING}, nil -} - -func (m bitMapper) dest(reflect.Type) (any, error) { - return &[]byte{}, nil -} - -func (m bitMapper) value(p any) (any, error) { - switch bs := p.(type) { - case *[]byte: - if *bs == nil { - return nil, nil - } - str := strings.Builder{} - for _, b := range *bs { - str.WriteString(fmt.Sprintf("%08b ", b)) - } - s := str.String()[:len(*bs)] - return s, nil - default: - return nil, fmt.Errorf("bitMapper: unsupported value type %v", bs) - } -} - -type charMapper struct{} - -func (m charMapper) runtimeType(reflect.Type) (*runtimev1.Type, error) { - return &runtimev1.Type{Code: runtimev1.Type_CODE_STRING}, nil -} - -func (m charMapper) dest(reflect.Type) (any, error) { - return new(sql.NullString), nil -} - -func (m charMapper) value(p any) (any, error) { - switch v := p.(type) { - case *sql.NullString: - vl, err := v.Value() - if err != nil { - return nil, err - } - return vl, nil - default: - return nil, fmt.Errorf("charMapper: unsupported value type %v", v) - } -} - -type byteMapper struct{} - -func (m byteMapper) runtimeType(reflect.Type) (*runtimev1.Type, error) { - return &runtimev1.Type{Code: runtimev1.Type_CODE_BYTES}, nil -} - -func (m byteMapper) dest(reflect.Type) (any, error) { - return &[]byte{}, nil -} - -func (m byteMapper) value(p any) (any, error) { - switch v := p.(type) { - case *[]byte: - if *v == nil { - return nil, nil - } - return *v, nil - default: - return nil, fmt.Errorf("byteMapper: unsupported value type %v", v) - } -} - -type dateMapper struct{} - -func (m dateMapper) runtimeType(reflect.Type) (*runtimev1.Type, error) { - return &runtimev1.Type{Code: runtimev1.Type_CODE_TIMESTAMP}, nil -} - -func (m dateMapper) dest(reflect.Type) (any, error) { - return new(sql.NullTime), nil -} - -func (m dateMapper) value(p any) (any, error) { - switch v := p.(type) { - case *sql.NullTime: - vl, err := v.Value() - if err != nil { - return nil, err - } - return vl, nil - default: - return nil, fmt.Errorf("dateMapper: unsupported value type %v", v) - } -} - -type jsonMapper struct{} - -func (m jsonMapper) runtimeType(reflect.Type) (*runtimev1.Type, error) { - return &runtimev1.Type{Code: runtimev1.Type_CODE_JSON}, nil -} - -func (m jsonMapper) dest(reflect.Type) (any, error) { - return new(sql.NullString), nil -} - -func (m jsonMapper) value(p any) (any, error) { - switch v := p.(type) { - case *sql.NullString: - vl, err := v.Value() - if err != nil { - return nil, err - } - return vl, nil - default: - return nil, fmt.Errorf("jsonMapper: unsupported value type %v", v) - } -} diff --git a/runtime/drivers/mysql/sql_store.go b/runtime/drivers/mysql/sql_store.go deleted file mode 100644 index 35d6d0ef9a0..00000000000 --- a/runtime/drivers/mysql/sql_store.go +++ /dev/null @@ -1,186 +0,0 @@ -package mysql - -import ( - "context" - "database/sql" - sqldriver "database/sql/driver" - "errors" - "fmt" - - "github.com/go-sql-driver/mysql" - "github.com/mitchellh/mapstructure" - runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" - "github.com/rilldata/rill/runtime/drivers" -) - -// Query implements drivers.SQLStore -func (c *connection) Query(ctx context.Context, props map[string]any) (drivers.RowIterator, error) { - srcProps, err := parseSourceProperties(props) - if err != nil { - return nil, err - } - - var dsn string - if srcProps.DSN != "" { // get from src properties - dsn = srcProps.DSN - } else if url, ok := c.config["dsn"].(string); ok && url != "" { // get from driver configs - dsn = url - } else { - return nil, fmt.Errorf("the property 'dsn' is required for MySQL. Provide 'dsn' in the YAML properties or pass '--env connector.mysql.dsn=...' to 'rill start'") - } - - conf, err := mysql.ParseDSN(dsn) - if err != nil { - return nil, err - } - conf.ParseTime = true // if set to false, time is scanned as an array rather than as time.Time - - db, err := sql.Open("mysql", conf.FormatDSN()) - if err != nil { - return nil, err - } - - // Validate DSN data: - err = db.Ping() - if err != nil { - db.Close() - return nil, err - } - - rows, err := db.QueryContext(ctx, srcProps.SQL) - if err != nil { - return nil, err - } - - iter := &rowIterator{ - db: db, - rows: rows, - } - - if err := iter.setSchema(); err != nil { - iter.Close() - return nil, err - } - return iter, nil -} - -type rowIterator struct { - db *sql.DB - rows *sql.Rows - - schema *runtimev1.StructType - row []sqldriver.Value - fieldMappers []mapper - fieldDests []any // Destinations are used while scanning rows - columnTypes []*sql.ColumnType -} - -// Close implements drivers.RowIterator. -func (r *rowIterator) Close() error { - r.rows.Close() - r.db.Close() - return nil -} - -// Next implements drivers.RowIterator. -func (r *rowIterator) Next(ctx context.Context) ([]sqldriver.Value, error) { - var err error - if !r.rows.Next() { - err := r.rows.Err() - if err == nil { - return nil, drivers.ErrIteratorDone - } - if errors.Is(err, sql.ErrNoRows) { - return nil, drivers.ErrNoRows - } - return nil, err - } - - // Scan expects destinations to be pointers - for i := range r.fieldDests { - r.fieldDests[i], err = r.fieldMappers[i].dest(r.columnTypes[i].ScanType()) - if err != nil { - return nil, err - } - } - - if err := r.rows.Scan(r.fieldDests...); err != nil { - return nil, err - } - - for i := range r.schema.Fields { - // Dereference destinations and fill the row - r.row[i], err = r.fieldMappers[i].value(r.fieldDests[i]) - if err != nil { - return nil, err - } - } - return r.row, nil -} - -// Schema implements drivers.RowIterator. -func (r *rowIterator) Schema(ctx context.Context) (*runtimev1.StructType, error) { - return r.schema, nil -} - -// Size implements drivers.RowIterator. -func (r *rowIterator) Size(unit drivers.ProgressUnit) (uint64, bool) { - return 0, false -} - -var _ drivers.RowIterator = &rowIterator{} - -func (r *rowIterator) setSchema() error { - cts, err := r.rows.ColumnTypes() - if err != nil { - return err - } - - mappers := make([]mapper, len(cts)) - fields := make([]*runtimev1.StructType_Field, len(cts)) - dbTypeNameToMapperMap := getDBTypeNameToMapperMap() - - for i, ct := range cts { - mapper, ok := dbTypeNameToMapperMap[ct.DatabaseTypeName()] - if !ok { - return fmt.Errorf("datatype %q is not supported", ct.DatabaseTypeName()) - } - mappers[i] = mapper - runtimeType, err := mapper.runtimeType(ct.ScanType()) - if err != nil { - return err - } - fields[i] = &runtimev1.StructType_Field{ - Name: ct.Name(), - Type: runtimeType, - } - } - - r.schema = &runtimev1.StructType{Fields: fields} - r.row = make([]sqldriver.Value, len(r.schema.Fields)) - r.fieldMappers = mappers - r.fieldDests = make([]any, len(r.schema.Fields)) - r.columnTypes, err = r.rows.ColumnTypes() - if err != nil { - return err - } - - return nil -} - -type sourceProperties struct { - SQL string `mapstructure:"sql"` - DSN string `mapstructure:"dsn"` -} - -func parseSourceProperties(props map[string]any) (*sourceProperties, error) { - conf := &sourceProperties{} - err := mapstructure.Decode(props, conf) - if err != nil { - return nil, err - } - if conf.SQL == "" { - return nil, fmt.Errorf("property 'sql' is mandatory for connector \"mysql\"") - } - return conf, err -} diff --git a/runtime/drivers/olap.go b/runtime/drivers/olap.go index 0bfa53ded69..b6d78ef8019 100644 --- a/runtime/drivers/olap.go +++ b/runtime/drivers/olap.go @@ -25,19 +25,35 @@ var ErrUnsupportedConnector = errors.New("drivers: connector not supported") // and ensuredCtx wraps a background context (ensuring it can never be cancelled). type WithConnectionFunc func(wrappedCtx context.Context, ensuredCtx context.Context, conn *sql.Conn) error +type CreateTableOptions struct { + View bool + BeforeCreate string + AfterCreate string + TableOpts map[string]any +} + +type InsertTableOptions struct { + BeforeInsert string + AfterInsert string + ByName bool + InPlace bool + Strategy IncrementalStrategy + UniqueKey []string +} + // OLAPStore is implemented by drivers that are capable of storing, transforming and serving analytical queries. // NOTE crud APIs are not safe to be called with `WithConnection` type OLAPStore interface { Dialect() Dialect - WithConnection(ctx context.Context, priority int, longRunning, tx bool, fn WithConnectionFunc) error + WithConnection(ctx context.Context, priority int, longRunning bool, fn WithConnectionFunc) error Exec(ctx context.Context, stmt *Statement) error Execute(ctx context.Context, stmt *Statement) (*Result, error) InformationSchema() InformationSchema - CreateTableAsSelect(ctx context.Context, name string, view bool, sql string, tableOpts map[string]any) error - InsertTableAsSelect(ctx context.Context, name, sql string, byName, inPlace bool, strategy IncrementalStrategy, uniqueKey []string) error - DropTable(ctx context.Context, name string, view bool) error - RenameTable(ctx context.Context, name, newName string, view bool) error + CreateTableAsSelect(ctx context.Context, name, sql string, opts *CreateTableOptions) error + InsertTableAsSelect(ctx context.Context, name, sql string, opts *InsertTableOptions) error + DropTable(ctx context.Context, name string) error + RenameTable(ctx context.Context, name, newName string) error AddTableColumn(ctx context.Context, tableName, columnName string, typ string) error AlterTableColumn(ctx context.Context, tableName, columnName string, newType string) error @@ -160,9 +176,10 @@ type IngestionSummary struct { type IncrementalStrategy string const ( - IncrementalStrategyUnspecified IncrementalStrategy = "" - IncrementalStrategyAppend IncrementalStrategy = "append" - IncrementalStrategyMerge IncrementalStrategy = "merge" + IncrementalStrategyUnspecified IncrementalStrategy = "" + IncrementalStrategyAppend IncrementalStrategy = "append" + IncrementalStrategyMerge IncrementalStrategy = "merge" + IncrementalStrategyPartitionOverwrite IncrementalStrategy = "partition_overwrite" ) // Dialect enumerates OLAP query languages. @@ -249,6 +266,9 @@ func (d Dialect) RequiresCastForLike() bool { // EscapeTable returns an esacped fully qualified table name func (d Dialect) EscapeTable(db, schema, table string) string { + if d == DialectDuckDB { + return d.EscapeIdentifier(table) + } var sb strings.Builder if db != "" { sb.WriteString(d.EscapeIdentifier(db)) diff --git a/runtime/drivers/pinot/olap.go b/runtime/drivers/pinot/olap.go index 4c56cb1a40b..78464e4eb6f 100644 --- a/runtime/drivers/pinot/olap.go +++ b/runtime/drivers/pinot/olap.go @@ -27,22 +27,22 @@ func (c *connection) AlterTableColumn(ctx context.Context, tableName, columnName } // CreateTableAsSelect implements drivers.OLAPStore. -func (c *connection) CreateTableAsSelect(ctx context.Context, name string, view bool, sql string, tableOpts map[string]any) error { +func (c *connection) CreateTableAsSelect(ctx context.Context, name, sql string, opts *drivers.CreateTableOptions) error { return fmt.Errorf("pinot: data transformation not yet supported") } // DropTable implements drivers.OLAPStore. -func (c *connection) DropTable(ctx context.Context, name string, view bool) error { +func (c *connection) DropTable(ctx context.Context, name string) error { return fmt.Errorf("pinot: data transformation not yet supported") } // InsertTableAsSelect implements drivers.OLAPStore. -func (c *connection) InsertTableAsSelect(ctx context.Context, name, sql string, byName, inPlace bool, strategy drivers.IncrementalStrategy, uniqueKey []string) error { +func (c *connection) InsertTableAsSelect(ctx context.Context, name, sql string, opts *drivers.InsertTableOptions) error { return fmt.Errorf("pinot: data transformation not yet supported") } // RenameTable implements drivers.OLAPStore. -func (c *connection) RenameTable(ctx context.Context, name, newName string, view bool) error { +func (c *connection) RenameTable(ctx context.Context, name, newName string) error { return fmt.Errorf("pinot: data transformation not yet supported") } @@ -50,7 +50,7 @@ func (c *connection) Dialect() drivers.Dialect { return drivers.DialectPinot } -func (c *connection) WithConnection(ctx context.Context, priority int, longRunning, tx bool, fn drivers.WithConnectionFunc) error { +func (c *connection) WithConnection(ctx context.Context, priority int, longRunning bool, fn drivers.WithConnectionFunc) error { return fmt.Errorf("pinot: WithConnection not supported") } diff --git a/runtime/drivers/pinot/pinot.go b/runtime/drivers/pinot/pinot.go index ca83ec1ff26..687d0e8b034 100644 --- a/runtime/drivers/pinot/pinot.go +++ b/runtime/drivers/pinot/pinot.go @@ -265,10 +265,6 @@ func (c *connection) AsWarehouse() (drivers.Warehouse, bool) { return nil, false } -func (c *connection) AsSQLStore() (drivers.SQLStore, bool) { - return nil, false -} - // AsNotifier implements drivers.Connection. func (c *connection) AsNotifier(properties map[string]any) (drivers.Notifier, error) { return nil, drivers.ErrNotNotifier diff --git a/runtime/drivers/postgres/parser.go b/runtime/drivers/postgres/parser.go deleted file mode 100644 index c3dbe76724b..00000000000 --- a/runtime/drivers/postgres/parser.go +++ /dev/null @@ -1,339 +0,0 @@ -package postgres - -import ( - "encoding/json" - "fmt" - "strings" - "time" - - "github.com/jackc/pgx/v5/pgtype" - runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" -) - -type mapper interface { - runtimeType() *runtimev1.Type - value(pgxVal any) (any, error) -} - -func register(oidToMapperMap map[string]mapper, typ string, m mapper) { - oidToMapperMap[typ] = m - // array of base type - oidToMapperMap[fmt.Sprintf("_%s", typ)] = &arrayMapper{baseMapper: m} -} - -// refer https://github.com/jackc/pgx/blob/master/pgtype/pgtype_default.go for base types -func getOidToMapperMap() map[string]mapper { - m := make(map[string]mapper) - register(m, "bit", &bitMapper{}) - register(m, "bool", &boolMapper{}) - register(m, "bpchar", &charMapper{}) - register(m, "bytea", &byteMapper{}) - register(m, "char", &charMapper{}) - register(m, "date", &dateMapper{}) - register(m, "float4", &float32Mapper{}) - register(m, "float8", &float64Mapper{}) - register(m, "int2", &int16Mapper{}) - register(m, "int4", &int32Mapper{}) - register(m, "int8", &int64Mapper{}) - register(m, "numeric", &numericMapper{}) - register(m, "text", &charMapper{}) - register(m, "time", &timeMapper{}) - register(m, "timestamp", &timeStampMapper{}) - register(m, "timestamptz", &timeStampMapper{}) - register(m, "uuid", &uuidMapper{}) - register(m, "varbit", &bitMapper{}) - register(m, "varchar", &charMapper{}) - register(m, "json", &jsonMapper{}) - register(m, "jsonb", &jsonMapper{}) - return m -} - -type bitMapper struct{} - -func (m *bitMapper) runtimeType() *runtimev1.Type { - // use bitstring once appender supports it - return &runtimev1.Type{Code: runtimev1.Type_CODE_STRING} -} - -func (m *bitMapper) value(pgxVal any) (any, error) { - switch b := pgxVal.(type) { - case pgtype.Bits: - str := strings.Builder{} - for _, n := range b.Bytes { - str.WriteString(fmt.Sprintf("%08b ", n)) - } - return str.String()[:b.Len], nil - default: - return nil, fmt.Errorf("bitMapper: unsupported type %v", b) - } -} - -type boolMapper struct{} - -func (m *boolMapper) runtimeType() *runtimev1.Type { - return &runtimev1.Type{Code: runtimev1.Type_CODE_BOOL} -} - -func (m *boolMapper) value(pgxVal any) (any, error) { - switch b := pgxVal.(type) { - case bool: - return b, nil - default: - return nil, fmt.Errorf("boolMapper: unsupported type %v", b) - } -} - -type charMapper struct{} - -func (m *charMapper) runtimeType() *runtimev1.Type { - return &runtimev1.Type{Code: runtimev1.Type_CODE_STRING} -} - -func (m *charMapper) value(pgxVal any) (any, error) { - switch b := pgxVal.(type) { - case string: - return b, nil - default: - return nil, fmt.Errorf("charMapper: unsupported type %v", b) - } -} - -type byteMapper struct{} - -func (m *byteMapper) runtimeType() *runtimev1.Type { - return &runtimev1.Type{Code: runtimev1.Type_CODE_BYTES} -} - -func (m *byteMapper) value(pgxVal any) (any, error) { - switch b := pgxVal.(type) { - case []byte: - return b, nil - default: - return nil, fmt.Errorf("byteMapper: unsupported type %v", b) - } -} - -type dateMapper struct{} - -func (m *dateMapper) runtimeType() *runtimev1.Type { - // Use runtimev1.Type_CODE_DATE once DATE is supported by DuckDB appender - return &runtimev1.Type{Code: runtimev1.Type_CODE_TIMESTAMP} -} - -func (m *dateMapper) value(pgxVal any) (any, error) { - switch b := pgxVal.(type) { - case time.Time: - return b, nil - default: - return nil, fmt.Errorf("dateMapper: unsupported type %v", b) - } -} - -type float32Mapper struct{} - -func (m *float32Mapper) runtimeType() *runtimev1.Type { - return &runtimev1.Type{Code: runtimev1.Type_CODE_FLOAT32} -} - -func (m *float32Mapper) value(pgxVal any) (any, error) { - switch b := pgxVal.(type) { - case float32: - return b, nil - default: - return nil, fmt.Errorf("float32Mapper: unsupported type %v", b) - } -} - -type float64Mapper struct{} - -func (m *float64Mapper) runtimeType() *runtimev1.Type { - return &runtimev1.Type{Code: runtimev1.Type_CODE_FLOAT64} -} - -func (m *float64Mapper) value(pgxVal any) (any, error) { - switch b := pgxVal.(type) { - case float64: - return b, nil - default: - return nil, fmt.Errorf("float64Mapper: unsupported type %v", b) - } -} - -type int16Mapper struct{} - -func (m *int16Mapper) runtimeType() *runtimev1.Type { - return &runtimev1.Type{Code: runtimev1.Type_CODE_INT16} -} - -func (m *int16Mapper) value(pgxVal any) (any, error) { - switch b := pgxVal.(type) { - case int16: - return b, nil - default: - return nil, fmt.Errorf("int16Mapper: unsupported type %v", b) - } -} - -type int32Mapper struct{} - -func (m *int32Mapper) runtimeType() *runtimev1.Type { - return &runtimev1.Type{Code: runtimev1.Type_CODE_INT32} -} - -func (m *int32Mapper) value(pgxVal any) (any, error) { - switch b := pgxVal.(type) { - case int32: - return b, nil - default: - return nil, fmt.Errorf("int32Mapper: unsupported type %v", b) - } -} - -type int64Mapper struct{} - -func (m *int64Mapper) runtimeType() *runtimev1.Type { - return &runtimev1.Type{Code: runtimev1.Type_CODE_INT64} -} - -func (m *int64Mapper) value(pgxVal any) (any, error) { - switch b := pgxVal.(type) { - case int64: - return b, nil - default: - return nil, fmt.Errorf("int64Mapper: unsupported type %v", b) - } -} - -type timeMapper struct{} - -func (m *timeMapper) runtimeType() *runtimev1.Type { - // Use runtimev1.Type_CODE_TIME once DATE is supported by DuckDB appender - return &runtimev1.Type{Code: runtimev1.Type_CODE_TIMESTAMP} -} - -func (m *timeMapper) value(pgxVal any) (any, error) { - switch b := pgxVal.(type) { - case pgtype.Time: - midnight := time.Date(time.Now().Year(), time.Now().Month(), time.Now().Day(), 0, 0, 0, 0, time.UTC) - duration := time.Duration(b.Microseconds) * time.Microsecond - midnight = midnight.Add(duration) - return midnight, nil - default: - return nil, fmt.Errorf("timeMapper: unsupported type %v", b) - } -} - -type timeStampMapper struct{} - -func (m *timeStampMapper) runtimeType() *runtimev1.Type { - return &runtimev1.Type{Code: runtimev1.Type_CODE_TIMESTAMP} -} - -func (m *timeStampMapper) value(pgxVal any) (any, error) { - switch b := pgxVal.(type) { - case time.Time: - return b, nil - default: - return nil, fmt.Errorf("timeStampMapper: unsupported type %v", b) - } -} - -type uuidMapper struct{} - -func (m *uuidMapper) runtimeType() *runtimev1.Type { - return &runtimev1.Type{Code: runtimev1.Type_CODE_UUID} -} - -func (m *uuidMapper) value(pgxVal any) (any, error) { - switch b := pgxVal.(type) { - case [16]byte: - return b, nil - default: - return nil, fmt.Errorf("uuidMapper: unsupported type %v", b) - } -} - -type numericMapper struct{} - -func (m *numericMapper) runtimeType() *runtimev1.Type { - return &runtimev1.Type{Code: runtimev1.Type_CODE_STRING} -} - -func (m *numericMapper) value(pgxVal any) (any, error) { - switch b := pgxVal.(type) { - case pgtype.NumericValuer: - f, err := b.NumericValue() - if err != nil { - return nil, err - } - bytes, err := f.MarshalJSON() - if err != nil { - return nil, err - } - return string(bytes), nil - case pgtype.Float64Valuer: - f, err := b.Float64Value() - if err != nil { - return nil, err - } - return fmt.Sprint(f.Float64), nil - case pgtype.Int64Valuer: - f, err := b.Int64Value() - if err != nil { - return nil, err - } - return fmt.Sprint(f.Int64), nil - default: - return nil, fmt.Errorf("numericMapper: unsupported type %v", b) - } -} - -type jsonMapper struct{} - -func (m *jsonMapper) runtimeType() *runtimev1.Type { - return &runtimev1.Type{Code: runtimev1.Type_CODE_JSON} -} - -func (m *jsonMapper) value(pgxVal any) (any, error) { - switch b := pgxVal.(type) { - case []byte: - return string(b), nil - case map[string]any: - enc, err := json.Marshal(b) - if err != nil { - return nil, err - } - return string(enc), nil - default: - return nil, fmt.Errorf("jsonMapper: unsupported type %v", b) - } -} - -type arrayMapper struct { - baseMapper mapper -} - -func (m *arrayMapper) runtimeType() *runtimev1.Type { - return &runtimev1.Type{Code: runtimev1.Type_CODE_JSON} -} - -func (m *arrayMapper) value(pgxVal any) (any, error) { - switch b := pgxVal.(type) { - case []interface{}: - arr := make([]any, len(b)) - for i, val := range b { - res, err := m.baseMapper.value(val) - if err != nil { - return nil, err - } - arr[i] = res - } - enc, err := json.Marshal(arr) - if err != nil { - return nil, err - } - return string(enc), nil - default: - return nil, fmt.Errorf("arrayMapper: unsupported type %v", b) - } -} diff --git a/runtime/drivers/postgres/postgres.go b/runtime/drivers/postgres/postgres.go index 2dc83d613ee..a34ce1531ba 100644 --- a/runtime/drivers/postgres/postgres.go +++ b/runtime/drivers/postgres/postgres.go @@ -172,11 +172,6 @@ func (c *connection) AsWarehouse() (drivers.Warehouse, bool) { return nil, false } -// AsSQLStore implements drivers.Connection. -func (c *connection) AsSQLStore() (drivers.SQLStore, bool) { - return c, true -} - // AsNotifier implements drivers.Connection. func (c *connection) AsNotifier(properties map[string]any) (drivers.Notifier, error) { return nil, drivers.ErrNotNotifier diff --git a/runtime/drivers/postgres/sql_store.go b/runtime/drivers/postgres/sql_store.go deleted file mode 100644 index d0412df8906..00000000000 --- a/runtime/drivers/postgres/sql_store.go +++ /dev/null @@ -1,253 +0,0 @@ -package postgres - -import ( - "context" - "database/sql" - sqldriver "database/sql/driver" - "errors" - "fmt" - "strings" - "time" - - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgtype" - "github.com/jackc/pgx/v5/pgxpool" - "github.com/mitchellh/mapstructure" - runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" - "github.com/rilldata/rill/runtime/drivers" -) - -// Query implements drivers.SQLStore -func (c *connection) Query(ctx context.Context, props map[string]any) (drivers.RowIterator, error) { - srcProps, err := parseSourceProperties(props) - if err != nil { - return nil, err - } - - var dsn string - if srcProps.DatabaseURL != "" { // get from src properties - dsn = srcProps.DatabaseURL - } else if url, ok := c.config["database_url"].(string); ok && url != "" { // get from driver configs - dsn = url - } else { - return nil, fmt.Errorf("the property 'database_url' is required for Postgres. Provide 'database_url' in the YAML properties or pass '--env connector.postgres.database_url=...' to 'rill start'") - } - - config, err := pgxpool.ParseConfig(dsn) - if err != nil { - return nil, err - } - // disable prepared statements which is not supported by some postgres providers like pgedge cloud for non admin users. - // prepared statements are also not supported by proxies like pgbouncer. - // The limiatation of not using prepared statements is not a problem for us as we don't support parameters in source queries. - config.ConnConfig.DefaultQueryExecMode = pgx.QueryExecModeSimpleProtocol - - pool, err := pgxpool.NewWithConfig(ctx, config) - if err != nil { - return nil, err - } - - conn, err := pool.Acquire(ctx) - if err != nil { - pool.Close() - return nil, err - } - - res, err := conn.Query(ctx, srcProps.SQL) - if err != nil { - conn.Release() - pool.Close() - return nil, err - } - - iter := &rowIterator{ - conn: conn, - rows: res, - pool: pool, - } - - if err := iter.setSchema(ctx); err != nil { - iter.Close() - return nil, err - } - return iter, nil -} - -type rowIterator struct { - conn *pgxpool.Conn - rows pgx.Rows - pool *pgxpool.Pool - schema *runtimev1.StructType - - row []sqldriver.Value - fieldMappers []mapper -} - -// Close implements drivers.RowIterator. -func (r *rowIterator) Close() error { - r.rows.Close() - r.conn.Release() - r.pool.Close() - return r.rows.Err() -} - -// Next implements drivers.RowIterator. -func (r *rowIterator) Next(ctx context.Context) ([]sqldriver.Value, error) { - if !r.rows.Next() { - err := r.rows.Err() - if err == nil { - return nil, drivers.ErrIteratorDone - } - if errors.Is(err, sql.ErrNoRows) { - return nil, drivers.ErrNoRows - } - return nil, err - } - - vals, err := r.rows.Values() - if err != nil { - return nil, err - } - - for i := range r.schema.Fields { - if vals[i] == nil { - r.row[i] = nil - continue - } - mapper := r.fieldMappers[i] - r.row[i], err = mapper.value(vals[i]) - if err != nil { - return nil, err - } - } - - return r.row, nil -} - -// Schema implements drivers.RowIterator. -func (r *rowIterator) Schema(ctx context.Context) (*runtimev1.StructType, error) { - return r.schema, nil -} - -// Size implements drivers.RowIterator. -func (r *rowIterator) Size(unit drivers.ProgressUnit) (uint64, bool) { - return 0, false -} - -var _ drivers.RowIterator = &rowIterator{} - -func (r *rowIterator) setSchema(ctx context.Context) error { - fds := r.rows.FieldDescriptions() - conn := r.rows.Conn() - if conn == nil { - // not possible but keeping it for graceful failures - return fmt.Errorf("nil pgx conn") - } - - mappers := make([]mapper, len(fds)) - fields := make([]*runtimev1.StructType_Field, len(fds)) - typeMap := conn.TypeMap() - oidToMapperMap := getOidToMapperMap() - - var newConn *pgxpool.Conn - defer func() { - if newConn != nil { - newConn.Release() - } - }() - for i, fd := range fds { - dt := columnTypeDatabaseTypeName(typeMap, fds[i].DataTypeOID) - if dt == "" { - var err error - if newConn == nil { - newConn, err = r.acquireConn(ctx) - if err != nil { - return err - } - } - dt, err = r.registerIfEnum(ctx, newConn.Conn(), oidToMapperMap, fds[i].DataTypeOID) - if err != nil { - return err - } - } - mapper, ok := oidToMapperMap[dt] - if !ok { - return fmt.Errorf("datatype %q is not supported", dt) - } - mappers[i] = mapper - fields[i] = &runtimev1.StructType_Field{ - Name: fd.Name, - Type: mapper.runtimeType(), - } - } - - r.schema = &runtimev1.StructType{Fields: fields} - r.fieldMappers = mappers - r.row = make([]sqldriver.Value, len(r.schema.Fields)) - return nil -} - -func (r *rowIterator) registerIfEnum(ctx context.Context, conn *pgx.Conn, oidToMapperMap map[string]mapper, oid uint32) (string, error) { - // custom datatypes are not supported - // but it is possible to support enum with this approach - var isEnum bool - var typName string - err := conn.QueryRow(ctx, "SELECT typtype = 'e' AS isEnum, typname FROM pg_type WHERE oid = $1", oid).Scan(&isEnum, &typName) - if err != nil { - return "", err - } - - if !isEnum { - return "", fmt.Errorf("custom datatypes are not supported") - } - - dataType, err := conn.LoadType(ctx, typName) - if err != nil { - return "", err - } - - r.rows.Conn().TypeMap().RegisterType(dataType) - oidToMapperMap[typName] = &charMapper{} - register(oidToMapperMap, typName, &charMapper{}) - return typName, nil -} - -func (r *rowIterator) acquireConn(ctx context.Context) (*pgxpool.Conn, error) { - // acquire another connection - ctxWithTimeOut, cancel := context.WithTimeout(ctx, time.Minute) - defer cancel() - - conn, err := r.pool.Acquire(ctxWithTimeOut) - if err != nil { - if errors.Is(err, context.DeadlineExceeded) { - return nil, fmt.Errorf("postgres connector require 2 connections. Set `max_connections` to atleast 2") - } - return nil, err - } - return conn, nil -} - -// columnTypeDatabaseTypeName returns the database system type name. If the name is unknown the OID is returned. -func columnTypeDatabaseTypeName(typeMap *pgtype.Map, datatypeOID uint32) string { - if dt, ok := typeMap.TypeForOID(datatypeOID); ok { - return strings.ToLower(dt.Name) - } - return "" -} - -type sourceProperties struct { - SQL string `mapstructure:"sql"` - DatabaseURL string `mapstructure:"database_url"` -} - -func parseSourceProperties(props map[string]any) (*sourceProperties, error) { - conf := &sourceProperties{} - err := mapstructure.Decode(props, conf) - if err != nil { - return nil, err - } - if conf.SQL == "" { - return nil, fmt.Errorf("property 'sql' is mandatory for connector \"postgres\"") - } - return conf, err -} diff --git a/runtime/drivers/redshift/redshift.go b/runtime/drivers/redshift/redshift.go index 6ef9e77467f..41c7e47336e 100644 --- a/runtime/drivers/redshift/redshift.go +++ b/runtime/drivers/redshift/redshift.go @@ -233,11 +233,6 @@ func (c *Connection) AsWarehouse() (drivers.Warehouse, bool) { return c, true } -// AsSQLStore implements drivers.Connection. -func (c *Connection) AsSQLStore() (drivers.SQLStore, bool) { - return nil, false -} - // AsAI implements drivers.Handle. func (c *Connection) AsAI(instanceID string) (drivers.AIService, bool) { return nil, false diff --git a/runtime/drivers/s3/s3.go b/runtime/drivers/s3/s3.go index 2570ecf7531..84bf706591d 100644 --- a/runtime/drivers/s3/s3.go +++ b/runtime/drivers/s3/s3.go @@ -248,11 +248,6 @@ func (c *Connection) AsWarehouse() (drivers.Warehouse, bool) { return nil, false } -// AsSQLStore implements drivers.Connection. -func (c *Connection) AsSQLStore() (drivers.SQLStore, bool) { - return nil, false -} - // AsNotifier implements drivers.Connection. func (c *Connection) AsNotifier(properties map[string]any) (drivers.Notifier, error) { return nil, drivers.ErrNotNotifier diff --git a/runtime/drivers/salesforce/salesforce.go b/runtime/drivers/salesforce/salesforce.go index 36171336129..5e31d3cae36 100644 --- a/runtime/drivers/salesforce/salesforce.go +++ b/runtime/drivers/salesforce/salesforce.go @@ -247,11 +247,6 @@ func (c *connection) AsWarehouse() (drivers.Warehouse, bool) { return c, true } -// AsSQLStore implements drivers.Connection. -func (c *connection) AsSQLStore() (drivers.SQLStore, bool) { - return nil, false -} - // AsNotifier implements drivers.Connection. func (c *connection) AsNotifier(properties map[string]any) (drivers.Notifier, error) { return nil, drivers.ErrNotNotifier diff --git a/runtime/drivers/slack/slack.go b/runtime/drivers/slack/slack.go index 046fd2b8f20..04490dd48ff 100644 --- a/runtime/drivers/slack/slack.go +++ b/runtime/drivers/slack/slack.go @@ -115,10 +115,6 @@ func (h *handle) AsAI(instanceID string) (drivers.AIService, bool) { return nil, false } -func (h *handle) AsSQLStore() (drivers.SQLStore, bool) { - return nil, false -} - func (h *handle) AsOLAP(instanceID string) (drivers.OLAPStore, bool) { return nil, false } diff --git a/runtime/drivers/snowflake/snowflake.go b/runtime/drivers/snowflake/snowflake.go index b86d938e870..678b105aed3 100644 --- a/runtime/drivers/snowflake/snowflake.go +++ b/runtime/drivers/snowflake/snowflake.go @@ -203,11 +203,6 @@ func (c *connection) AsWarehouse() (drivers.Warehouse, bool) { return c, true } -// AsSQLStore implements drivers.Connection. -func (c *connection) AsSQLStore() (drivers.SQLStore, bool) { - return nil, false -} - // AsNotifier implements drivers.Connection. func (c *connection) AsNotifier(properties map[string]any) (drivers.Notifier, error) { return nil, drivers.ErrNotNotifier diff --git a/runtime/drivers/snowflake/sql_store.go b/runtime/drivers/snowflake/warehouse.go similarity index 100% rename from runtime/drivers/snowflake/sql_store.go rename to runtime/drivers/snowflake/warehouse.go diff --git a/runtime/drivers/sql_store.go b/runtime/drivers/sql_store.go deleted file mode 100644 index 072304d850a..00000000000 --- a/runtime/drivers/sql_store.go +++ /dev/null @@ -1,34 +0,0 @@ -package drivers - -import ( - "context" - "database/sql/driver" - "errors" - - runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" -) - -var ErrIteratorDone = errors.New("empty iterator") - -var ErrNoRows = errors.New("no rows found for the query") - -// SQLStore is implemented by drivers capable of running sql queries and generating an iterator to consume results. -// In future the results can be produced in other formats like arrow as well. -// May be call it DataWarehouse to differentiate from OLAP or postgres? -type SQLStore interface { - // Query returns driver.RowIterator to iterate over results row by row - Query(ctx context.Context, props map[string]any) (RowIterator, error) -} - -// RowIterator returns an iterator to iterate over result of a sql query -type RowIterator interface { - // Schema of the underlying data - Schema(ctx context.Context) (*runtimev1.StructType, error) - // Next fetches next row - Next(ctx context.Context) ([]driver.Value, error) - // Close closes the iterator and frees resources - Close() error - // Size returns total size of data downloaded in unit. - // Returns 0,false if not able to compute size in given unit - Size(unit ProgressUnit) (uint64, bool) -} diff --git a/runtime/drivers/sqlite/sqlite.go b/runtime/drivers/sqlite/sqlite.go index 7d490c551fd..18c1f5a5970 100644 --- a/runtime/drivers/sqlite/sqlite.go +++ b/runtime/drivers/sqlite/sqlite.go @@ -181,11 +181,6 @@ func (c *connection) AsWarehouse() (drivers.Warehouse, bool) { return nil, false } -// AsSQLStore implements drivers.Connection. -func (c *connection) AsSQLStore() (drivers.SQLStore, bool) { - return nil, false -} - // AsNotifier implements drivers.Connection. func (c *connection) AsNotifier(properties map[string]any) (drivers.Notifier, error) { return nil, drivers.ErrNotNotifier diff --git a/runtime/drivers/warehouse.go b/runtime/drivers/warehouse.go index a9fa7a963bc..5dd8fbcc6f8 100644 --- a/runtime/drivers/warehouse.go +++ b/runtime/drivers/warehouse.go @@ -2,8 +2,11 @@ package drivers import ( "context" + "errors" ) +var ErrNoRows = errors.New("no rows found for the query") + type Warehouse interface { // QueryAsFiles downloads results into files and returns an iterator to iterate over them QueryAsFiles(ctx context.Context, props map[string]any) (FileIterator, error) diff --git a/runtime/metricsview/executor.go b/runtime/metricsview/executor.go index c6936622f5b..9268fb9f773 100644 --- a/runtime/metricsview/executor.go +++ b/runtime/metricsview/executor.go @@ -71,12 +71,6 @@ func (e *Executor) Cacheable(qry *Query) bool { return e.olap.Dialect() == drivers.DialectDuckDB } -// ValidateMetricsView validates the dimensions and measures in the executor's metrics view. -func (e *Executor) ValidateMetricsView(ctx context.Context) error { - // TODO: Implement it - panic("not implemented") -} - // ValidateQuery validates the provided query against the executor's metrics view. func (e *Executor) ValidateQuery(qry *Query) error { // TODO: Implement it diff --git a/runtime/metricsview/executor_pivot.go b/runtime/metricsview/executor_pivot.go index 6996e96b21d..b28d19839bc 100644 --- a/runtime/metricsview/executor_pivot.go +++ b/runtime/metricsview/executor_pivot.go @@ -146,7 +146,7 @@ func (e *Executor) executePivotExport(ctx context.Context, ast *AST, pivot *pivo } defer release() var path string - err = olap.WithConnection(ctx, e.priority, false, false, func(wrappedCtx context.Context, ensuredCtx context.Context, conn *sql.Conn) error { + err = olap.WithConnection(ctx, e.priority, false, func(wrappedCtx context.Context, ensuredCtx context.Context, conn *sql.Conn) error { // Stage the underlying data in a temporary table alias, err := randomString("t", 8) if err != nil { diff --git a/runtime/metricsview/executor_rewrite_approx_comparisons.go b/runtime/metricsview/executor_rewrite_approx_comparisons.go index 5f57f7355be..0a599d397f3 100644 --- a/runtime/metricsview/executor_rewrite_approx_comparisons.go +++ b/runtime/metricsview/executor_rewrite_approx_comparisons.go @@ -112,15 +112,18 @@ func (e *Executor) rewriteApproxComparisonNode(a *AST, n *SelectNode) bool { n.FromSelect.Limit = a.Root.Limit n.FromSelect.Offset = a.Root.Offset - // ---- CTE Optimization ---- // - // make FromSelect a CTE - a.convertToCTE(n.FromSelect) - - // now change the JoinComparisonSelect WHERE clause to use selected dim values from CTE - for _, dim := range n.JoinComparisonSelect.DimFields { - dimName := a.dialect.EscapeIdentifier(dim.Name) - dimExpr := "(" + dim.Expr + ")" // wrap in parentheses to handle expressions - n.JoinComparisonSelect.Where = n.JoinComparisonSelect.Where.and(fmt.Sprintf("%[1]s IS NULL OR %[1]s IN (SELECT %[2]q.%[3]s FROM %[2]q)", dimExpr, n.FromSelect.Alias, dimName), nil) + // don't optimize for ClickHouse as its unable to inline the CTE results causing multiple scans + if e.olap.Dialect() != drivers.DialectClickHouse { + // ---- CTE Optimization ---- // + // make FromSelect a CTE + a.convertToCTE(n.FromSelect) + + // now change the JoinComparisonSelect WHERE clause to use selected dim values from CTE + for _, dim := range n.JoinComparisonSelect.DimFields { + dimName := a.dialect.EscapeIdentifier(dim.Name) + dimExpr := "(" + dim.Expr + ")" // wrap in parentheses to handle expressions + n.JoinComparisonSelect.Where = n.JoinComparisonSelect.Where.and(fmt.Sprintf("%[1]s IS NULL OR %[1]s IN (SELECT %[2]q.%[3]s FROM %[2]q)", dimExpr, n.FromSelect.Alias, dimName), nil) + } } } else if sortComparison { // We're sorting by a measure in JoinComparisonSelect. We can do a RIGHT JOIN and push down the order/limit to it. @@ -131,15 +134,18 @@ func (e *Executor) rewriteApproxComparisonNode(a *AST, n *SelectNode) bool { n.JoinComparisonSelect.Limit = a.Root.Limit n.JoinComparisonSelect.Offset = a.Root.Offset - // ---- CTE Optimization ---- // - // make JoinComparisonSelect a CTE - a.convertToCTE(n.JoinComparisonSelect) - - // now change the FromSelect WHERE clause to use selected dim values from CTE - for _, dim := range n.FromSelect.DimFields { - dimName := a.dialect.EscapeIdentifier(dim.Name) - dimExpr := "(" + dim.Expr + ")" // wrap in parentheses to handle expressions - n.FromSelect.Where = n.FromSelect.Where.and(fmt.Sprintf("%[1]s IS NULL OR %[1]s IN (SELECT %[2]q.%[3]s FROM %[2]q)", dimExpr, n.JoinComparisonSelect.Alias, dimName), nil) + // don't optimize for ClickHouse as its unable to inline the CTE results causing multiple scans + if e.olap.Dialect() != drivers.DialectClickHouse { + // ---- CTE Optimization ---- // + // make JoinComparisonSelect a CTE + a.convertToCTE(n.JoinComparisonSelect) + + // now change the FromSelect WHERE clause to use selected dim values from CTE + for _, dim := range n.FromSelect.DimFields { + dimName := a.dialect.EscapeIdentifier(dim.Name) + dimExpr := "(" + dim.Expr + ")" // wrap in parentheses to handle expressions + n.FromSelect.Where = n.FromSelect.Where.and(fmt.Sprintf("%[1]s IS NULL OR %[1]s IN (SELECT %[2]q.%[3]s FROM %[2]q)", dimExpr, n.JoinComparisonSelect.Alias, dimName), nil) + } } } else if sortDim { // We're sorting by a dimension. We do a LEFT JOIN that only returns values present in the base query. @@ -149,15 +155,18 @@ func (e *Executor) rewriteApproxComparisonNode(a *AST, n *SelectNode) bool { n.FromSelect.Limit = a.Root.Limit n.FromSelect.Offset = a.Root.Offset - // ---- CTE Optimization ---- // - // make FromSelect a CTE - a.convertToCTE(n.FromSelect) - - // now change the JoinComparisonSelect WHERE clause to use selected dim values from CTE - for _, dim := range n.JoinComparisonSelect.DimFields { - dimName := a.dialect.EscapeIdentifier(dim.Name) - dimExpr := "(" + dim.Expr + ")" // wrap in parentheses to handle expressions - n.JoinComparisonSelect.Where = n.JoinComparisonSelect.Where.and(fmt.Sprintf("%[1]s IS NULL OR %[1]s IN (SELECT %[2]q.%[3]s FROM %[2]q)", dimExpr, n.FromSelect.Alias, dimName), nil) + // don't optimize for ClickHouse as its unable to inline the CTE results causing multiple scans + if e.olap.Dialect() != drivers.DialectClickHouse { + // ---- CTE Optimization ---- // + // make FromSelect a CTE + a.convertToCTE(n.FromSelect) + + // now change the JoinComparisonSelect WHERE clause to use selected dim values from CTE + for _, dim := range n.JoinComparisonSelect.DimFields { + dimName := a.dialect.EscapeIdentifier(dim.Name) + dimExpr := "(" + dim.Expr + ")" // wrap in parentheses to handle expressions + n.JoinComparisonSelect.Where = n.JoinComparisonSelect.Where.and(fmt.Sprintf("%[1]s IS NULL OR %[1]s IN (SELECT %[2]q.%[3]s FROM %[2]q)", dimExpr, n.FromSelect.Alias, dimName), nil) + } } } else if sortDelta { return false diff --git a/runtime/validate.go b/runtime/metricsview/executor_validate.go similarity index 69% rename from runtime/validate.go rename to runtime/metricsview/executor_validate.go index 0e84c604313..dee231252dd 100644 --- a/runtime/validate.go +++ b/runtime/metricsview/executor_validate.go @@ -1,4 +1,4 @@ -package runtime +package metricsview import ( "context" @@ -10,12 +10,14 @@ import ( "sync" runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" + "github.com/rilldata/rill/runtime" "github.com/rilldata/rill/runtime/drivers" "golang.org/x/sync/errgroup" ) const validateConcurrencyLimit = 10 +// ValidateMetricsViewResult contains the results of validating a metrics view. type ValidateMetricsViewResult struct { TimeDimensionErr error DimensionErrs []IndexErr @@ -23,11 +25,13 @@ type ValidateMetricsViewResult struct { OtherErrs []error } +// IndexErr contains an error and the index of the dimension or measure that caused the error. type IndexErr struct { Idx int Err error } +// IsZero returns true if the result contains no errors. func (r *ValidateMetricsViewResult) IsZero() bool { return r.TimeDimensionErr == nil && len(r.DimensionErrs) == 0 && len(r.MeasureErrs) == 0 && len(r.OtherErrs) == 0 } @@ -49,25 +53,14 @@ func (r *ValidateMetricsViewResult) Error() error { return errors.Join(errs...) } -// ValidateMetricsView validates a metrics view spec. -// NOTE: If we need validation for more resources, we should consider moving it to the queries (or a dedicated validation package). -func (r *Runtime) ValidateMetricsView(ctx context.Context, instanceID string, mv *runtimev1.MetricsViewSpec) (*ValidateMetricsViewResult, error) { - ctrl, err := r.Controller(ctx, instanceID) - if err != nil { - return nil, err - } - - olap, release, err := ctrl.AcquireOLAP(ctx, mv.Connector) - if err != nil { - return nil, err - } - defer release() - +// ValidateMetricsView validates the dimensions and measures in the executor's metrics view. +func (e *Executor) ValidateMetricsView(ctx context.Context) (*ValidateMetricsViewResult, error) { // Create the result res := &ValidateMetricsViewResult{} // Check underlying table exists - t, err := olap.InformationSchema().Lookup(ctx, mv.Database, mv.DatabaseSchema, mv.Table) + mv := e.metricsView + t, err := e.olap.InformationSchema().Lookup(ctx, mv.Database, mv.DatabaseSchema, mv.Table) if err != nil { if errors.Is(err, drivers.ErrNotFound) { res.OtherErrs = append(res.OtherErrs, fmt.Errorf("table %q does not exist", mv.Table)) @@ -112,7 +105,7 @@ func (r *Runtime) ValidateMetricsView(ctx context.Context, instanceID string, mv // ClickHouse specifically does not support using a column name as a dimension or measure name if the dimension or measure has an expression. // This is due to ClickHouse's aggressive substitution of aliases: https://github.com/ClickHouse/ClickHouse/issues/9715. - if olap.Dialect() == drivers.DialectClickHouse { + if e.olap.Dialect() == drivers.DialectClickHouse { for _, d := range mv.Dimensions { if d.Expression == "" && !d.Unnest { continue @@ -133,25 +126,39 @@ func (r *Runtime) ValidateMetricsView(ctx context.Context, instanceID string, mv } // For performance, attempt to validate all dimensions and measures at once - err = validateAllDimensionsAndMeasures(ctx, olap, t, mv) + err = e.validateAllDimensionsAndMeasures(ctx, t, mv) if err != nil { // One or more dimension/measure expressions failed to validate. We need to check each one individually to provide useful errors. - validateIndividualDimensionsAndMeasures(ctx, olap, t, mv, cols, res) + e.validateIndividualDimensionsAndMeasures(ctx, t, mv, cols, res) } // Pinot does have any native support for time shift using time grain specifiers - if olap.Dialect() == drivers.DialectPinot && (mv.FirstDayOfWeek > 1 || mv.FirstMonthOfYear > 1) { + if e.olap.Dialect() == drivers.DialectPinot && (mv.FirstDayOfWeek > 1 || mv.FirstMonthOfYear > 1) { res.OtherErrs = append(res.OtherErrs, fmt.Errorf("time shift not supported for Pinot dialect, so FirstDayOfWeek and FirstMonthOfYear should be 1")) } // Check the default theme exists if mv.DefaultTheme != "" { - _, err := ctrl.Get(ctx, &runtimev1.ResourceName{Kind: ResourceKindTheme, Name: mv.DefaultTheme}, false) + ctrl, err := e.rt.Controller(ctx, e.instanceID) + if err != nil { + return nil, fmt.Errorf("could not get controller: %w", err) + } + + _, err = ctrl.Get(ctx, &runtimev1.ResourceName{Kind: runtime.ResourceKindTheme, Name: mv.DefaultTheme}, false) if err != nil { if errors.Is(err, drivers.ErrNotFound) { res.OtherErrs = append(res.OtherErrs, fmt.Errorf("theme %q does not exist", mv.DefaultTheme)) + } else { + return nil, fmt.Errorf("could not find theme %q: %w", mv.DefaultTheme, err) } - return nil, fmt.Errorf("could not find theme %q: %w", mv.DefaultTheme, err) + } + } + + // Validate the metrics view schema. + if res.IsZero() { // All dimensions and measures need to be valid to compute the schema. + err = e.validateSchema(ctx, res) + if err != nil { + res.OtherErrs = append(res.OtherErrs, fmt.Errorf("failed to validate metrics view schema: %w", err)) } } @@ -159,8 +166,8 @@ func (r *Runtime) ValidateMetricsView(ctx context.Context, instanceID string, mv } // validateAllDimensionsAndMeasures validates all dimensions and measures with one query. It returns an error if any of the expressions are invalid. -func validateAllDimensionsAndMeasures(ctx context.Context, olap drivers.OLAPStore, t *drivers.Table, mv *runtimev1.MetricsViewSpec) error { - dialect := olap.Dialect() +func (e *Executor) validateAllDimensionsAndMeasures(ctx context.Context, t *drivers.Table, mv *runtimev1.MetricsViewSpec) error { + dialect := e.olap.Dialect() var dimExprs []string var unnestClauses []string var groupIndexes []string @@ -186,13 +193,13 @@ func validateAllDimensionsAndMeasures(ctx context.Context, olap drivers.OLAPStor } if len(dimExprs) == 0 { // Only metrics - query = fmt.Sprintf("SELECT 1, %s FROM %s GROUP BY 1", strings.Join(metricExprs, ","), olap.Dialect().EscapeTable(t.Database, t.DatabaseSchema, t.Name)) + query = fmt.Sprintf("SELECT 1, %s FROM %s GROUP BY 1", strings.Join(metricExprs, ","), e.olap.Dialect().EscapeTable(t.Database, t.DatabaseSchema, t.Name)) } else if len(metricExprs) == 0 { // No metrics query = fmt.Sprintf( "SELECT %s FROM %s %s GROUP BY %s", strings.Join(dimExprs, ","), - olap.Dialect().EscapeTable(t.Database, t.DatabaseSchema, t.Name), + e.olap.Dialect().EscapeTable(t.Database, t.DatabaseSchema, t.Name), strings.Join(unnestClauses, ""), strings.Join(groupIndexes, ","), ) @@ -201,12 +208,12 @@ func validateAllDimensionsAndMeasures(ctx context.Context, olap drivers.OLAPStor "SELECT %s, %s FROM %s %s GROUP BY %s", strings.Join(dimExprs, ","), strings.Join(metricExprs, ","), - olap.Dialect().EscapeTable(t.Database, t.DatabaseSchema, t.Name), + e.olap.Dialect().EscapeTable(t.Database, t.DatabaseSchema, t.Name), strings.Join(unnestClauses, ""), strings.Join(groupIndexes, ","), ) } - err := olap.Exec(ctx, &drivers.Statement{ + err := e.olap.Exec(ctx, &drivers.Statement{ Query: query, DryRun: true, }) @@ -218,7 +225,7 @@ func validateAllDimensionsAndMeasures(ctx context.Context, olap drivers.OLAPStor // validateIndividualDimensionsAndMeasures validates each dimension and measure individually. // It adds validation errors to the provided res. -func validateIndividualDimensionsAndMeasures(ctx context.Context, olap drivers.OLAPStore, t *drivers.Table, mv *runtimev1.MetricsViewSpec, cols map[string]*runtimev1.StructType_Field, res *ValidateMetricsViewResult) { +func (e *Executor) validateIndividualDimensionsAndMeasures(ctx context.Context, t *drivers.Table, mv *runtimev1.MetricsViewSpec, cols map[string]*runtimev1.StructType_Field, res *ValidateMetricsViewResult) { // Validate dimensions and measures concurrently with a limit of 10 concurrent validations var mu sync.Mutex var grp errgroup.Group @@ -229,7 +236,7 @@ func validateIndividualDimensionsAndMeasures(ctx context.Context, olap drivers.O idx := idx d := d grp.Go(func() error { - err := validateDimension(ctx, olap, t, d, cols) + err := e.validateDimension(ctx, t, d, cols) if err != nil { mu.Lock() defer mu.Unlock() @@ -252,7 +259,7 @@ func validateIndividualDimensionsAndMeasures(ctx context.Context, olap drivers.O idx := idx m := m grp.Go(func() error { - err := validateMeasure(ctx, olap, t, m) + err := e.validateMeasure(ctx, t, m) if err != nil { mu.Lock() defer mu.Unlock() @@ -275,7 +282,7 @@ func validateIndividualDimensionsAndMeasures(ctx context.Context, olap drivers.O } // validateDimension validates a metrics view dimension. -func validateDimension(ctx context.Context, olap drivers.OLAPStore, t *drivers.Table, d *runtimev1.MetricsViewSpec_DimensionV2, fields map[string]*runtimev1.StructType_Field) error { +func (e *Executor) validateDimension(ctx context.Context, t *drivers.Table, d *runtimev1.MetricsViewSpec_DimensionV2, fields map[string]*runtimev1.StructType_Field) error { // Validate with a simple check if it's a column if d.Column != "" { if _, isColumn := fields[strings.ToLower(d.Column)]; !isColumn { @@ -287,11 +294,11 @@ func validateDimension(ctx context.Context, olap drivers.OLAPStore, t *drivers.T } } - dialect := olap.Dialect() + dialect := e.olap.Dialect() expr, unnestClause := dialect.DimensionSelect(t.Database, t.DatabaseSchema, t.Name, d) // Validate with a query if it's an expression - err := olap.Exec(ctx, &drivers.Statement{ + err := e.olap.Exec(ctx, &drivers.Statement{ Query: fmt.Sprintf("SELECT %s FROM %s %s GROUP BY 1", expr, dialect.EscapeTable(t.Database, t.DatabaseSchema, t.Name), unnestClause), DryRun: true, }) @@ -302,10 +309,42 @@ func validateDimension(ctx context.Context, olap drivers.OLAPStore, t *drivers.T } // validateMeasure validates a metrics view measure. -func validateMeasure(ctx context.Context, olap drivers.OLAPStore, t *drivers.Table, m *runtimev1.MetricsViewSpec_MeasureV2) error { - err := olap.Exec(ctx, &drivers.Statement{ - Query: fmt.Sprintf("SELECT 1, (%s) FROM %s GROUP BY 1", m.Expression, olap.Dialect().EscapeTable(t.Database, t.DatabaseSchema, t.Name)), +func (e *Executor) validateMeasure(ctx context.Context, t *drivers.Table, m *runtimev1.MetricsViewSpec_MeasureV2) error { + err := e.olap.Exec(ctx, &drivers.Statement{ + Query: fmt.Sprintf("SELECT 1, (%s) FROM %s GROUP BY 1", m.Expression, e.olap.Dialect().EscapeTable(t.Database, t.DatabaseSchema, t.Name)), DryRun: true, }) return err } + +// validateSchema validates that the metrics view's measures are numeric. +func (e *Executor) validateSchema(ctx context.Context, res *ValidateMetricsViewResult) error { + // Resolve the schema of the metrics view's dimensions and measures + schema, err := e.Schema(ctx) + if err != nil { + return err + } + types := make(map[string]*runtimev1.Type, len(schema.Fields)) + for _, f := range schema.Fields { + types[f.Name] = f.Type + } + + // Check that the measures are not strings + for i, m := range e.metricsView.Measures { + typ, ok := types[m.Name] + if !ok { + // Don't error: schemas are not always reliable + continue + } + + switch typ.Code { + case runtimev1.Type_CODE_TIMESTAMP, runtimev1.Type_CODE_DATE, runtimev1.Type_CODE_TIME, runtimev1.Type_CODE_STRING, runtimev1.Type_CODE_BYTES, runtimev1.Type_CODE_ARRAY, runtimev1.Type_CODE_STRUCT, runtimev1.Type_CODE_MAP, runtimev1.Type_CODE_JSON, runtimev1.Type_CODE_UUID: + res.MeasureErrs = append(res.MeasureErrs, IndexErr{ + Idx: i, + Err: fmt.Errorf("measure %q is of type %s, but must be a numeric type", m.Name, typ.Code), + }) + } + } + + return nil +} diff --git a/runtime/validate_test.go b/runtime/metricsview/executor_validate_test.go similarity index 92% rename from runtime/validate_test.go rename to runtime/metricsview/executor_validate_test.go index 84e34a554f3..826f1266d1b 100644 --- a/runtime/validate_test.go +++ b/runtime/metricsview/executor_validate_test.go @@ -1,4 +1,4 @@ -package runtime_test +package metricsview_test import ( "context" @@ -6,26 +6,32 @@ import ( runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" "github.com/rilldata/rill/runtime" + "github.com/rilldata/rill/runtime/metricsview" "github.com/rilldata/rill/runtime/testruntime" "github.com/stretchr/testify/require" ) func TestValidateMetricsView(t *testing.T) { rt, instanceID := testruntime.NewInstanceForProject(t, "ad_bids") - res, err := rt.ValidateMetricsView(context.Background(), instanceID, &runtimev1.MetricsViewSpec{ + mv := &runtimev1.MetricsViewSpec{ Connector: "duckdb", Table: "ad_bids", DisplayName: "Ad Bids", TimeDimension: "timestamp", Dimensions: []*runtimev1.MetricsViewSpec_DimensionV2{ - {Column: "publisher"}, + {Name: "publisher", Column: "publisher"}, }, Measures: []*runtimev1.MetricsViewSpec_MeasureV2{ {Name: "records", Expression: "count(*)", Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE}, {Name: "invalid_nested_aggregation", Expression: "MAX(COUNT(DISTINCT publisher))", Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE}, {Name: "invalid_partition", Expression: "AVG(bid_price) OVER (PARTITION BY publisher)", Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE}, }, - }) + } + + e, err := metricsview.NewExecutor(context.Background(), rt, instanceID, mv, runtime.ResolvedSecurityOpen, 0) + require.NoError(t, err) + + res, err := e.ValidateMetricsView(context.Background()) require.NoError(t, err) require.Empty(t, res.TimeDimensionErr) require.Empty(t, res.DimensionErrs) diff --git a/runtime/pkg/jsonval/jsonval.go b/runtime/pkg/jsonval/jsonval.go index 384b9d748db..2a7e503ab7f 100644 --- a/runtime/pkg/jsonval/jsonval.go +++ b/runtime/pkg/jsonval/jsonval.go @@ -14,6 +14,7 @@ import ( "github.com/marcboeker/go-duckdb" "github.com/paulmach/orb" runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" + "github.com/rilldata/rill/runtime/drivers/clickhouse" ) // ToValue converts a value scanned from a database/sql driver to a Go type that can be marshaled to JSON. @@ -58,12 +59,22 @@ func ToValue(v any, t *runtimev1.Type) (any, error) { } return v, nil case string: - if t != nil && t.Code == runtimev1.Type_CODE_DECIMAL { - // Evil cast to float until frontend can deal with bigs: - v2, ok := new(big.Float).SetString(v) - if ok { - f, _ := v2.Float64() - return f, nil + if t != nil { + switch t.Code { + case runtimev1.Type_CODE_DECIMAL: + // Evil cast to float until frontend can deal with bigs: + v2, ok := new(big.Float).SetString(v) + if ok { + f, _ := v2.Float64() + return f, nil + } + case runtimev1.Type_CODE_INTERVAL: + // ClickHouse currently returns INTERVALs as strings. + // Our current policy is to convert INTERVALs to milliseconds, treating one month as 30 days. + v2, ok := clickhouse.ParseIntervalToMillis(v) + if ok { + return v2, nil + } } } return strings.ToValidUTF8(v, "�"), nil @@ -110,7 +121,11 @@ func ToValue(v any, t *runtimev1.Type) (any, error) { case duckdb.Map: return ToValue(map[any]any(v), t) case duckdb.Interval: - return map[string]any{"months": v.Months, "days": v.Days, "micros": v.Micros}, nil + // Our current policy is to convert INTERVALs to milliseconds, treating one month as 30 days. + ms := v.Micros / 1000 + ms += int64(v.Days) * 24 * 60 * 60 * 1000 + ms += int64(v.Months) * 30 * 24 * 60 * 60 * 1000 + return ms, nil case net.IP: return v.String(), nil case orb.Point: diff --git a/runtime/pkg/pbutil/pbutil.go b/runtime/pkg/pbutil/pbutil.go index 1860759c77a..8d7cc874c32 100644 --- a/runtime/pkg/pbutil/pbutil.go +++ b/runtime/pkg/pbutil/pbutil.go @@ -13,6 +13,7 @@ import ( "github.com/marcboeker/go-duckdb" "github.com/paulmach/orb" runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" + "github.com/rilldata/rill/runtime/drivers/clickhouse" "google.golang.org/protobuf/types/known/structpb" ) @@ -103,12 +104,11 @@ func ToValue(v any, t *runtimev1.Type) (*structpb.Value, error) { } return structpb.NewStructValue(v2), nil case duckdb.Interval: - m := map[string]any{"months": v.Months, "days": v.Days, "micros": v.Micros} - v2, err := ToStruct(m, nil) - if err != nil { - return nil, err - } - return structpb.NewStructValue(v2), nil + // Our current policy is to convert INTERVALs to milliseconds, treating one month as 30 days. + ms := v.Micros / 1000 + ms += int64(v.Days) * 24 * 60 * 60 * 1000 + ms += int64(v.Months) * 30 * 24 * 60 * 60 * 1000 + return structpb.NewNumberValue(float64(ms)), nil case []byte: if t != nil && t.Code == runtimev1.Type_CODE_UUID { uid, err := uuid.FromBytes(v) @@ -117,12 +117,22 @@ func ToValue(v any, t *runtimev1.Type) (*structpb.Value, error) { } } case string: - if t != nil && t.Code == runtimev1.Type_CODE_DECIMAL { - // Evil cast to float until frontend can deal with bigs: - v2, ok := new(big.Float).SetString(v) - if ok { - f, _ := v2.Float64() - return structpb.NewNumberValue(f), nil + if t != nil { + switch t.Code { + case runtimev1.Type_CODE_DECIMAL: + // Evil cast to float until frontend can deal with bigs: + v2, ok := new(big.Float).SetString(v) + if ok { + f, _ := v2.Float64() + return structpb.NewNumberValue(f), nil + } + case runtimev1.Type_CODE_INTERVAL: + // ClickHouse currently returns INTERVALs as strings. + // Our current policy is to convert INTERVALs to milliseconds, treating one month as 30 days. + v2, ok := clickhouse.ParseIntervalToMillis(v) + if ok { + return structpb.NewNumberValue(float64(v2)), nil + } } } return structpb.NewStringValue(strings.ToValidUTF8(v, "�")), nil diff --git a/runtime/pkg/rduckdb/db.go b/runtime/pkg/rduckdb/db.go index 34f4d5c7753..cdce90781d1 100644 --- a/runtime/pkg/rduckdb/db.go +++ b/runtime/pkg/rduckdb/db.go @@ -202,6 +202,7 @@ type CreateTableOptions struct { // If BeforeCreateFn is set, it will be executed before the create query is executed. BeforeCreateFn func(ctx context.Context, conn *sqlx.Conn) error // If AfterCreateFn is set, it will be executed after the create query is executed. + // This will execute even if the create query fails. AfterCreateFn func(ctx context.Context, conn *sqlx.Conn) error } @@ -266,7 +267,7 @@ func NewDB(ctx context.Context, opts *DBOptions) (DB, error) { opts.Logger, ) - db.dbHandle, err = db.openDBAndAttach(ctx, "", "", true) + db.dbHandle, err = db.openDBAndAttach(ctx, filepath.Join(db.localPath, "main.db"), "", true) if err != nil { if strings.Contains(err.Error(), "Symbol not found") { fmt.Printf("Your version of macOS is not supported. Please upgrade to the latest major release of macOS. See this link for details: https://support.apple.com/en-in/macos/upgrade") @@ -274,6 +275,7 @@ func NewDB(ctx context.Context, opts *DBOptions) (DB, error) { } return nil, err } + go db.localDBMonitor() return db, nil } @@ -397,16 +399,24 @@ func (d *db) CreateTableAsSelect(ctx context.Context, name, query string, opts * return fmt.Errorf("create: BeforeCreateFn returned error: %w", err) } } - // ingest data - _, err = conn.ExecContext(ctx, fmt.Sprintf("CREATE OR REPLACE %s %s AS (%s\n)", typ, safeName, query), nil) - if err != nil { - return fmt.Errorf("create: create %s %q failed: %w", typ, name, err) - } - if opts.AfterCreateFn != nil { + execAfterCreate := func() error { + if opts.AfterCreateFn == nil { + return nil + } err = opts.AfterCreateFn(ctx, conn) if err != nil { return fmt.Errorf("create: AfterCreateFn returned error: %w", err) } + return nil + } + // ingest data + _, err = conn.ExecContext(ctx, fmt.Sprintf("CREATE OR REPLACE %s %s AS (%s\n)", typ, safeName, query), nil) + if err != nil { + return errors.Join(fmt.Errorf("create: create %s %q failed: %w", typ, name, err), execAfterCreate()) + } + err = execAfterCreate() + if err != nil { + return err } // close write handle before syncing local so that temp files or wal files are removed @@ -650,6 +660,7 @@ func (d *db) localDBMonitor() { if err != nil && !errors.Is(err, context.Canceled) { d.logger.Error("localDBMonitor: error in pulling from remote", slog.String("error", err.Error())) } + d.localDirty = false d.writeSem.Release(1) } } @@ -723,15 +734,22 @@ func (d *db) openDBAndAttach(ctx context.Context, uri, ignoreTable string, read // this is required since spaces and other special characters are valid in db file path but invalid and hence encoded in URL connector, err := duckdb.NewConnector(generateDSN(dsn.Path, query.Encode()), func(execer driver.ExecerContext) error { for _, qry := range d.opts.InitQueries { - _, err := execer.ExecContext(context.Background(), qry, nil) + _, err := execer.ExecContext(ctx, qry, nil) if err != nil && strings.Contains(err.Error(), "Failed to download extension") { // Retry using another mirror. Based on: https://github.com/duckdb/duckdb/issues/9378 - _, err = execer.ExecContext(context.Background(), qry+" FROM 'http://nightly-extensions.duckdb.org'", nil) + _, err = execer.ExecContext(ctx, qry+" FROM 'http://nightly-extensions.duckdb.org'", nil) } if err != nil { return err } } + if !read { + // disable any more configuration changes on the write handle via init queries + _, err = execer.ExecContext(ctx, "SET lock_configuration TO true", nil) + if err != nil { + return err + } + } return nil }) if err != nil { @@ -954,7 +972,7 @@ func (d *db) removeTableVersion(ctx context.Context, name, version string) error } defer d.metaSem.Release(1) - _, err = d.dbHandle.ExecContext(ctx, "DETACH DATABASE IF EXISTS "+dbName(name, version)) + _, err = d.dbHandle.ExecContext(ctx, "DETACH DATABASE IF EXISTS "+safeSQLName(dbName(name, version))) if err != nil { return err } @@ -966,7 +984,7 @@ func (d *db) deleteLocalTableFiles(name, version string) error { return os.RemoveAll(d.localTableDir(name, version)) } -func (d *db) iterateLocalTables(removeInvalidTable bool, fn func(name string, meta *tableMeta) error) error { +func (d *db) iterateLocalTables(cleanup bool, fn func(name string, meta *tableMeta) error) error { entries, err := os.ReadDir(d.localPath) if err != nil { return err @@ -977,15 +995,36 @@ func (d *db) iterateLocalTables(removeInvalidTable bool, fn func(name string, me } meta, err := d.tableMeta(entry.Name()) if err != nil { - if !removeInvalidTable { + if !cleanup { continue } + d.logger.Debug("cleanup: remove table", slog.String("table", entry.Name())) err = d.deleteLocalTableFiles(entry.Name(), "") if err != nil { return err } continue } + // also remove older versions + if cleanup { + versions, err := os.ReadDir(d.localTableDir(entry.Name(), "")) + if err != nil { + return err + } + for _, version := range versions { + if !version.IsDir() { + continue + } + if version.Name() == meta.Version { + continue + } + d.logger.Debug("cleanup: remove old version", slog.String("table", entry.Name()), slog.String("version", version.Name())) + err = d.deleteLocalTableFiles(entry.Name(), version.Name()) + if err != nil { + return err + } + } + } err = fn(entry.Name(), meta) if err != nil { return err @@ -1031,7 +1070,7 @@ func (d *db) removeSnapshot(ctx context.Context, id int) error { } defer d.metaSem.Release(1) - _, err = d.dbHandle.Exec(fmt.Sprintf("DROP SCHEMA %s CASCADE", schemaName(id))) + _, err = d.dbHandle.Exec(fmt.Sprintf("DROP SCHEMA IF EXISTS %s CASCADE", schemaName(id))) return err } diff --git a/runtime/pkg/rduckdb/remote.go b/runtime/pkg/rduckdb/remote.go index a4bf9eaf659..150c255e588 100644 --- a/runtime/pkg/rduckdb/remote.go +++ b/runtime/pkg/rduckdb/remote.go @@ -21,8 +21,14 @@ import ( // pullFromRemote updates local data with the latest data from remote. // This is not safe for concurrent calls. func (d *db) pullFromRemote(ctx context.Context, updateCatalog bool) error { - if !d.localDirty { + if !d.localDirty || d.remote == nil { // optimisation to skip sync if write was already synced + if !updateCatalog { + // cleanup of older versions of table + _ = d.iterateLocalTables(true, func(name string, meta *tableMeta) error { + return nil + }) + } return nil } d.logger.Debug("syncing from remote") @@ -91,7 +97,7 @@ func (d *db) pullFromRemote(ctx context.Context, updateCatalog bool) error { // check if table is locally present meta, _ := d.tableMeta(table) if meta != nil && meta.Version == remoteMeta.Version { - d.logger.Debug("SyncWithObjectStorage: local table is not present in catalog", slog.String("table", table)) + d.logger.Debug("SyncWithObjectStorage: local table is in sync with remote", slog.String("table", table)) continue } if err := d.initLocalTable(table, remoteMeta.Version); err != nil { @@ -191,6 +197,9 @@ func (d *db) pullFromRemote(ctx context.Context, updateCatalog bool) error { // pushToRemote syncs the remote location with the local path for given table. // If oldVersion is specified, it is deleted after successful sync. func (d *db) pushToRemote(ctx context.Context, table string, oldMeta, meta *tableMeta) error { + if d.remote == nil { + return nil + } if meta.Type == "TABLE" { localPath := d.localTableDir(table, meta.Version) entries, err := os.ReadDir(localPath) @@ -249,9 +258,13 @@ func (d *db) pushToRemote(ctx context.Context, table string, oldMeta, meta *tabl // If table is specified, only that table is deleted. // If table and version is specified, only that version of the table is deleted. func (d *db) deleteRemote(ctx context.Context, table, version string) error { + if d.remote == nil { + return nil + } if table == "" && version != "" { return fmt.Errorf("table must be specified if version is specified") } + d.logger.Debug("deleting remote", slog.String("table", table), slog.String("version", version)) var prefix string if table != "" { if version != "" { diff --git a/runtime/queries/column_timeseries.go b/runtime/queries/column_timeseries.go index 7af2e3172ca..888cbf4709f 100644 --- a/runtime/queries/column_timeseries.go +++ b/runtime/queries/column_timeseries.go @@ -113,7 +113,7 @@ func (q *ColumnTimeseries) Resolve(ctx context.Context, rt *runtime.Runtime, ins timezone = q.TimeZone } - return olap.WithConnection(ctx, priority, false, false, func(ctx context.Context, ensuredCtx context.Context, _ *sql.Conn) error { + return olap.WithConnection(ctx, priority, false, func(ctx context.Context, ensuredCtx context.Context, _ *sql.Conn) error { tsAlias := tempName("_ts_") temporaryTableName := tempName("_timeseries_") diff --git a/runtime/queries/metricsview.go b/runtime/queries/metricsview.go index 7f96fe86703..f83044107af 100644 --- a/runtime/queries/metricsview.go +++ b/runtime/queries/metricsview.go @@ -648,7 +648,7 @@ func WriteParquet(meta []*runtimev1.MetricsViewColumn, data []*structpb.Struct, arrowField.Type = arrow.PrimitiveTypes.Float32 case runtimev1.Type_CODE_FLOAT64: arrowField.Type = arrow.PrimitiveTypes.Float64 - case runtimev1.Type_CODE_STRUCT, runtimev1.Type_CODE_UUID, runtimev1.Type_CODE_ARRAY, runtimev1.Type_CODE_STRING, runtimev1.Type_CODE_MAP: + case runtimev1.Type_CODE_STRUCT, runtimev1.Type_CODE_UUID, runtimev1.Type_CODE_ARRAY, runtimev1.Type_CODE_STRING, runtimev1.Type_CODE_MAP, runtimev1.Type_CODE_INTERVAL: arrowField.Type = arrow.BinaryTypes.String case runtimev1.Type_CODE_TIMESTAMP, runtimev1.Type_CODE_DATE, runtimev1.Type_CODE_TIME: arrowField.Type = arrow.FixedWidthTypes.Timestamp_us @@ -707,6 +707,15 @@ func WriteParquet(meta []*runtimev1.MetricsViewColumn, data []*structpb.Struct, } recordBuilder.Field(idx).(*array.StringBuilder).Append(string(bts)) + case runtimev1.Type_CODE_INTERVAL: + switch v := v.GetKind().(type) { + case *structpb.Value_NumberValue: + s := fmt.Sprintf("%f", v.NumberValue) + recordBuilder.Field(idx).(*array.StringBuilder).Append(s) + case *structpb.Value_StringValue: + recordBuilder.Field(idx).(*array.StringBuilder).Append(v.StringValue) + default: + } } } } diff --git a/runtime/queries/metricsview_aggregation_test.go b/runtime/queries/metricsview_aggregation_test.go index ca06462f542..275ba972407 100644 --- a/runtime/queries/metricsview_aggregation_test.go +++ b/runtime/queries/metricsview_aggregation_test.go @@ -657,7 +657,6 @@ func TestMetricsViewsAggregation_pivot_export_nolabel(t *testing.T) { { Name: "nolabel_pub", }, - { Name: "timestamp", TimeGrain: runtimev1.TimeGrain_TIME_GRAIN_MONTH, @@ -711,7 +710,6 @@ func TestMetricsViewsAggregation_pivot_export_nolabel_measure(t *testing.T) { { Name: "nolabel_pub", }, - { Name: "timestamp", TimeGrain: runtimev1.TimeGrain_TIME_GRAIN_MONTH, @@ -741,9 +739,9 @@ func TestMetricsViewsAggregation_pivot_export_nolabel_measure(t *testing.T) { require.Equal(t, 4, len(q.Result.Schema.Fields)) require.Equal(t, "nolabel_pub", q.Result.Schema.Fields[0].Name) - require.Equal(t, "2022-01-01 00:00:00_m1", q.Result.Schema.Fields[1].Name) - require.Equal(t, "2022-02-01 00:00:00_m1", q.Result.Schema.Fields[2].Name) - require.Equal(t, "2022-03-01 00:00:00_m1", q.Result.Schema.Fields[3].Name) + require.Equal(t, "2022-01-01 00:00:00_M1", q.Result.Schema.Fields[1].Name) + require.Equal(t, "2022-02-01 00:00:00_M1", q.Result.Schema.Fields[2].Name) + require.Equal(t, "2022-03-01 00:00:00_M1", q.Result.Schema.Fields[3].Name) i := 0 require.Equal(t, "Facebook", fieldsToString(rows[i], "nolabel_pub")) diff --git a/runtime/queries/table_columns.go b/runtime/queries/table_columns.go index bce8ea59bd5..e6009b89577 100644 --- a/runtime/queries/table_columns.go +++ b/runtime/queries/table_columns.go @@ -70,7 +70,7 @@ func (q *TableColumns) Resolve(ctx context.Context, rt *runtime.Runtime, instanc switch olap.Dialect() { case drivers.DialectDuckDB: - return olap.WithConnection(ctx, priority, false, false, func(ctx context.Context, ensuredCtx context.Context, _ *sql.Conn) error { + return olap.WithConnection(ctx, priority, false, func(ctx context.Context, ensuredCtx context.Context, _ *sql.Conn) error { // views return duplicate column names, so we need to create a temporary table temporaryTableName := tempName("profile_columns_") err = olap.Exec(ctx, &drivers.Statement{ diff --git a/runtime/reconcilers/alert_test.go b/runtime/reconcilers/alert_test.go index 42b1fe7facb..a9dc81db229 100644 --- a/runtime/reconcilers/alert_test.go +++ b/runtime/reconcilers/alert_test.go @@ -8,6 +8,7 @@ import ( runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" "github.com/rilldata/rill/runtime" + "github.com/rilldata/rill/runtime/compilers/rillv1" "github.com/rilldata/rill/runtime/pkg/email" "github.com/rilldata/rill/runtime/testruntime" "github.com/stretchr/testify/require" @@ -26,7 +27,6 @@ SELECT '2024-01-01T00:00:00Z'::TIMESTAMP as __time, 'Denmark' as country "/metrics/mv1.yaml": ` version: 1 type: metrics_view -display_name: mv1 model: bar timeseries: __time dimensions: @@ -162,7 +162,7 @@ SELECT '2024-01-04T00:00:00Z'::TIMESTAMP as __time, 'Denmark' as country Status: runtimev1.AssertionStatus_ASSERTION_STATUS_FAIL, FailRow: must(structpb.NewStruct(map[string]any{ "country": "Denmark", - "measure_0": "4", + "Measure 0": "4", })), }, SentNotifications: true, @@ -187,7 +187,6 @@ SELECT '2024-01-01T00:00:00Z'::TIMESTAMP as __time, 'Denmark' as country "/metrics/mv1.yaml": ` version: 1 type: metrics_view -display_name: mv1 model: bar timeseries: __time dimensions: @@ -321,7 +320,7 @@ SELECT '2024-01-04T00:00:00Z'::TIMESTAMP as __time, 'Denmark' as country Status: runtimev1.AssertionStatus_ASSERTION_STATUS_FAIL, FailRow: must(structpb.NewStruct(map[string]any{ "country": "Denmark", - "measure_0": "4", + "Measure 0": "4", })), }, SentNotifications: true, @@ -472,7 +471,6 @@ SELECT '2024-01-01T00:00:00Z'::TIMESTAMP as __time, 'Denmark' as country "/metrics/mv1.yaml": ` version: 1 type: metrics_view -display_name: mv1 model: bar timeseries: __time dimensions: @@ -580,7 +578,7 @@ func newMetricsView(name, model, timeDim string, measures, dimensions []string) Spec: &runtimev1.MetricsViewSpec{ Connector: "duckdb", Model: model, - DisplayName: name, + DisplayName: rillv1.ToDisplayName(name), TimeDimension: timeDim, Measures: make([]*runtimev1.MetricsViewSpec_MeasureV2, len(measures)), Dimensions: make([]*runtimev1.MetricsViewSpec_DimensionV2, len(dimensions)), @@ -590,7 +588,7 @@ func newMetricsView(name, model, timeDim string, measures, dimensions []string) Connector: "duckdb", Table: model, Model: model, - DisplayName: name, + DisplayName: rillv1.ToDisplayName(name), TimeDimension: timeDim, Measures: make([]*runtimev1.MetricsViewSpec_MeasureV2, len(measures)), Dimensions: make([]*runtimev1.MetricsViewSpec_DimensionV2, len(dimensions)), @@ -598,25 +596,30 @@ func newMetricsView(name, model, timeDim string, measures, dimensions []string) }, } for i, measure := range measures { + name := fmt.Sprintf("measure_%d", i) metrics.Spec.Measures[i] = &runtimev1.MetricsViewSpec_MeasureV2{ - Name: fmt.Sprintf("measure_%d", i), - Expression: measure, - Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE, + Name: name, + DisplayName: rillv1.ToDisplayName(name), + Expression: measure, + Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE, } metrics.State.ValidSpec.Measures[i] = &runtimev1.MetricsViewSpec_MeasureV2{ - Name: fmt.Sprintf("measure_%d", i), - Expression: measure, - Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE, + Name: name, + DisplayName: rillv1.ToDisplayName(name), + Expression: measure, + Type: runtimev1.MetricsViewSpec_MEASURE_TYPE_SIMPLE, } } for i, dimension := range dimensions { metrics.Spec.Dimensions[i] = &runtimev1.MetricsViewSpec_DimensionV2{ - Name: dimension, - Column: dimension, + Name: dimension, + DisplayName: rillv1.ToDisplayName(dimension), + Column: dimension, } metrics.State.ValidSpec.Dimensions[i] = &runtimev1.MetricsViewSpec_DimensionV2{ - Name: dimension, - Column: dimension, + Name: dimension, + DisplayName: rillv1.ToDisplayName(dimension), + Column: dimension, } } metricsRes := &runtimev1.Resource{ diff --git a/runtime/reconcilers/metrics_view.go b/runtime/reconcilers/metrics_view.go index 3eb5dd3f7e6..0fe95511828 100644 --- a/runtime/reconcilers/metrics_view.go +++ b/runtime/reconcilers/metrics_view.go @@ -7,6 +7,7 @@ import ( runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" "github.com/rilldata/rill/runtime" + "github.com/rilldata/rill/runtime/metricsview" ) func init() { @@ -85,7 +86,12 @@ func (r *MetricsViewReconciler) Reconcile(ctx context.Context, n *runtimev1.Reso // NOTE: Not checking refs for errors since they may still be valid even if they have errors. Instead, we just validate the metrics view against the table name. // Validate the metrics view and update ValidSpec - validateResult, validateErr := r.C.Runtime.ValidateMetricsView(ctx, r.C.InstanceID, mv.Spec) + e, err := metricsview.NewExecutor(ctx, r.C.Runtime, r.C.InstanceID, mv.Spec, runtime.ResolvedSecurityOpen, 0) + if err != nil { + return runtime.ReconcileResult{Err: fmt.Errorf("failed to create metrics view executor: %w", err)} + } + defer e.Close() + validateResult, validateErr := e.ValidateMetricsView(ctx) if validateErr == nil { validateErr = validateResult.Error() } diff --git a/runtime/reconcilers/source.go b/runtime/reconcilers/source.go index af519dd2d1d..1605074f8a3 100644 --- a/runtime/reconcilers/source.go +++ b/runtime/reconcilers/source.go @@ -80,8 +80,8 @@ func (r *SourceReconciler) Reconcile(ctx context.Context, n *runtimev1.ResourceN // Handle deletion if self.Meta.DeletedOn != nil { - olapDropTableIfExists(ctx, r.C, src.State.Connector, src.State.Table, false) - olapDropTableIfExists(ctx, r.C, src.State.Connector, r.stagingTableName(tableName), false) + olapDropTableIfExists(ctx, r.C, src.State.Connector, src.State.Table) + olapDropTableIfExists(ctx, r.C, src.State.Connector, r.stagingTableName(tableName)) return runtime.ReconcileResult{} } @@ -115,7 +115,7 @@ func (r *SourceReconciler) Reconcile(ctx context.Context, n *runtimev1.ResourceN if err != nil { if !src.Spec.StageChanges && src.State.Table != "" { // Remove previously ingested table - olapDropTableIfExists(ctx, r.C, src.State.Connector, src.State.Table, false) + olapDropTableIfExists(ctx, r.C, src.State.Connector, src.State.Table) src.State.Connector = "" src.State.Table = "" src.State.SpecHash = "" @@ -170,8 +170,8 @@ func (r *SourceReconciler) Reconcile(ctx context.Context, n *runtimev1.ResourceN // If the SinkConnector was changed, drop data in the old connector if src.State.Table != "" && src.State.Connector != src.Spec.SinkConnector { - olapDropTableIfExists(ctx, r.C, src.State.Connector, src.State.Table, false) - olapDropTableIfExists(ctx, r.C, src.State.Connector, r.stagingTableName(src.State.Table), false) + olapDropTableIfExists(ctx, r.C, src.State.Connector, src.State.Table) + olapDropTableIfExists(ctx, r.C, src.State.Connector, r.stagingTableName(src.State.Table)) } // Prepare for ingestion @@ -183,7 +183,7 @@ func (r *SourceReconciler) Reconcile(ctx context.Context, n *runtimev1.ResourceN // Should never happen, but if somehow the staging table was corrupted into a view, drop it if t, ok := olapTableInfo(ctx, r.C, connector, stagingTableName); ok && t.View { - olapDropTableIfExists(ctx, r.C, connector, stagingTableName, t.View) + olapDropTableIfExists(ctx, r.C, connector, stagingTableName) } // Execute ingestion @@ -226,11 +226,11 @@ func (r *SourceReconciler) Reconcile(ctx context.Context, n *runtimev1.ResourceN src.State.RefreshedOn = timestamppb.Now() } else if src.Spec.StageChanges { // Failed ingestion to staging table - olapDropTableIfExists(cleanupCtx, r.C, connector, stagingTableName, false) + olapDropTableIfExists(cleanupCtx, r.C, connector, stagingTableName) } else { // Failed ingestion to main table update = true - olapDropTableIfExists(cleanupCtx, r.C, connector, tableName, false) + olapDropTableIfExists(cleanupCtx, r.C, connector, tableName) src.State.Connector = "" src.State.Table = "" src.State.SpecHash = "" diff --git a/runtime/reconcilers/util.go b/runtime/reconcilers/util.go index 217ee739b9f..b967fdae5ed 100644 --- a/runtime/reconcilers/util.go +++ b/runtime/reconcilers/util.go @@ -117,7 +117,7 @@ func olapTableInfo(ctx context.Context, c *runtime.Controller, connector, table } // olapDropTableIfExists drops a table from an OLAP connector. -func olapDropTableIfExists(ctx context.Context, c *runtime.Controller, connector, table string, view bool) { +func olapDropTableIfExists(ctx context.Context, c *runtime.Controller, connector, table string) { if table == "" { return } @@ -128,7 +128,7 @@ func olapDropTableIfExists(ctx context.Context, c *runtime.Controller, connector } defer release() - _ = olap.DropTable(ctx, table, view) + _ = olap.DropTable(ctx, table) } // olapForceRenameTable renames a table or view from fromName to toName in the OLAP connector. @@ -159,7 +159,7 @@ func olapForceRenameTable(ctx context.Context, c *runtime.Controller, connector, // Renaming a table to the same name with different casing is not supported. Workaround by renaming to a temporary name first. if strings.EqualFold(fromName, toName) { tmpName := fmt.Sprintf("__rill_tmp_rename_%s_%s", typ, toName) - err = olap.RenameTable(ctx, fromName, tmpName, fromIsView) + err = olap.RenameTable(ctx, fromName, tmpName) if err != nil { return err } @@ -167,7 +167,7 @@ func olapForceRenameTable(ctx context.Context, c *runtime.Controller, connector, } // Do the rename - return olap.RenameTable(ctx, fromName, toName, fromIsView) + return olap.RenameTable(ctx, fromName, toName) } func resolveTemplatedProps(ctx context.Context, c *runtime.Controller, self compilerv1.TemplateResource, props map[string]any) (map[string]any, error) { diff --git a/runtime/registry_test.go b/runtime/registry_test.go index 6d097b1f6bb..7d3779f7ceb 100644 --- a/runtime/registry_test.go +++ b/runtime/registry_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/c2h5oh/datasize" + "github.com/marcboeker/go-duckdb" runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" "github.com/rilldata/rill/runtime/drivers" "github.com/rilldata/rill/runtime/pkg/activity" @@ -496,9 +497,9 @@ func TestRuntime_DeleteInstance_DropCorrupted(t *testing.T) { require.NoError(t, err) // Check we can't open it anymore - _, _, err = rt.OLAP(ctx, inst.ID, "") + conn, err := duckdb.NewConnector(dbpath, nil) require.Error(t, err) - require.FileExists(t, dbpath) + require.Nil(t, conn) // Delete instance and check it still drops the .db file for DuckDB err = rt.DeleteInstance(ctx, inst.ID) diff --git a/runtime/resolvers/glob.go b/runtime/resolvers/glob.go index a925fd1f6fe..34f5ae6f6dd 100644 --- a/runtime/resolvers/glob.go +++ b/runtime/resolvers/glob.go @@ -321,7 +321,7 @@ func (r *globResolver) transformResult(ctx context.Context, rows []map[string]an } defer os.Remove(jsonFile) - err = olap.WithConnection(ctx, 0, false, false, func(wrappedCtx context.Context, ensuredCtx context.Context, _ *databasesql.Conn) error { + err = olap.WithConnection(ctx, 0, false, func(wrappedCtx context.Context, ensuredCtx context.Context, _ *databasesql.Conn) error { // Load the JSON file into a temporary table err = olap.Exec(wrappedCtx, &drivers.Statement{ Query: fmt.Sprintf("CREATE TEMPORARY TABLE %s AS (SELECT * FROM read_ndjson_auto(%s))", olap.Dialect().EscapeIdentifier(r.tmpTableName), olap.Dialect().EscapeStringValue(jsonFile)), diff --git a/runtime/resolvers/resolvers_test.go b/runtime/resolvers/resolvers_test.go index d4dd267a1d9..03230fea308 100644 --- a/runtime/resolvers/resolvers_test.go +++ b/runtime/resolvers/resolvers_test.go @@ -6,6 +6,7 @@ import ( "encoding/csv" "encoding/json" "flag" + "fmt" "os" "path/filepath" "strings" @@ -16,7 +17,6 @@ import ( "github.com/rilldata/rill/runtime/pkg/fileutil" "github.com/rilldata/rill/runtime/testruntime" "github.com/stretchr/testify/require" - "golang.org/x/exp/maps" "gopkg.in/yaml.v3" ) @@ -103,8 +103,11 @@ func TestResolvers(t *testing.T) { for _, connector := range tf.Connectors { acquire, ok := testruntime.Connectors[connector] require.True(t, ok, "unknown connector %q", connector) - connectorVars := acquire(t) - maps.Copy(vars, connectorVars) + cfg := acquire(t) + for k, v := range cfg { + k = fmt.Sprintf("connector.%s.%s", connector, k) + vars[k] = v + } } // Create the test runtime instance. diff --git a/runtime/security.go b/runtime/security.go index f84576b346d..39950e32d24 100644 --- a/runtime/security.go +++ b/runtime/security.go @@ -149,16 +149,16 @@ func (r *ResolvedSecurity) QueryFilter() *runtimev1.Expression { // truth is the compass that guides us through the labyrinth of existence. var truth = true -// openAccess allows access to a resource. -var openAccess = &ResolvedSecurity{ +// ResolvedSecurityOpen is a ResolvedSecurity that allows access with no restrictions. +var ResolvedSecurityOpen = &ResolvedSecurity{ access: &truth, fieldAccess: nil, rowFilter: "", queryFilter: nil, } -// closedAccess denies access to a resource. -var closedAccess = &ResolvedSecurity{ +// ResolvedSecurityClosed is a ResolvedSecurity that denies access. +var ResolvedSecurityClosed = &ResolvedSecurity{ access: nil, fieldAccess: nil, rowFilter: "", @@ -194,7 +194,7 @@ func newSecurityEngine(cacheSize int, logger *zap.Logger) *securityEngine { func (p *securityEngine) resolveSecurity(instanceID, environment string, vars map[string]string, claims *SecurityClaims, r *runtimev1.Resource) (*ResolvedSecurity, error) { // If security checks are skipped, return open access if claims.SkipChecks { - return openAccess, nil + return ResolvedSecurityOpen, nil } // Combine rules with any contained in the resource itself @@ -209,7 +209,7 @@ func (p *securityEngine) resolveSecurity(instanceID, environment string, vars ma } } if !validRule { - return closedAccess, nil + return ResolvedSecurityClosed, nil } cacheKey, err := computeCacheKey(instanceID, environment, claims, r) diff --git a/runtime/server/generate_metrics_view.go b/runtime/server/generate_metrics_view.go index c2570be3784..7bb4053d8fd 100644 --- a/runtime/server/generate_metrics_view.go +++ b/runtime/server/generate_metrics_view.go @@ -13,6 +13,7 @@ import ( runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1" "github.com/rilldata/rill/runtime" "github.com/rilldata/rill/runtime/drivers" + "github.com/rilldata/rill/runtime/metricsview" "github.com/rilldata/rill/runtime/pkg/activity" "github.com/rilldata/rill/runtime/pkg/observability" "github.com/rilldata/rill/runtime/server/auth" @@ -252,7 +253,13 @@ func (s *Server) generateMetricsViewYAMLWithAI(ctx context.Context, instanceID, FormatPreset: measure.FormatPreset, }) } - validateResult, err := s.runtime.ValidateMetricsView(ctx, instanceID, spec) + + e, err := metricsview.NewExecutor(ctx, s.runtime, instanceID, spec, runtime.ResolvedSecurityOpen, 0) + if err != nil { + return nil, err + } + defer e.Close() + validateResult, err := e.ValidateMetricsView(ctx) if err != nil { return nil, err } diff --git a/runtime/testruntime/connectors.go b/runtime/testruntime/connectors.go index c9eb6d20ea6..8d907ada28d 100644 --- a/runtime/testruntime/connectors.go +++ b/runtime/testruntime/connectors.go @@ -13,8 +13,21 @@ import ( "github.com/testcontainers/testcontainers-go/modules/clickhouse" ) +// AcquireConnector acquires a test connector by name. +// For a list of available connectors, see the Connectors map below. +func AcquireConnector(t TestingT, name string) map[string]any { + acquire, ok := Connectors[name] + require.True(t, ok, "connector not found") + vars := acquire(t) + cfg := make(map[string]any, len(vars)) + for k, v := range vars { + cfg[k] = v + } + return cfg +} + // ConnectorAcquireFunc is a function that acquires a connector for a test. -// It should return a map of variables to add to the test runtime instance. +// It should return a map of config keys suitable for passing to drivers.Open. type ConnectorAcquireFunc func(t TestingT) (vars map[string]string) // Connectors is a map of available connectors for use in tests. @@ -61,7 +74,7 @@ var Connectors = map[string]ConnectorAcquireFunc{ require.NoError(t, err) dsn := fmt.Sprintf("clickhouse://clickhouse:clickhouse@%v:%v", host, port.Port()) - return map[string]string{"connector.clickhouse.dsn": dsn} + return map[string]string{"dsn": dsn} }, // druid connects to a real Druid cluster using the connection string in RILL_RUNTIME_DRUID_TEST_DSN. @@ -77,6 +90,6 @@ var Connectors = map[string]ConnectorAcquireFunc{ dsn := os.Getenv("RILL_RUNTIME_DRUID_TEST_DSN") require.NotEmpty(t, dsn, "Druid test DSN not configured") - return map[string]string{"connector.druid.dsn": dsn} + return map[string]string{"dsn": dsn} }, } diff --git a/runtime/testruntime/testdata/ad_bids/apis/mv_sql_policy_api.yaml b/runtime/testruntime/testdata/ad_bids/apis/mv_sql_policy_api.yaml index ad74d8a022b..86352b648d5 100644 --- a/runtime/testruntime/testdata/ad_bids/apis/mv_sql_policy_api.yaml +++ b/runtime/testruntime/testdata/ad_bids/apis/mv_sql_policy_api.yaml @@ -2,7 +2,7 @@ kind : api metrics_sql: | select - publisher, + publisher_dim, domain, "total impressions", "total volume" diff --git a/runtime/testruntime/testdata/ad_bids/dashboards/ad_bids_metrics.yaml b/runtime/testruntime/testdata/ad_bids/dashboards/ad_bids_metrics.yaml index b0734816a3f..010406168bb 100644 --- a/runtime/testruntime/testdata/ad_bids/dashboards/ad_bids_metrics.yaml +++ b/runtime/testruntime/testdata/ad_bids/dashboards/ad_bids_metrics.yaml @@ -16,6 +16,7 @@ dimensions: property: domain description: "" - name: nolabel_pub + display_name: nolabel_pub property: publisher - name: space_label display_name: Space Label diff --git a/runtime/testruntime/testdata/ad_bids/dashboards/ad_bids_mini_metrics_with_policy.yaml b/runtime/testruntime/testdata/ad_bids/dashboards/ad_bids_mini_metrics_with_policy.yaml index 6ea0a916e9b..724f2b1fa2b 100644 --- a/runtime/testruntime/testdata/ad_bids/dashboards/ad_bids_mini_metrics_with_policy.yaml +++ b/runtime/testruntime/testdata/ad_bids/dashboards/ad_bids_mini_metrics_with_policy.yaml @@ -6,7 +6,7 @@ timeseries: timestamp smallest_time_grain: "" dimensions: - - name: publisher + - name: publisher_dim display_name: Publisher expression: upper(publisher) description: "" diff --git a/runtime/testruntime/testdata/ad_bids_clickhouse/dashboards/ad_bids_mini_metrics_with_policy.yaml b/runtime/testruntime/testdata/ad_bids_clickhouse/dashboards/ad_bids_mini_metrics_with_policy.yaml index c730cbad7b7..05d9741f6f3 100644 --- a/runtime/testruntime/testdata/ad_bids_clickhouse/dashboards/ad_bids_mini_metrics_with_policy.yaml +++ b/runtime/testruntime/testdata/ad_bids_clickhouse/dashboards/ad_bids_mini_metrics_with_policy.yaml @@ -1,6 +1,5 @@ model: ad_bids_mini display_name: Ad bids -display_name: "" timeseries: timestamp diff --git a/web-admin/src/features/bookmarks/BookmarksDropdownMenuContent.svelte b/web-admin/src/features/bookmarks/BookmarksDropdownMenuContent.svelte index 660e7e26ccd..e0cfbe739a5 100644 --- a/web-admin/src/features/bookmarks/BookmarksDropdownMenuContent.svelte +++ b/web-admin/src/features/bookmarks/BookmarksDropdownMenuContent.svelte @@ -27,6 +27,7 @@ import { getDefaultExplorePreset } from "@rilldata/web-common/features/dashboards/url-state/getDefaultExplorePreset"; import { ResourceKind } from "@rilldata/web-common/features/entity-management/resource-selectors"; import { useExploreValidSpec } from "@rilldata/web-common/features/explores/selectors"; + import { createQueryServiceMetricsViewSchema } from "@rilldata/web-common/runtime-client"; import { runtime } from "@rilldata/web-common/runtime-client/runtime-store"; import { BookmarkPlusIcon } from "lucide-svelte"; import { createEventDispatcher } from "svelte"; @@ -51,6 +52,10 @@ exploreSpec, $metricsViewTimeRange.data, ); + $: schemaResp = createQueryServiceMetricsViewSchema( + $runtime.instanceId, + metricsViewName, + ); $: projectIdResp = useProjectId(organization, project); const userResp = createAdminServiceGetCurrentUser(); @@ -72,7 +77,7 @@ $bookamrksResp.data?.bookmarks ?? [], metricsViewSpec, exploreSpec, - {}, + $schemaResp.data?.schema, $exploreState, defaultExplorePreset, ); diff --git a/web-admin/src/features/bookmarks/selectors.ts b/web-admin/src/features/bookmarks/selectors.ts index 58e764722fb..026c7627edc 100644 --- a/web-admin/src/features/bookmarks/selectors.ts +++ b/web-admin/src/features/bookmarks/selectors.ts @@ -67,6 +67,7 @@ export function categorizeBookmarks( personal: [], shared: [], }; + if (!exploreState) return bookmarks; bookmarkResp?.forEach((bookmarkResource) => { const bookmark = parseBookmark( bookmarkResource, @@ -155,6 +156,7 @@ export function convertBookmarkToUrlSearchParams( ...exploreStateFromBookmark, } as MetricsExplorerEntity, exploreSpec, + undefined, // TODO defaultExplorePreset, ); } diff --git a/web-admin/src/features/dashboards/query-mappers/mapQueryToDashboard.ts b/web-admin/src/features/dashboards/query-mappers/mapQueryToDashboard.ts index 4a65ce3d724..628e6e9e5b6 100644 --- a/web-admin/src/features/dashboards/query-mappers/mapQueryToDashboard.ts +++ b/web-admin/src/features/dashboards/query-mappers/mapQueryToDashboard.ts @@ -5,8 +5,10 @@ import type { QueryRequests, } from "@rilldata/web-admin/features/dashboards/query-mappers/types"; import type { CompoundQueryResult } from "@rilldata/web-common/features/compound-query-result"; -import { getDefaultExploreState } from "@rilldata/web-common/features/dashboards/stores/dashboard-store-defaults"; +import { getFullInitExploreState } from "@rilldata/web-common/features/dashboards/stores/dashboard-store-defaults"; import type { MetricsExplorerEntity } from "@rilldata/web-common/features/dashboards/stores/metrics-explorer-entity"; +import { convertPresetToExploreState } from "@rilldata/web-common/features/dashboards/url-state/convertPresetToExploreState"; +import { getDefaultExplorePreset } from "@rilldata/web-common/features/dashboards/url-state/getDefaultExplorePreset"; import { initLocalUserPreferenceStore } from "@rilldata/web-common/features/dashboards/user-preferences"; import { useExploreValidSpec } from "@rilldata/web-common/features/explores/selectors"; import { queryClient } from "@rilldata/web-common/lib/svelte-query/globalQueryClient"; @@ -117,12 +119,19 @@ export function mapQueryToDashboard( const { metricsView, explore } = validSpecResp.data; initLocalUserPreferenceStore(metricsViewName); - const defaultExploreState = getDefaultExploreState( - metricsViewName, - metricsView, - explore, + const defaultExplorePreset = getDefaultExplorePreset( + validSpecResp.data.explore, timeRangeSummary.data, ); + const { partialExploreState } = convertPresetToExploreState( + validSpecResp.data.metricsView, + validSpecResp.data.explore, + defaultExplorePreset, + ); + const defaultExploreState = getFullInitExploreState( + metricsViewName, + partialExploreState, + ); getDashboardState({ queryClient, instanceId, diff --git a/web-admin/src/features/dashboards/query-mappers/utils.ts b/web-admin/src/features/dashboards/query-mappers/utils.ts index 9168f2bb090..f2c14056076 100644 --- a/web-admin/src/features/dashboards/query-mappers/utils.ts +++ b/web-admin/src/features/dashboards/query-mappers/utils.ts @@ -1,5 +1,6 @@ import { createInExpression } from "@rilldata/web-common/features/dashboards/stores/filter-utils"; import type { MetricsExplorerEntity } from "@rilldata/web-common/features/dashboards/stores/metrics-explorer-entity"; +import { getTimeControlState } from "@rilldata/web-common/features/dashboards/time-controls/time-control-store"; import { PreviousCompleteRangeMap } from "@rilldata/web-common/features/dashboards/time-controls/time-range-mappers"; import { convertExploreStateToURLSearchParams } from "@rilldata/web-common/features/dashboards/url-state/convertExploreStateToURLSearchParams"; import { getDefaultExplorePreset } from "@rilldata/web-common/features/dashboards/url-state/getDefaultExplorePreset"; @@ -182,6 +183,7 @@ export async function getExplorePageUrl( const url = new URL(`${curPageUrl.protocol}//${curPageUrl.host}`); url.pathname = `/${organization}/${project}/explore/${exploreName}`; + const metricsViewSpec = metricsView?.metricsView?.state?.validSpec ?? {}; const exploreSpec = explore?.explore?.state?.validSpec ?? {}; const metricsViewName = exploreSpec.metricsView; @@ -206,6 +208,12 @@ export async function getExplorePageUrl( url.search = convertExploreStateToURLSearchParams( exploreState, exploreSpec, + getTimeControlState( + metricsViewSpec, + exploreSpec, + fullTimeRange?.timeRangeSummary, + exploreState, + ), getDefaultExplorePreset(exploreSpec, fullTimeRange), ); return url.toString(); diff --git a/web-admin/src/features/navigation/TopNavigationBar.svelte b/web-admin/src/features/navigation/TopNavigationBar.svelte index e6f4813aa25..aef178babb4 100644 --- a/web-admin/src/features/navigation/TopNavigationBar.svelte +++ b/web-admin/src/features/navigation/TopNavigationBar.svelte @@ -110,7 +110,8 @@ ); $: projectPaths = projects.reduce( - (map, { name }) => map.set(name.toLowerCase(), { label: name }), + (map, { name }) => + map.set(name.toLowerCase(), { label: name, preloadData: false }), new Map(), ); diff --git a/web-admin/src/features/projects/environment-variables/ActionsCell.svelte b/web-admin/src/features/projects/environment-variables/ActionsCell.svelte index 2a2cf6b5ddb..c875613b19f 100644 --- a/web-admin/src/features/projects/environment-variables/ActionsCell.svelte +++ b/web-admin/src/features/projects/environment-variables/ActionsCell.svelte @@ -24,7 +24,7 @@ - + { diff --git a/web-admin/src/features/projects/environment-variables/AddDialog.svelte b/web-admin/src/features/projects/environment-variables/AddDialog.svelte index ce5692c7466..c413a2268ae 100644 --- a/web-admin/src/features/projects/environment-variables/AddDialog.svelte +++ b/web-admin/src/features/projects/environment-variables/AddDialog.svelte @@ -27,6 +27,7 @@ import IconButton from "@rilldata/web-common/components/button/IconButton.svelte"; import { Trash2Icon, UploadIcon } from "lucide-svelte"; import { getCurrentEnvironment, isDuplicateKey } from "./utils"; + import { parse as parseDotenv } from "dotenv"; export let open = false; export let variableNames: VariableNames = []; @@ -36,15 +37,16 @@ let isDevelopment = true; let isProduction = true; let fileInput: HTMLInputElement; + let showEnvironmentError = false; $: organization = $page.params.organization; $: project = $page.params.project; - $: isEnvironmentSelected = isDevelopment || isProduction; $: hasExistingKeys = Object.values(inputErrors).some((error) => error); $: hasNewChanges = $form.variables.some( (variable) => variable.key !== "" || variable.value !== "", ); + $: hasNoEnvironment = showEnvironmentError && !isDevelopment && !isProduction; const queryClient = useQueryClient(); const updateProjectVariables = createAdminServiceUpdateProjectVariables(); @@ -72,11 +74,11 @@ }), ); - const { form, enhance, submit, submitting, errors, allErrors, reset } = - superForm(defaults(initialValues, schema), { + const { form, enhance, submit, submitting, errors, allErrors } = superForm( + defaults(initialValues, schema), + { SPA: true, validators: schema, - // See: https://superforms.rocks/concepts/nested-data dataType: "json", async onUpdate({ form }) { if (!form.valid) return; @@ -85,11 +87,9 @@ // Check for duplicates before proceeding const duplicates = checkForExistingKeys(); if (duplicates > 0) { - // Early return without resetting the form return; } - // Only filter and process if there are no duplicates const filteredVariables = values.variables.filter( ({ key }) => key !== "", ); @@ -106,7 +106,8 @@ console.error(error); } }, - }); + }, + ); async function handleUpdateProjectVariables( flatVariables: AdminServiceUpdateProjectVariablesBodyVariables, @@ -144,6 +145,8 @@ function handleKeyChange(index: number, event: Event) { const target = event.target as HTMLInputElement; $form.variables[index].key = target.value; + delete inputErrors[index]; + isKeyAlreadyExists = false; } function handleValueChange(index: number, event: Event) { @@ -157,31 +160,37 @@ } function handleReset() { - reset(); + $form = initialValues; isDevelopment = true; isProduction = true; inputErrors = {}; isKeyAlreadyExists = false; + showEnvironmentError = false; } function checkForExistingKeys() { inputErrors = {}; isKeyAlreadyExists = false; - const existingKeys = $form.variables.map((variable) => { - return { - environment: getCurrentEnvironment(isDevelopment, isProduction), - name: variable.key, - }; - }); + const existingKeys = $form.variables + .filter((variable) => variable.key.trim() !== "") + .map((variable) => { + return { + environment: getCurrentEnvironment(isDevelopment, isProduction), + name: variable.key, + }; + }); let duplicateCount = 0; - existingKeys.forEach((key, idx) => { + existingKeys.forEach((key, _idx) => { const variableEnvironment = key.environment; const variableKey = key.name; if (isDuplicateKey(variableEnvironment, variableKey, variableNames)) { - inputErrors[idx] = true; + const originalIndex = $form.variables.findIndex( + (v) => v.key === variableKey, + ); + inputErrors[originalIndex] = true; isKeyAlreadyExists = true; duplicateCount++; } @@ -190,55 +199,62 @@ return duplicateCount; } - function handleFileUpload(event) { - const file = event.target.files[0]; + function handleFileUpload(event: Event) { + const file = (event.target as HTMLInputElement).files?.[0]; if (file) { const reader = new FileReader(); - reader.onload = (e) => { - const contents = e.target.result; - parseFile(contents); - checkForExistingKeys(); + reader.onload = (e: ProgressEvent) => { + const contents = e.target?.result; + if (typeof contents === "string") { + parseFile(contents); + checkForExistingKeys(); + } }; reader.readAsText(file); } } - function parseFile(contents) { - const lines = contents.split("\n"); + function parseFile(contents: string) { + const parsedVariables = parseDotenv(contents); - lines.forEach((line) => { - // Trim the line and check if it starts with '#' - const trimmedLine = line.trim(); - if (trimmedLine.startsWith("#")) { - return; // Skip comment lines - } + for (const [key, value] of Object.entries(parsedVariables)) { + const filteredVariables = $form.variables.filter( + (variable) => + variable.key.trim() !== "" || variable.value.trim() !== "", + ); - const [key, value] = trimmedLine.split("="); - if (key && value) { - if (key.trim() && value.trim()) { - const filteredVariables = $form.variables.filter( - (variable) => - variable.key.trim() !== "" || variable.value.trim() !== "", - ); - - $form.variables = [ - ...filteredVariables, - { key: key.trim(), value: value.trim() }, - ]; - } - } - }); + $form.variables = [...filteredVariables, { key, value }]; + } } function getKeyFromError(error: { path: string; messages: string[] }) { return error.path.split("[")[1].split("]")[0]; } + + function handleEnvironmentChange() { + showEnvironmentError = true; + checkForExistingKeys(); + } + + $: isSubmitDisabled = + $submitting || + hasExistingKeys || + !hasNewChanges || + hasNoEnvironment || + Object.values($form.variables).every((v) => !v.key.trim()); handleReset()} - onOutsideClick={() => handleReset()} + onOpenChange={(isOpen) => { + if (!isOpen) { + handleReset(); + } + }} + onOutsideClick={() => { + open = false; + handleReset(); + }} > @@ -279,18 +295,25 @@
Environment
+ {#if hasNoEnvironment} +
+

+ You must select at least one environment +

+
+ {/if}
Variables
@@ -378,17 +401,18 @@ on:click={() => { open = false; handleReset(); - }}>Cancel + Cancel + + Create +
diff --git a/web-admin/src/features/projects/environment-variables/EditDialog.svelte b/web-admin/src/features/projects/environment-variables/EditDialog.svelte index f6c1f662021..50ccf108f1a 100644 --- a/web-admin/src/features/projects/environment-variables/EditDialog.svelte +++ b/web-admin/src/features/projects/environment-variables/EditDialog.svelte @@ -23,11 +23,9 @@ import { object, string } from "yup"; import { EnvironmentType, type VariableNames } from "./types"; import Input from "@rilldata/web-common/components/forms/Input.svelte"; - import { - getCurrentEnvironment, - getEnvironmentType, - isDuplicateKey, - } from "./utils"; + import { getCurrentEnvironment, isDuplicateKey } from "./utils"; + import { onMount } from "svelte"; + import { debounce } from "lodash"; export let open = false; export let id: string; @@ -40,21 +38,22 @@ isDevelopment: boolean; isProduction: boolean; }; - let isDevelopment: boolean; - let isProduction: boolean; + let isDevelopment = false; + let isProduction = false; let isKeyAlreadyExists = false; let inputErrors: { [key: number]: boolean } = {}; + let showEnvironmentError = false; $: organization = $page.params.organization; $: project = $page.params.project; - $: isEnvironmentSelected = isDevelopment || isProduction; $: hasNewChanges = $form.key !== initialValues.key || $form.value !== initialValues.value || initialEnvironment?.isDevelopment !== isDevelopment || initialEnvironment?.isProduction !== isProduction; $: hasExistingKeys = Object.values(inputErrors).some((error) => error); + $: hasNoEnvironment = showEnvironmentError && !isDevelopment && !isProduction; const queryClient = useQueryClient(); const updateProjectVariables = createAdminServiceUpdateProjectVariables(); @@ -79,40 +78,32 @@ }), ); - const { - form, - enhance, - formId, - submit, - errors, - allErrors, - submitting, - reset, - } = superForm(defaults(initialValues, schema), { - // See: https://superforms.rocks/concepts/multiple-forms#setting-id-on-the-client - id: id, - SPA: true, - validators: schema, - async onUpdate({ form }) { - if (!form.valid) return; - const values = form.data; - - checkForExistingKeys(); - if (isKeyAlreadyExists) return; - - const flatVariable = { - [values.key]: values.value, - }; - - try { - await handleUpdateProjectVariables(flatVariable); - open = false; - handleReset(); - } catch (error) { - console.error(error); - } - }, - }); + const { form, enhance, formId, submit, errors, allErrors, submitting } = + superForm(defaults(initialValues, schema), { + id: id, + SPA: true, + validators: schema, + resetForm: false, + async onUpdate({ form }) { + if (!form.valid) return; + const values = form.data; + + checkForExistingKeys(); + if (isKeyAlreadyExists) return; + + const flatVariable = { + [values.key]: values.value, + }; + + try { + await handleUpdateProjectVariables(flatVariable); + open = false; + handleReset(); + } catch (error) { + console.error(error); + } + }, + }); async function handleUpdateProjectVariables( flatVariable: AdminServiceUpdateProjectVariablesBodyVariables, @@ -200,28 +191,30 @@ } function handleReset() { - reset(); + $form.environment = initialValues.environment; + $form.key = initialValues.key; + $form.value = initialValues.value; isDevelopment = false; isProduction = false; inputErrors = {}; isKeyAlreadyExists = false; + showEnvironmentError = false; } function checkForExistingKeys() { inputErrors = {}; isKeyAlreadyExists = false; - const existingKey = { - environment: getEnvironmentType( - getCurrentEnvironment(isDevelopment, isProduction), - ), - name: $form.key, - }; - - const variableEnvironment = existingKey.environment; - const variableKey = existingKey.name; + const newEnvironment = getCurrentEnvironment(isDevelopment, isProduction); - if (isDuplicateKey(variableEnvironment, variableKey, variableNames)) { + if ( + isDuplicateKey( + newEnvironment, + $form.key, + variableNames, + initialValues.key, + ) + ) { inputErrors[0] = true; isKeyAlreadyExists = true; } @@ -240,19 +233,37 @@ isDevelopment = true; isProduction = true; } - } - function handleDialogOpen() { - setInitialCheckboxState(); initialEnvironment = { isDevelopment, isProduction, }; } + function handleDialogOpen() { + handleReset(); + setInitialCheckboxState(); + } + + function handleEnvironmentChange() { + showEnvironmentError = true; + } + + onMount(() => { + handleDialogOpen(); + }); + $: if (open) { handleDialogOpen(); } + + const debouncedCheckForExistingKeys = debounce(() => { + checkForExistingKeys(); + }, 500); + + $: if ($form.key) { + debouncedCheckForExistingKeys(); + } + {#if hasNoEnvironment} +
+

+ You must select at least one environment +

+
+ {/if}
Variable
@@ -349,11 +369,12 @@ form={$formId} disabled={$submitting || !hasNewChanges || - !isEnvironmentSelected || hasExistingKeys || - $allErrors.length > 0} - submitForm>Edit + Edit +
diff --git a/web-admin/src/features/projects/environment-variables/EnvironmentVariablesTable.svelte b/web-admin/src/features/projects/environment-variables/EnvironmentVariablesTable.svelte index e896e6d9222..34bfbcb1996 100644 --- a/web-admin/src/features/projects/environment-variables/EnvironmentVariablesTable.svelte +++ b/web-admin/src/features/projects/environment-variables/EnvironmentVariablesTable.svelte @@ -42,7 +42,6 @@ { header: "Activity", accessorFn: (row) => row.createdOn, - enableSorting: false, cell: ({ row }) => { return flexRender(ActivityCell, { updatedOn: row.original.updatedOn, diff --git a/web-admin/src/features/projects/environment-variables/ValueCell.svelte b/web-admin/src/features/projects/environment-variables/ValueCell.svelte index b37f8cf0284..3c9a1267e4a 100644 --- a/web-admin/src/features/projects/environment-variables/ValueCell.svelte +++ b/web-admin/src/features/projects/environment-variables/ValueCell.svelte @@ -1,8 +1,10 @@ diff --git a/web-admin/src/features/projects/status/ProjectParseErrors.svelte b/web-admin/src/features/projects/status/ProjectParseErrors.svelte index eb1ec2ad4e8..ff32404d884 100644 --- a/web-admin/src/features/projects/status/ProjectParseErrors.svelte +++ b/web-admin/src/features/projects/status/ProjectParseErrors.svelte @@ -8,10 +8,21 @@ import { createRuntimeServiceGetResource } from "@rilldata/web-common/runtime-client"; import { runtime } from "@rilldata/web-common/runtime-client/runtime-store"; - $: projectParserQuery = createRuntimeServiceGetResource($runtime.instanceId, { - "name.kind": ResourceKind.ProjectParser, - "name.name": SingletonProjectParserName, - }); + $: ({ instanceId } = $runtime); + + $: projectParserQuery = createRuntimeServiceGetResource( + instanceId, + { + "name.kind": ResourceKind.ProjectParser, + "name.name": SingletonProjectParserName, + }, + { + query: { + refetchOnMount: true, + refetchOnWindowFocus: true, + }, + }, + ); $: ({ isLoading, isSuccess, data, error } = $projectParserQuery); $: parseErrors = data?.resource?.projectParser.state.parseErrors; @@ -30,7 +41,7 @@ {:else if isSuccess} {#if parseErrors && parseErrors.length > 0}