Add ClickHouse server and provisioner in rill devtool

rilldata · Jan 3, 2025 · 735a63b · 735a63b
1 parent 4a4f30b
commit 735a63b
Show file tree

Hide file tree

Showing 5 changed files with 75 additions and 10 deletions.
diff --git a/cli/cmd/devtool/data/clickhouse-config.xml b/cli/cmd/devtool/data/clickhouse-config.xml
@@ -0,0 +1,13 @@
+<clickhouse replace="true">
+    <logger>
+        <level>information</level>
+        <console>true</console>
+    </logger>
+    <display_name>rill devtool clickhouse</display_name>
+    <tcp_port>9000</tcp_port>
+    <http_port>8123</http_port>
+    <timezone>UTC</timezone>
+    <access_control_improvements>
+        <select_from_information_schema_requires_grant>true</select_from_information_schema_requires_grant>
+    </access_control_improvements>
+</clickhouse>
diff --git a/cli/cmd/devtool/data/clickhouse-users.xml b/cli/cmd/devtool/data/clickhouse-users.xml
@@ -0,0 +1,37 @@
+<clickhouse replace="true">
+    <profiles>
+        <default>
+            <max_memory_usage>10000000000</max_memory_usage>
+            <use_uncompressed_cache>0</use_uncompressed_cache>
+            <load_balancing>in_order</load_balancing>
+            <log_queries>1</log_queries>
+            <date_time_input_format>best_effort</date_time_input_format>
+        </default>
+    </profiles>
+    <quotas>
+        <default>
+            <interval>
+                <duration>3600</duration>
+                <queries>0</queries>
+                <errors>0</errors>
+                <result_rows>0</result_rows>
+                <read_rows>0</read_rows>
+                <execution_time>0</execution_time>
+            </interval>
+        </default>
+    </quotas>
+    <users>
+        <default>
+            <password>default</password>
+            <profile>default</profile>
+            <quota>default</quota>
+            <networks>
+                <ip>::/0</ip>
+            </networks>
+            <access_management>1</access_management>
+            <named_collection_control>1</named_collection_control>
+            <show_named_collections>1</show_named_collections>
+            <show_named_collections_secrets>1</show_named_collections_secrets>
+        </default>
+    </users>
+</clickhouse>
diff --git a/cli/cmd/devtool/data/cloud-deps.docker-compose.yml b/cli/cmd/devtool/data/cloud-deps.docker-compose.yml
@@ -64,3 +64,12 @@ services:
     command: listen --forward-to http://host.docker.internal:8080/payment/webhook --config /etc/stripe-config.toml
     volumes:
       - ../../../../${RILL_DEVTOOL_STATE_DIRECTORY}/stripe-config.toml:/etc/stripe-config.toml
+  clickhouse:
+    image: 'clickhouse/clickhouse-server:24.11.1.2557'
+    volumes:
+      - ./clickhouse-config.xml:/etc/clickhouse-server/config.d/config.xml
+      - ./clickhouse-users.xml:/etc/clickhouse-server/users.d/users.xml
+      - ../../../../${RILL_DEVTOOL_STATE_DIRECTORY}/clickhouse:/var/lib/clickhouse
+    ports:
+      - '9000:9000' # Native port
+      - '8123:8123' # HTTP port
diff --git a/cli/cmd/devtool/start.go b/cli/cmd/devtool/start.go
@@ -252,12 +252,11 @@ func (s cloud) start(ctx context.Context, ch *cmdutil.Helper, verbose, reset, re
 
 	g, ctx := errgroup.WithContext(ctx)
 
-	stateDir := lookupDotenv("RILL_DEVTOOL_STATE_DIRECTORY")
-	err = os.MkdirAll(stateDir, os.ModePerm)
+	err = os.MkdirAll(stateDirectory(), os.ModePerm)
 	if err != nil {
-		return fmt.Errorf("failed to create state dir %q: %w", stateDir, err)
+		return fmt.Errorf("failed to create state dir %q: %w", stateDirectory(), err)
 	}
-	logInfo.Printf("State dir set to %s\n", stateDir)
+	logInfo.Printf("State directory is %q\n", stateDirectory())
 
 	if services.deps {
 		g.Go(func() error { return s.runDeps(ctx, verbose) })
@@ -385,8 +384,8 @@ func (s cloud) resetState(ctx context.Context) (err error) {
 		}
 	}()
 
-	stateDir := lookupDotenv("RILL_DEVTOOL_STATE_DIRECTORY")
-	_ = os.RemoveAll(stateDir)
+	_ = os.RemoveAll(stateDirectory())
+
 	return newCmd(ctx, "docker", "compose", "--env-file", ".env", "-f", "cli/cmd/devtool/data/cloud-deps.docker-compose.yml", "down", "--volumes").Run()
 }
 
@@ -724,8 +723,7 @@ func (s local) awaitUI(ctx context.Context) error {
 
 func prepareStripeConfig() error {
 	templateFile := "cli/cmd/devtool/data/stripe-config.template"
-	stateDir := lookupDotenv("RILL_DEVTOOL_STATE_DIRECTORY")
-	outputFile := filepath.Join(stateDir, "stripe-config.toml")
+	outputFile := filepath.Join(stateDirectory(), "stripe-config.toml")
 
 	apiKey := lookupDotenv("RILL_DEVTOOL_STRIPE_CLI_API_KEY")
 	if apiKey == "" {
@@ -785,3 +783,13 @@ func lookupDotenv(key string) string {
 	}
 	return env[key]
 }
+
+// stateDirectory returns the directory where the devtool's state is stored.
+// Deleting this directory will reset the state of the local development environment.
+func stateDirectory() string {
+	dir := lookupDotenv("RILL_DEVTOOL_STATE_DIRECTORY")
+	if dir == "" {
+		dir = "dev-cloud-state"
+	}
+	return dir
+}
diff --git a/runtime/testruntime/connectors.go b/runtime/testruntime/connectors.go
@@ -3,7 +3,6 @@ package testruntime
 import (
 	"context"
 	"fmt"
-	"log"
 	"os"
 	"path/filepath"
 	goruntime "runtime"
@@ -65,7 +64,6 @@ var Connectors = map[string]ConnectorAcquireFunc{
 		require.NoError(t, err)
 
 		t.Cleanup(func() {
-			log.Printf("TERMINATING")
 			err := clickHouseContainer.Terminate(ctx)
 			require.NoError(t, err)
 		})