diff --git a/CHANGELOG.md b/CHANGELOG.md index f32eafa..5dd83b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 1.2.1 +### New features +* Use HoneySQL2 in the driver + # 1.2.0 ### New features diff --git a/resources/metabase-plugin.yaml b/resources/metabase-plugin.yaml index 01e2fcd..e82bef3 100644 --- a/resources/metabase-plugin.yaml +++ b/resources/metabase-plugin.yaml @@ -1,6 +1,6 @@ info: name: Metabase ClickHouse Driver - version: 1.2.0 + version: 1.2.1 description: Allows Metabase to connect to ClickHouse databases. contact-info: name: ClickHouse diff --git a/src/metabase/driver/clickhouse.clj b/src/metabase/driver/clickhouse.clj index a77fad4..e8619fe 100644 --- a/src/metabase/driver/clickhouse.clj +++ b/src/metabase/driver/clickhouse.clj @@ -3,32 +3,17 @@ #_{:clj-kondo/ignore [:unsorted-required-namespaces]} (:require [clojure.java.jdbc :as jdbc] [clojure.string :as str] - [honeysql [core :as hsql] [format :as hformat]] - [java-time :as t] [metabase [config :as config] [driver :as driver] [util :as u]] + [metabase.driver.clickhouse-qp] [metabase.driver.clickhouse-nippy] [metabase.driver.ddl.interface :as ddl.i] [metabase.driver.sql :as driver.sql] [metabase.driver.sql-jdbc [common :as sql-jdbc.common] - [connection :as sql-jdbc.conn] [execute :as sql-jdbc.execute] - [sync :as sql-jdbc.sync]] - [metabase.driver.sql.query-processor :as sql.qp :refer [add-interval-honeysql-form]] - [metabase.driver.sql.util.unprepare :as unprepare] - [metabase.mbql.schema :as mbql.s] - [metabase.mbql.util :as mbql.u] - [metabase.util.date-2 :as u.date] - [metabase.util.honeysql-extensions :as hx] - [schema.core :as s]) - (:import [com.clickhouse.data.value ClickHouseArrayValue] - [java.sql ResultSet ResultSetMetaData Types] - [java.time - LocalDate - LocalDateTime - LocalTime - OffsetDateTime - OffsetTime - ZonedDateTime] - java.util.Arrays)) + [connection :as sql-jdbc.conn] + [sync :as sql-jdbc.sync]]) + (:import (java.sql DatabaseMetaData))) + +(set! *warn-on-reflection* true) (driver/register! :clickhouse :parent :sql-jdbc) @@ -83,7 +68,7 @@ (def ^:private default-connection-details {:user "default", :password "", :dbname "default", :host "localhost", :port "8123"}) -(def ^:private product-name "metabase/1.2.0") +(def ^:private product-name "metabase/1.2.1") (defmethod sql-jdbc.conn/connection-details->spec :clickhouse [_ details] @@ -119,8 +104,8 @@ :description (when-not (str/blank? remarks) remarks)})))) (defn- get-tables-from-metadata - [metadata schema-pattern] - (.getTables metadata ; com.clickhouse.jdbc.ClickHouseDatabaseMetaData#getTables + [^DatabaseMetaData metadata schema-pattern] + (.getTables metadata nil ; catalog - unused in the source code there schema-pattern "%" ; tablePattern "%" = match all tables @@ -153,10 +138,9 @@ (or (get-in db [:details :dbname]) (get-in db [:details :db]))) -(def ^:private db-names-separator #" ") (defn- get-tables-in-dbs [db-or-dbs] (->> (for [db (as-> (or (get-db-name db-or-dbs) "default") dbs - (str/split dbs db-names-separator) + (str/split dbs #" ") (remove empty? dbs) (map (comp #(ddl.i/format-name :clickhouse %) str/trim) dbs))] (jdbc/with-db-metadata [metadata (->spec db-or-dbs)] @@ -192,398 +176,6 @@ updated-field)] (merge table-metadata {:fields (set filtered-fields)}))) -(defmethod sql.qp/date [:clickhouse :day-of-week] - [_ _ expr] - (sql.qp/adjust-day-of-week :clickhouse (hsql/call :dayOfWeek expr))) - -(defmethod sql.qp/date [:clickhouse :default] [_ _ expr] expr) - -(defmethod sql.qp/date [:clickhouse :minute] - [_ _ expr] - (hsql/call :toStartOfMinute (hsql/call :toDateTime expr))) - -(defmethod sql.qp/date [:clickhouse :minute-of-hour] - [_ _ expr] - (hsql/call :toMinute (hsql/call :toDateTime expr))) - -(defmethod sql.qp/date [:clickhouse :hour] [_ _ expr] - (hsql/call :toStartOfHour (hsql/call :toDateTime expr))) - -(defmethod sql.qp/date [:clickhouse :hour-of-day] [_ _ expr] - (hsql/call :toHour (hsql/call :toDateTime expr))) - -(defmethod sql.qp/date [:clickhouse :day-of-month] - [_ _ expr] - (hsql/call :toDayOfMonth (hsql/call :toDateTime expr))) - -(defn- to-start-of-week - [expr] - ;; ClickHouse weeks usually start on Monday - (hsql/call :toMonday expr)) - -(defn- to-start-of-year - [expr] - (hsql/call :toStartOfYear (hsql/call :toDateTime expr))) - -(defn- to-relative-day-num - [expr] - (hsql/call :toRelativeDayNum (hsql/call :toDateTime expr))) - -(defn- to-day-of-year - [expr] - (hx/+ (hx/- (to-relative-day-num expr) - (to-relative-day-num (to-start-of-year expr))) - 1)) - -(defmethod sql.qp/date [:clickhouse :day-of-year] - [_ _ expr] - (to-day-of-year expr)) - -(defmethod sql.qp/date [:clickhouse :week-of-year-iso] - [_ _ expr] - (hsql/call :toISOWeek expr)) - -(defmethod sql.qp/date [:clickhouse :month] [_ _ expr] - (hsql/call :toStartOfMonth (hsql/call :toDateTime expr))) - -(defmethod sql.qp/date [:clickhouse :month-of-year] - [_ _ expr] - (hsql/call :toMonth (hsql/call :toDateTime expr))) - -(defmethod sql.qp/date [:clickhouse :quarter-of-year] - [_ _ expr] - (hsql/call :toQuarter expr)) - -(defmethod sql.qp/date [:clickhouse :year] [_ _ expr] - (hsql/call :toStartOfYear (hsql/call :toDateTime expr))) - -(defmethod sql.qp/date [:clickhouse :day] [_ _ expr] - (hsql/call :toDate expr)) - -(defmethod sql.qp/date [:clickhouse :week] - [driver _ expr] - (sql.qp/adjust-start-of-week driver to-start-of-week expr)) -(defmethod sql.qp/date [:clickhouse :quarter] - [_ _ expr] - (hsql/call :toStartOfQuarter (hsql/call :toDateTime expr))) - -(defmethod sql.qp/unix-timestamp->honeysql [:clickhouse :seconds] - [_ _ expr] - (hsql/call :toDateTime expr)) - -(defmethod sql.qp/unix-timestamp->honeysql [:clickhouse :milliseconds] - [_ _ expr] - (hsql/call :toDateTime64 (hx// expr 1000), 3)) - -(defmethod unprepare/unprepare-value [:clickhouse LocalDate] - [_ t] - (format "toDate('%s')" (t/format "yyyy-MM-dd" t))) - -(defmethod unprepare/unprepare-value [:clickhouse LocalTime] - [_ t] - (format "'%s'" (t/format "HH:mm:ss.SSS" t))) - -(defmethod unprepare/unprepare-value [:clickhouse OffsetTime] - [_ t] - (format "'%s'" (t/format "HH:mm:ss.SSSZZZZZ" t))) - -(defmethod unprepare/unprepare-value [:clickhouse LocalDateTime] - [_ t] - (format "'%s'" (t/format "yyyy-MM-dd HH:mm:ss.SSS" t))) - -(defmethod unprepare/unprepare-value [:clickhouse OffsetDateTime] - [_ t] - (format "%s('%s')" - (if (zero? (.getNano t)) "parseDateTimeBestEffort" "parseDateTime64BestEffort") - (t/format "yyyy-MM-dd HH:mm:ss.SSSZZZZZ" t))) - -(defmethod unprepare/unprepare-value [:clickhouse ZonedDateTime] - [_ t] - (format "'%s'" (t/format "yyyy-MM-dd HH:mm:ss.SSSZZZZZ" t))) - -;; Metabase supplies parameters for Date fields as ZonedDateTime -;; ClickHouse complains about too long parameter values. This is unfortunate -;; because it eats some performance, but I do not know a better solution -(defmethod sql.qp/->honeysql [:clickhouse ZonedDateTime] - [_ t] - (hsql/call (if (zero? (.getNano t)) :parseDateTimeBestEffort :parseDateTime64BestEffort) - (t/format "yyyy-MM-dd HH:mm:ss.SSSZZZZZ" t))) - -(defmethod sql.qp/->honeysql [:clickhouse LocalDateTime] - [_ t] - (hsql/call (if (zero? (.getNano t)) :parseDateTimeBestEffort :parseDateTime64BestEffort) - (t/format "yyyy-MM-dd HH:mm:ss.SSS" t))) - -(defmethod sql.qp/->honeysql [:clickhouse OffsetDateTime] - [_ t] - (hsql/call (if (zero? (.getNano t)) :parseDateTimeBestEffort :parseDateTime64BestEffort) - (t/format "yyyy-MM-dd HH:mm:ss.SSSZZZZZ" t))) - -(defmethod sql.qp/->honeysql [:clickhouse LocalDate] - [_ t] - (hsql/call :parseDateTimeBestEffort t)) - -(defmethod sql.qp/->honeysql [:clickhouse LocalTime] - [driver t] - (sql.qp/->honeysql driver (t/local-date-time (t/local-date 1970 1 1) t))) - -(defmethod sql.qp/->honeysql [:clickhouse OffsetTime] - [driver t] - (sql.qp/->honeysql driver - (t/offset-date-time (t/local-date-time - (t/local-date 1970 1 1) - (.toLocalTime t)) - (.getOffset t)))) - -;; We still need this for the tests that use multiple case statements where -;; we can have either Int or Float in different branches, -;; so we just coerce everything to Float64. -;; -;; See metabase.query-processor-test.expressions-test "Can use expressions as values" -(defmethod sql.qp/->honeysql [:clickhouse :/] - [driver args] - (let [args (for [arg args] - (hsql/call :toFloat64 (sql.qp/->honeysql driver arg)))] - ((get-method sql.qp/->honeysql [:sql :/]) driver args))) - -(defn- interval? [expr] - (mbql.u/is-clause? :interval expr)) - -(defmethod sql.qp/->honeysql [:clickhouse :+] - [driver [_ & args]] - (if (some interval? args) - (if-let [[field intervals] (u/pick-first (complement interval?) args)] - (reduce (fn [hsql-form [_ amount unit]] - (add-interval-honeysql-form driver hsql-form amount unit)) - (sql.qp/->honeysql driver field) - intervals) - (throw (ex-info "Summing intervals is not supported" {:args args}))) - (apply hsql/call :+ - (map #(hsql/call :toFloat64 (sql.qp/->honeysql driver %)) args)))) - -(defmethod sql.qp/->honeysql [:clickhouse :log] - [driver [_ field]] - (hsql/call :log10 (sql.qp/->honeysql driver field))) - -(defmethod hformat/fn-handler "quantile" - [_ field p] - (str "quantile(" (hformat/to-sql p) ")(" (hformat/to-sql field) ")")) - -(defmethod sql.qp/->honeysql [:clickhouse :percentile] - [driver [_ field p]] - (hsql/call :quantile - (sql.qp/->honeysql driver field) - (sql.qp/->honeysql driver p))) - -(defmethod hformat/fn-handler "extract_ch" - [_ s p] - (str "extract(" (hformat/to-sql s) "," (hformat/to-sql p) ")")) - -(defmethod sql.qp/->honeysql [:clickhouse :regex-match-first] - [driver [_ arg pattern]] - (hsql/call :extract_ch (sql.qp/->honeysql driver arg) pattern)) - -(defmethod sql.qp/->honeysql [:clickhouse :stddev] - [driver [_ field]] - (hsql/call :stddevPop (sql.qp/->honeysql driver field))) - -;; Substring does not work for Enums, so we need to cast to String -(defmethod sql.qp/->honeysql [:clickhouse :substring] - [driver [_ arg start length]] - (if length - (hsql/call :substring - (hsql/call :toString (sql.qp/->honeysql driver arg)) - (sql.qp/->honeysql driver start) - (sql.qp/->honeysql driver length)) - (hsql/call :substring - (hsql/call :toString (sql.qp/->honeysql driver arg)) - (sql.qp/->honeysql driver start)))) - -(defmethod sql.qp/->honeysql [:clickhouse :var] - [driver [_ field]] - (hsql/call :varPop (sql.qp/->honeysql driver field))) - -(defmethod sql.qp/->float :clickhouse [_ value] (hsql/call :toFloat64 value)) - -(defmethod sql.qp/->honeysql [:clickhouse :value] - [driver value] - (let [[_ value {base-type :base_type}] value] - (when (some? value) - (condp #(isa? %2 %1) base-type - :type/IPAddress (hsql/call :toIPv4 value) - (sql.qp/->honeysql driver value))))) - -;; the filter criterion reads "is empty" -;; also see desugar.clj -(defmethod sql.qp/->honeysql [:clickhouse :=] - [driver [_ field value]] - (let [[qual valuevalue fieldinfo] value] - (if (and (isa? qual :value) - (isa? (:base_type fieldinfo) :type/Text) - (nil? valuevalue)) - [:or - [:= (sql.qp/->honeysql driver field) (sql.qp/->honeysql driver value)] - [:= (hsql/call :empty (sql.qp/->honeysql driver field)) 1]] - ((get-method sql.qp/->honeysql [:sql :=]) driver [_ field value])))) - -;; the filter criterion reads "not empty" -;; also see desugar.clj -(defmethod sql.qp/->honeysql [:clickhouse :!=] - [driver [_ field value]] - (let [[qual valuevalue fieldinfo] value] - (if (and (isa? qual :value) - (isa? (:base_type fieldinfo) :type/Text) - (nil? valuevalue)) - [:and - [:!= (sql.qp/->honeysql driver field) (sql.qp/->honeysql driver value)] - [:= (hsql/call :notEmpty (sql.qp/->honeysql driver field)) 1]] - ((get-method sql.qp/->honeysql [:sql :!=]) driver [_ field value])))) - -;; I do not know why the tests expect nil counts for empty results -;; but that's how it is :-) -;; -;; It would even be better if we could use countIf and sumIf directly -;; -;; metabase.query-processor-test.count-where-test -;; metabase.query-processor-test.share-test -(defmethod sql.qp/->honeysql [:clickhouse :count-where] - [driver [_ pred]] - (hsql/call :case (hsql/call :> (hsql/call :count) 0) - (hsql/call :sum - (hsql/call :case (sql.qp/->honeysql driver pred) 1 - :else 0)) - :else nil)) - -(defmethod sql.qp/->honeysql [:clickhouse :sum-where] - [driver [_ field pred]] - (hsql/call :sum (hsql/call - :case (sql.qp/->honeysql driver pred) (sql.qp/->honeysql driver field) - :else 0))) - -(defmethod sql.qp/quote-style :clickhouse [_] :mysql) - -(defmethod sql.qp/add-interval-honeysql-form :clickhouse - [_ dt amount unit] - (hx/+ (hx/->timestamp dt) - (hsql/raw (format "INTERVAL %d %s" (int amount) (name unit))))) - -;; The following lines make sure we call lowerUTF8 instead of lower -(defn- ch-like-clause - [driver field value options] - (if (get options :case-sensitive true) - [:like field (sql.qp/->honeysql driver value)] - [:like (hsql/call :lowerUTF8 field) - (sql.qp/->honeysql driver (update value 1 str/lower-case))])) - -(s/defn ^:private update-string-value :- mbql.s/value - [value :- (s/constrained mbql.s/value #(string? (second %)) "string value") f] - (update value 1 f)) - -(defmethod sql.qp/->honeysql [:clickhouse :contains] - [driver [_ field value options]] - (ch-like-clause driver - (sql.qp/->honeysql driver field) - (update-string-value value #(str \% % \%)) - options)) - -(defn- clickhouse-string-fn - [fn-name field value options] - (let [field (sql.qp/->honeysql :clickhouse field) - value (sql.qp/->honeysql :clickhouse value)] - (if (get options :case-sensitive true) - (hsql/call fn-name field value) - (hsql/call fn-name (hsql/call :lowerUTF8 field) (str/lower-case value))))) - -(defmethod sql.qp/->honeysql [:clickhouse :starts-with] - [_ [_ field value options]] - (clickhouse-string-fn :startsWith field value options)) - -(defmethod sql.qp/->honeysql [:clickhouse :ends-with] - [_ [_ field value options]] - (clickhouse-string-fn :endsWith field value options)) - -;; We do not have Time data types, so we cheat a little bit -(defmethod sql.qp/cast-temporal-string [:clickhouse :Coercion/ISO8601->Time] - [_driver _special_type expr] - (hx/->timestamp (hsql/call :parseDateTimeBestEffort - (hsql/call :concat "1970-01-01T" expr)))) - -(defmethod sql.qp/cast-temporal-byte [:clickhouse :Coercion/ISO8601->Time] - [_driver _special_type expr] - (hx/->timestamp expr)) - -(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/TINYINT] - [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] - (fn [] - (.getByte rs i))) - -(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/SMALLINT] - [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] - (fn [] - (.getShort rs i))) - -;; This is for tests only - some of them expect nil values -;; getInt/getLong return 0 in case of a NULL value in the result set -;; the only way to check if it was actually NULL - call ResultSet.wasNull afterwards -(defn ^:private with-null-check - [rs get-value-fn] - (let [value (get-value-fn)] - (if (.wasNull rs) nil value))) - -(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/BIGINT] - [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] - (fn [] - (with-null-check rs #(.getLong rs i)))) - -(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/INTEGER] - [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] - (fn [] - (with-null-check rs #(.getInt rs i)))) - -(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/TIMESTAMP] - [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] - (fn [] - (let [r (.getObject rs i LocalDateTime)] - (cond (nil? r) nil - (= (.toLocalDate r) (t/local-date 1970 1 1)) (.toLocalTime r) - :else r)))) - -(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/TIMESTAMP_WITH_TIMEZONE] - [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] - (fn [] - (when-let [s (.getString rs i)] - (u.date/parse s)))) - -(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/TIME] - [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] - (fn [] - (.getObject rs i OffsetTime))) - -(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/NUMERIC] - [_ ^ResultSet rs ^ResultSetMetaData rsmeta ^Integer i] - (fn [] - ; For some reason "count" is labeled as NUMERIC in the JDBC driver - ; despite being just an UInt64, and it may break some Metabase tests - (if (= (.getColumnLabel rsmeta i) "count") - (.getLong rs i) - (.getBigDecimal rs i)))) - -(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/ARRAY] - [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] - (fn [] - (when-let [arr (.getArray rs i)] - (let [inner (.getArray arr)] - (cond - ;; Booleans are returned as just bytes - (bytes? inner) - (str "[" (str/join ", " (map #(if (= 1 %) "true" "false") inner)) "]") - ;; All other primitives - (.isPrimitive (.getComponentType (.getClass inner))) - (Arrays/toString inner) - ;; Complex types - :else - (.asString (ClickHouseArrayValue/of inner))))))) - (defmethod driver/display-name :clickhouse [_] "ClickHouse") (doseq [[feature supported?] {:standard-deviation-aggregations true diff --git a/src/metabase/driver/clickhouse_qp.clj b/src/metabase/driver/clickhouse_qp.clj new file mode 100644 index 0000000..e5772cf --- /dev/null +++ b/src/metabase/driver/clickhouse_qp.clj @@ -0,0 +1,434 @@ +(ns metabase.driver.clickhouse-qp + "CLickHouse driver: QueryProcessor-related definition" + #_{:clj-kondo/ignore [:unsorted-required-namespaces]} + (:require [clojure.string :as str] + [honey.sql :as sql] + [java-time.api :as t] + [metabase [util :as u]] + [metabase.driver.clickhouse-nippy] + [metabase.driver.sql-jdbc [execute :as sql-jdbc.execute]] + [metabase.driver.sql.query-processor :as sql.qp :refer [add-interval-honeysql-form]] + [metabase.driver.sql.util.unprepare :as unprepare] + [metabase.mbql.schema :as mbql.s] + [metabase.mbql.util :as mbql.u] + [metabase.util.date-2 :as u.date] + [metabase.util.honey-sql-2 :as h2x] + [schema.core :as s]) + (:import [com.clickhouse.data.value ClickHouseArrayValue] + [java.sql ResultSet ResultSetMetaData Types] + [java.time + LocalDate + LocalDateTime + LocalTime + OffsetDateTime + OffsetTime + ZonedDateTime] + java.util.Arrays)) + +;; (set! *warn-on-reflection* true) ;; isn't enabled because of Arrays/toString call + +(defmethod sql.qp/quote-style :clickhouse [_] :mysql) +(defmethod sql.qp/honey-sql-version :clickhouse [_] 2) + +(defn- clickhouse-datetime-fn + [fn-name expr] + [fn-name (h2x/->datetime expr)]) + +(defmethod sql.qp/date [:clickhouse :day-of-week] + [_ _ expr] + ;; a tick in the function name prevents HSQL2 to make the function call UPPERCASE + ;; https://cljdoc.org/d/com.github.seancorfield/honeysql/2.4.1011/doc/getting-started/other-databases#clickhouse + (sql.qp/adjust-day-of-week :clickhouse [:'dayOfWeek expr])) + +(defmethod sql.qp/date [:clickhouse :default] + [_ _ expr] + expr) + +(defmethod sql.qp/date [:clickhouse :minute] + [_ _ expr] + (clickhouse-datetime-fn :'toStartOfMinute expr)) + +(defmethod sql.qp/date [:clickhouse :minute-of-hour] + [_ _ expr] + (clickhouse-datetime-fn :'toMinute expr)) + +(defmethod sql.qp/date [:clickhouse :hour] + [_ _ expr] + (clickhouse-datetime-fn :'toStartOfHour expr)) + +(defmethod sql.qp/date [:clickhouse :hour-of-day] + [_ _ expr] + (clickhouse-datetime-fn :'toHour expr)) + +(defmethod sql.qp/date [:clickhouse :day-of-month] + [_ _ expr] + (clickhouse-datetime-fn :'toDayOfMonth expr)) + +(defn- to-start-of-week + [expr] + ;; ClickHouse weeks usually start on Monday + (clickhouse-datetime-fn :'toMonday expr)) + +(defn- to-start-of-year + [expr] + (clickhouse-datetime-fn :'toStartOfYear expr)) + +(defn- to-relative-day-num + [expr] + (clickhouse-datetime-fn :'toRelativeDayNum expr)) + +(defn- to-day-of-year + [expr] + (h2x/+ (h2x/- (to-relative-day-num expr) + (to-relative-day-num (to-start-of-year expr))) + 1)) + +(defmethod sql.qp/date [:clickhouse :day-of-year] + [_ _ expr] + (to-day-of-year expr)) + +(defmethod sql.qp/date [:clickhouse :week-of-year-iso] + [_ _ expr] + (clickhouse-datetime-fn :'toISOWeek expr)) + +(defmethod sql.qp/date [:clickhouse :month] + [_ _ expr] + (clickhouse-datetime-fn :'toStartOfMonth expr)) + +(defmethod sql.qp/date [:clickhouse :month-of-year] + [_ _ expr] + (clickhouse-datetime-fn :'toMonth expr)) + +(defmethod sql.qp/date [:clickhouse :quarter-of-year] + [_ _ expr] + (clickhouse-datetime-fn :'toQuarter expr)) + +(defmethod sql.qp/date [:clickhouse :year] + [_ _ expr] + (clickhouse-datetime-fn :'toStartOfYear expr)) + +(defmethod sql.qp/date [:clickhouse :day] + [_ _ expr] + (h2x/->date expr)) + +(defmethod sql.qp/date [:clickhouse :week] + [driver _ expr] + (sql.qp/adjust-start-of-week driver to-start-of-week expr)) + +(defmethod sql.qp/date [:clickhouse :quarter] + [_ _ expr] + (clickhouse-datetime-fn :'toStartOfQuarter expr)) + +(defmethod sql.qp/unix-timestamp->honeysql [:clickhouse :seconds] + [_ _ expr] + (h2x/->datetime expr)) + +(defmethod sql.qp/unix-timestamp->honeysql [:clickhouse :milliseconds] + [_ _ expr] + [:'toDateTime64 (h2x// expr 1000) 3]) + +(defn- date-time-parse-fn + [nano] + (if (zero? nano) :'parseDateTimeBestEffort :'parseDateTime64BestEffort)) + +(defmethod sql.qp/->honeysql [:clickhouse LocalDateTime] + [_ ^java.time.LocalDateTime t] + (let [formatted (t/format "yyyy-MM-dd HH:mm:ss.SSS" t) + fn (date-time-parse-fn (.getNano t))] + [fn formatted])) + +(defmethod sql.qp/->honeysql [:clickhouse ZonedDateTime] + [_ ^java.time.ZonedDateTime t] + (let [formatted (t/format "yyyy-MM-dd HH:mm:ss.SSSZZZZZ" t) + fn (date-time-parse-fn (.getNano t))] + [fn formatted])) + +(defmethod sql.qp/->honeysql [:clickhouse OffsetDateTime] + [_ ^java.time.OffsetDateTime t] + ;; copy-paste due to reflection warnings + (let [formatted (t/format "yyyy-MM-dd HH:mm:ss.SSSZZZZZ" t) + fn (date-time-parse-fn (.getNano t))] + [fn formatted])) + +(defmethod sql.qp/->honeysql [:clickhouse LocalDate] + [_ ^java.time.LocalDate t] + [:'parseDateTimeBestEffort t]) + +(defn- local-date-time + [^java.time.LocalTime t] + (t/local-date-time (t/local-date 1970 1 1) t)) + +(defmethod sql.qp/->honeysql [:clickhouse LocalTime] + [driver ^java.time.LocalTime t] + (sql.qp/->honeysql driver (local-date-time t))) + +(defmethod sql.qp/->honeysql [:clickhouse OffsetTime] + [driver ^java.time.OffsetTime t] + (sql.qp/->honeysql driver (t/offset-date-time + (local-date-time (.toLocalTime t)) + (.getOffset t)))) + +(defn- args->float64 + [args] + (map (fn [arg] [:'toFloat64 (sql.qp/->honeysql :clickhouse arg)]) args)) + +(defn- interval? [expr] + (mbql.u/is-clause? :interval expr)) + +(defmethod sql.qp/->honeysql [:clickhouse :+] + [driver [_ & args]] + (if (some interval? args) + (if-let [[field intervals] (u/pick-first (complement interval?) args)] + (reduce (fn [hsql-form [_ amount unit]] + (add-interval-honeysql-form driver hsql-form amount unit)) + (sql.qp/->honeysql driver field) + intervals) + (throw (ex-info "Summing intervals is not supported" {:args args}))) + (into [:+] (args->float64 args)))) + +(defmethod sql.qp/->honeysql [:clickhouse :log] + [driver [_ field]] + [:log10 (sql.qp/->honeysql driver field)]) + +(defn- format-expr + [expr] + (first (sql/format-expr (sql.qp/->honeysql :clickhouse expr) {:nested true}))) + +(defmethod sql.qp/->honeysql [:clickhouse :percentile] + [_ [_ field p]] + [:raw (format "quantile(%s)(%s)" (format-expr p) (format-expr field))]) + +(defmethod sql.qp/->honeysql [:clickhouse :regex-match-first] + [driver [_ arg pattern]] + [:'extract (sql.qp/->honeysql driver arg) pattern]) + +(defmethod sql.qp/->honeysql [:clickhouse :stddev] + [driver [_ field]] + [:'stddevPop (sql.qp/->honeysql driver field)]) + +(defmethod sql.qp/->honeysql [:clickhouse :median] + [driver [_ field]] + [:'median (sql.qp/->honeysql driver field)]) + +;; Substring does not work for Enums, so we need to cast to String +(defmethod sql.qp/->honeysql [:clickhouse :substring] + [driver [_ arg start length]] + (let [str [:'toString (sql.qp/->honeysql driver arg)]] + (if length + [:'substring str + (sql.qp/->honeysql driver start) + (sql.qp/->honeysql driver length)] + [:'substring str + (sql.qp/->honeysql driver start)]))) + +(defmethod sql.qp/->honeysql [:clickhouse :var] + [driver [_ field]] + [:'varPop (sql.qp/->honeysql driver field)]) + +(defmethod sql.qp/->float :clickhouse + [_ value] + [:'toFloat64 value]) + +(defmethod sql.qp/->honeysql [:clickhouse :value] + [driver value] + (let [[_ value {base-type :base_type}] value] + (when (some? value) + (condp #(isa? %2 %1) base-type + :type/IPAddress [:'toIPv4 value] + (sql.qp/->honeysql driver value))))) + +;; the filter criterion reads "is empty" +;; also see desugar.clj +(defmethod sql.qp/->honeysql [:clickhouse :=] + [driver [op field value]] + (let [[qual valuevalue fieldinfo] value + hsql-field (sql.qp/->honeysql driver field) + hsql-value (sql.qp/->honeysql driver value)] + (if (and (isa? qual :value) + (isa? (:base_type fieldinfo) :type/Text) + (nil? valuevalue)) + [:or + [:= hsql-field hsql-value] + [:= [:'empty hsql-field] 1]] + ((get-method sql.qp/->honeysql [:sql :=]) driver [op field value])))) + +;; the filter criterion reads "not empty" +;; also see desugar.clj +(defmethod sql.qp/->honeysql [:clickhouse :!=] + [driver [op field value]] + (let [[qual valuevalue fieldinfo] value + hsql-field (sql.qp/->honeysql driver field) + hsql-value (sql.qp/->honeysql driver value)] + (if (and (isa? qual :value) + (isa? (:base_type fieldinfo) :type/Text) + (nil? valuevalue)) + [:and + [:!= hsql-field hsql-value] + [:= [:'notEmpty hsql-field] 1]] + ((get-method sql.qp/->honeysql [:sql :!=]) driver [op field value])))) + +;; I do not know why the tests expect nil counts for empty results +;; but that's how it is :-) +;; +;; It would even be better if we could use countIf and sumIf directly +;; +;; metabase.query-processor-test.count-where-test +;; metabase.query-processor-test.share-test +(defmethod sql.qp/->honeysql [:clickhouse :count-where] + [driver [_ pred]] + [:case + [:> [:'count] 0] + [:sum [:case (sql.qp/->honeysql driver pred) 1 :else 0]] + :else nil]) + +(defmethod sql.qp/->honeysql [:clickhouse :sum-where] + [driver [_ field pred]] + [:sum [:case (sql.qp/->honeysql driver pred) (sql.qp/->honeysql driver field) + :else 0]]) + +(defmethod sql.qp/add-interval-honeysql-form :clickhouse + [_ dt amount unit] + (h2x/+ (h2x/->timestamp dt) + [:raw (format "INTERVAL %d %s" (int amount) (name unit))])) + +;; The following lines make sure we call lowerUTF8 instead of lower +(defn- ch-like-clause + [driver field value options] + (if (get options :case-sensitive true) + [:like field (sql.qp/->honeysql driver value)] + [:like [:'lowerUTF8 field] + (sql.qp/->honeysql driver (update value 1 metabase.util/lower-case-en))])) + +(s/defn ^:private update-string-value :- mbql.s/value + [value :- (s/constrained mbql.s/value #(string? (second %)) ":string value") f] + (update value 1 f)) + +(defmethod sql.qp/->honeysql [:clickhouse :contains] + [driver [_ field value options]] + (ch-like-clause driver + (sql.qp/->honeysql driver field) + (update-string-value value #(str \% % \%)) + options)) + +(defn- clickhouse-string-fn + [fn-name field value options] + (let [field (sql.qp/->honeysql :clickhouse field) + value (sql.qp/->honeysql :clickhouse value)] + (if (get options :case-sensitive true) + [fn-name field value] + [fn-name [:'lowerUTF8 field] (metabase.util/lower-case-en value)]))) + +(defmethod sql.qp/->honeysql [:clickhouse :starts-with] + [_ [_ field value options]] + (clickhouse-string-fn :'startsWith field value options)) + +(defmethod sql.qp/->honeysql [:clickhouse :ends-with] + [_ [_ field value options]] + (clickhouse-string-fn :'endsWith field value options)) + +;; We do not have Time data types, so we cheat a little bit +(defmethod sql.qp/cast-temporal-string [:clickhouse :Coercion/ISO8601->Time] + [_driver _special_type expr] + (h2x/->timestamp [:'parseDateTimeBestEffort [:'concat "1970-01-01T" expr]])) + +(defmethod sql.qp/cast-temporal-byte [:clickhouse :Coercion/ISO8601->Time] + [_driver _special_type expr] + (h2x/->timestamp expr)) + +(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/TINYINT] + [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] + (fn [] + (.getByte rs i))) + +(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/SMALLINT] + [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] + (fn [] + (.getShort rs i))) + +;; This is for tests only - some of them expect nil values +;; getInt/getLong return 0 in case of a NULL value in the result set +;; the only way to check if it was actually NULL - call ResultSet.wasNull afterwards +(defn- with-null-check + [^ResultSet rs value] + (if (.wasNull rs) nil value)) + +(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/BIGINT] + [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] + (fn [] + (with-null-check rs (.getLong rs i)))) + +(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/INTEGER] + [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] + (fn [] + (with-null-check rs (.getInt rs i)))) + +(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/TIMESTAMP] + [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] + (fn [] + (let [^java.time.LocalDateTime r (.getObject rs i LocalDateTime)] + (cond (nil? r) nil + (= (.toLocalDate r) (t/local-date 1970 1 1)) (.toLocalTime r) + :else r)))) + +(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/TIMESTAMP_WITH_TIMEZONE] + [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] + (fn [] + (when-let [s (.getString rs i)] + (u.date/parse s)))) + +(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/TIME] + [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] + (fn [] + (.getObject rs i OffsetTime))) + +(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/NUMERIC] + [_ ^ResultSet rs ^ResultSetMetaData rsmeta ^Integer i] + (fn [] + ; count is NUMERIC cause UInt64 is too large for the canonical SQL BIGINT, + ; and defaults to BigDecimal, but we want it to be coerced to java Long + ; cause it still fits and the tests are expecting that + (if (= (.getColumnLabel rsmeta i) "count") + (.getLong rs i) + (.getBigDecimal rs i)))) + +(defmethod sql-jdbc.execute/read-column-thunk [:clickhouse Types/ARRAY] + [_ ^ResultSet rs ^ResultSetMetaData _ ^Integer i] + (fn [] + (when-let [arr (.getArray rs i)] + (let [inner (.getArray arr)] + (cond + ;; Booleans are returned as just bytes + (bytes? inner) + (str "[" (str/join ", " (map #(if (= 1 %) "true" "false") inner)) "]") + ;; All other primitives + (.isPrimitive (.getComponentType (.getClass inner))) + (Arrays/toString inner) + ;; Complex types + :else + (.asString (ClickHouseArrayValue/of inner))))))) + +(defmethod unprepare/unprepare-value [:clickhouse LocalDate] + [_ t] + (format "toDate('%s')" (t/format "yyyy-MM-dd" t))) + +(defmethod unprepare/unprepare-value [:clickhouse LocalTime] + [_ t] + (format "'%s'" (t/format "HH:mm:ss.SSS" t))) + +(defmethod unprepare/unprepare-value [:clickhouse OffsetTime] + [_ t] + (format "'%s'" (t/format "HH:mm:ss.SSSZZZZZ" t))) + +(defmethod unprepare/unprepare-value [:clickhouse LocalDateTime] + [_ t] + (format "'%s'" (t/format "yyyy-MM-dd HH:mm:ss.SSS" t))) + +(defmethod unprepare/unprepare-value [:clickhouse OffsetDateTime] + [_ ^OffsetDateTime t] + (format "%s('%s')" + (if (zero? (.getNano t)) "parseDateTimeBestEffort" "parseDateTime64BestEffort") + (t/format "yyyy-MM-dd HH:mm:ss.SSSZZZZZ" t))) + +(defmethod unprepare/unprepare-value [:clickhouse ZonedDateTime] + [_ t] + (format "'%s'" (t/format "yyyy-MM-dd HH:mm:ss.SSSZZZZZ" t))) diff --git a/test/metabase/test/data/clickhouse.clj b/test/metabase/test/data/clickhouse.clj index efded56..38fecee 100644 --- a/test/metabase/test/data/clickhouse.clj +++ b/test/metabase/test/data/clickhouse.clj @@ -89,7 +89,7 @@ :ssl false :use_no_proxy false :use_server_time_zone_for_dates true - :product_name "metabase/1.2.0"}) + :product_name "metabase/1.2.1"}) (defn rows-without-index "Remove the Metabase index which is the first column in the result set"