test: split sqllogictest and testdrive tests

We're at the point where we have enough test files that intermingling
testdrive and sqllogictest files is confusing. Split them into separate
directories.

This also opens the door for non-testdrive non-sqllogictest integration
tests, like (perhaps) chbench.

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "sqllogictest"]
-	path = test/sqlite
+	path = test/sqllogictest/sqlite
 	url = https://github.com/MaterializeInc/sqllogictest.git

ci/slt/slt.sh

@@ -21,7 +21,7 @@ mkdir -p target
 
 sqllogictest \
     -v --json-summary-file=target/slt-summary.json --no-fail "$@" \
-    test/*.slt \
-    test/cockroach/*.slt \
-    test/sqlite/test \
+    test/sqllogictest/*.slt \
+    test/sqllogictest/cockroach/*.slt \
+    test/sqllogictest/sqlite/test \
     | tee target/slt.log

ci/test/slt-fast.sh

@@ -18,136 +18,136 @@ fi
 export RUST_BACKTRACE=full
 
 tests=(
-    test/*.slt
-    # test/sqlite/test/evidence/in1.test
-    test/sqlite/test/evidence/in2.test
-    test/sqlite/test/evidence/slt_lang_aggfunc.test
-    test/sqlite/test/evidence/slt_lang_createtrigger.test
-    test/sqlite/test/evidence/slt_lang_createview.test
-    test/sqlite/test/evidence/slt_lang_dropindex.test
-    test/sqlite/test/evidence/slt_lang_droptable.test
-    test/sqlite/test/evidence/slt_lang_droptrigger.test
-    test/sqlite/test/evidence/slt_lang_dropview.test
-    test/sqlite/test/evidence/slt_lang_reindex.test
-    test/sqlite/test/evidence/slt_lang_replace.test
-    test/sqlite/test/evidence/slt_lang_update.test
-    # test/sqlite/test/index/between/1/slt_good_0.test
-    # test/sqlite/test/index/commute/10/slt_good_0.test
-    # test/sqlite/test/index/delete/1/slt_good_0.test
-    # test/sqlite/test/index/in/10/slt_good_0.test
-    # test/sqlite/test/index/orderby_nosort/10/slt_good_0.test
-    # test/sqlite/test/index/orderby/10/slt_good_0.test
-    test/sqlite/test/index/random/10/slt_good_0.test
-    test/sqlite/test/index/view/10/slt_good_0.test
-    test/sqlite/test/random/aggregates/slt_good_0.test
-    test/sqlite/test/random/expr/slt_good_0.test
-    test/sqlite/test/random/groupby/slt_good_0.test
-    test/sqlite/test/random/select/slt_good_0.test
-    # test/sqlite/test/select1.test
-    # test/sqlite/test/select2.test
-    # test/sqlite/test/select3.test
-    # test/sqlite/test/select4.test
-    test/sqlite/test/select5.test
-    # test/cockroach/aggregate.slt
-    test/cockroach/alias_types.slt
-    test/cockroach/alter_column_type.slt
-    test/cockroach/alter_table.slt
-    # test/cockroach/apply_join.slt
-    test/cockroach/array.slt
-    test/cockroach/as_of.slt
-    test/cockroach/bit.slt
-    # test/cockroach/builtin_function.slt
-    test/cockroach/bytes.slt
-    test/cockroach/case_sensitive_names.slt
-    # test/cockroach/collatedstring_constraint.slt
-    test/cockroach/collatedstring_index1.slt
-    test/cockroach/collatedstring_index2.slt
-    test/cockroach/collatedstring_normalization.slt
-    test/cockroach/collatedstring_nullinindex.slt
-    test/cockroach/collatedstring_uniqueindex1.slt
-    test/cockroach/collatedstring_uniqueindex2.slt
-    test/cockroach/collatedstring.slt
-    test/cockroach/computed.slt
-    # test/cockroach/conditional.slt
-    test/cockroach/create_as.slt
-    test/cockroach/custom_escape_character.slt
-    test/cockroach/database.slt
-    # test/cockroach/datetime.slt
-    # test/cockroach/decimal.slt
-    test/cockroach/delete.slt
-    test/cockroach/discard.slt
-    test/cockroach/drop_database.slt
-    test/cockroach/drop_table.slt
-    test/cockroach/drop_user.slt
-    test/cockroach/drop_view.slt
-    test/cockroach/errors.slt
-    # test/cockroach/exec_hash_join.slt
-    # test/cockroach/exec_merge_join.slt
-    test/cockroach/exec_window.slt
-    test/cockroach/extract.slt
-    # test/cockroach/float.slt
-    test/cockroach/inet.slt
-    test/cockroach/information_schema.slt
-    test/cockroach/insert.slt
-    test/cockroach/int_size.slt
-    # test/cockroach/join.slt
-    # test/cockroach/json_builtins.slt
-    # test/cockroach/json.slt
-    test/cockroach/like.slt
-    test/cockroach/limit.slt
-    # test/cockroach/lookup_join.slt
-    test/cockroach/namespace.slt
-    # test/cockroach/no_primary_key.slt
-    test/cockroach/order_by.slt
-    test/cockroach/ordinal_references.slt
-    test/cockroach/ordinality.slt
-    test/cockroach/orms-opt.slt
-    test/cockroach/orms.slt
-    test/cockroach/pg_catalog.slt
-    test/cockroach/pgoidtype.slt
-    # test/cockroach/postgres_jsonb.slt
-    # test/cockroach/postgresjoin.slt
-    test/cockroach/prepare.slt
-    test/cockroach/rename_column.slt
-    test/cockroach/rename_constraint.slt
-    test/cockroach/rename_database.slt
-    test/cockroach/rename_table.slt
-    test/cockroach/rename_view.slt
-    test/cockroach/returning.slt
-    test/cockroach/rows_from.slt
-    test/cockroach/scale.slt
-    test/cockroach/select_index_flags.slt
-    # test/cockroach/select_index_span_ranges.slt
-    # test/cockroach/select_index.slt
-    test/cockroach/select_search_path.slt
-    # test/cockroach/select_table_alias.slt
-    # test/cockroach/select.slt
-    test/cockroach/shift.slt
-    # test/cockroach/sqlsmith.slt
-    test/cockroach/srfs.slt
-    # test/cockroach/statement_source.slt
-    # test/cockroach/suboperators.slt
-    # test/cockroach/subquery_correlated.slt
-    # test/cockroach/subquery-opt.slt
-    # test/cockroach/subquery.slt
-    test/cockroach/table.slt
-    # test/cockroach/target_names.slt
-    # test/cockroach/time.slt
-    test/cockroach/timestamp.slt
-    test/cockroach/truncate.slt
-    test/cockroach/tuple.slt
-    # test/cockroach/typing.slt
-    # test/cockroach/union-opt.slt
-    test/cockroach/union.slt
-    test/cockroach/update.slt
-    test/cockroach/upsert.slt
-    test/cockroach/uuid.slt
-    # test/cockroach/values.slt
-    # test/cockroach/views.slt
-    # test/cockroach/where.slt
-    test/cockroach/window.slt
-    test/cockroach/with.slt
-    # test/cockroach/zero.slt
+    test/sqllogictest/*.slt
+    # test/sqllogictest/sqlite/test/evidence/in1.test
+    test/sqllogictest/sqlite/test/evidence/in2.test
+    test/sqllogictest/sqlite/test/evidence/slt_lang_aggfunc.test
+    test/sqllogictest/sqlite/test/evidence/slt_lang_createtrigger.test
+    test/sqllogictest/sqlite/test/evidence/slt_lang_createview.test
+    test/sqllogictest/sqlite/test/evidence/slt_lang_dropindex.test
+    test/sqllogictest/sqlite/test/evidence/slt_lang_droptable.test
+    test/sqllogictest/sqlite/test/evidence/slt_lang_droptrigger.test
+    test/sqllogictest/sqlite/test/evidence/slt_lang_dropview.test
+    test/sqllogictest/sqlite/test/evidence/slt_lang_reindex.test
+    test/sqllogictest/sqlite/test/evidence/slt_lang_replace.test
+    test/sqllogictest/sqlite/test/evidence/slt_lang_update.test
+    # test/sqllogictest/sqlite/test/index/between/1/slt_good_0.test
+    # test/sqllogictest/sqlite/test/index/commute/10/slt_good_0.test
+    # test/sqllogictest/sqlite/test/index/delete/1/slt_good_0.test
+    # test/sqllogictest/sqlite/test/index/in/10/slt_good_0.test
+    # test/sqllogictest/sqlite/test/index/orderby_nosort/10/slt_good_0.test
+    # test/sqllogictest/sqlite/test/index/orderby/10/slt_good_0.test
+    test/sqllogictest/sqlite/test/index/random/10/slt_good_0.test
+    test/sqllogictest/sqlite/test/index/view/10/slt_good_0.test
+    test/sqllogictest/sqlite/test/random/aggregates/slt_good_0.test
+    test/sqllogictest/sqlite/test/random/expr/slt_good_0.test
+    test/sqllogictest/sqlite/test/random/groupby/slt_good_0.test
+    test/sqllogictest/sqlite/test/random/select/slt_good_0.test
+    # test/sqllogictest/sqlite/test/select1.test
+    # test/sqllogictest/sqlite/test/select2.test
+    # test/sqllogictest/sqlite/test/select3.test
+    # test/sqllogictest/sqlite/test/select4.test
+    test/sqllogictest/sqlite/test/select5.test
+    # test/sqllogictest/cockroach/aggregate.slt
+    test/sqllogictest/cockroach/alias_types.slt
+    test/sqllogictest/cockroach/alter_column_type.slt
+    test/sqllogictest/cockroach/alter_table.slt
+    # test/sqllogictest/cockroach/apply_join.slt
+    test/sqllogictest/cockroach/array.slt
+    test/sqllogictest/cockroach/as_of.slt
+    test/sqllogictest/cockroach/bit.slt
+    # test/sqllogictest/cockroach/builtin_function.slt
+    test/sqllogictest/cockroach/bytes.slt
+    test/sqllogictest/cockroach/case_sensitive_names.slt
+    # test/sqllogictest/cockroach/collatedstring_constraint.slt
+    test/sqllogictest/cockroach/collatedstring_index1.slt
+    test/sqllogictest/cockroach/collatedstring_index2.slt
+    test/sqllogictest/cockroach/collatedstring_normalization.slt
+    test/sqllogictest/cockroach/collatedstring_nullinindex.slt
+    test/sqllogictest/cockroach/collatedstring_uniqueindex1.slt
+    test/sqllogictest/cockroach/collatedstring_uniqueindex2.slt
+    test/sqllogictest/cockroach/collatedstring.slt
+    test/sqllogictest/cockroach/computed.slt
+    # test/sqllogictest/cockroach/conditional.slt
+    test/sqllogictest/cockroach/create_as.slt
+    test/sqllogictest/cockroach/custom_escape_character.slt
+    test/sqllogictest/cockroach/database.slt
+    # test/sqllogictest/cockroach/datetime.slt
+    # test/sqllogictest/cockroach/decimal.slt
+    test/sqllogictest/cockroach/delete.slt
+    test/sqllogictest/cockroach/discard.slt
+    test/sqllogictest/cockroach/drop_database.slt
+    test/sqllogictest/cockroach/drop_table.slt
+    test/sqllogictest/cockroach/drop_user.slt
+    test/sqllogictest/cockroach/drop_view.slt
+    test/sqllogictest/cockroach/errors.slt
+    # test/sqllogictest/cockroach/exec_hash_join.slt
+    # test/sqllogictest/cockroach/exec_merge_join.slt
+    test/sqllogictest/cockroach/exec_window.slt
+    test/sqllogictest/cockroach/extract.slt
+    # test/sqllogictest/cockroach/float.slt
+    test/sqllogictest/cockroach/inet.slt
+    test/sqllogictest/cockroach/information_schema.slt
+    test/sqllogictest/cockroach/insert.slt
+    test/sqllogictest/cockroach/int_size.slt
+    # test/sqllogictest/cockroach/join.slt
+    # test/sqllogictest/cockroach/json_builtins.slt
+    # test/sqllogictest/cockroach/json.slt
+    test/sqllogictest/cockroach/like.slt
+    test/sqllogictest/cockroach/limit.slt
+    # test/sqllogictest/cockroach/lookup_join.slt
+    test/sqllogictest/cockroach/namespace.slt
+    # test/sqllogictest/cockroach/no_primary_key.slt
+    test/sqllogictest/cockroach/order_by.slt
+    test/sqllogictest/cockroach/ordinal_references.slt
+    test/sqllogictest/cockroach/ordinality.slt
+    test/sqllogictest/cockroach/orms-opt.slt
+    test/sqllogictest/cockroach/orms.slt
+    test/sqllogictest/cockroach/pg_catalog.slt
+    test/sqllogictest/cockroach/pgoidtype.slt
+    # test/sqllogictest/cockroach/postgres_jsonb.slt
+    # test/sqllogictest/cockroach/postgresjoin.slt
+    test/sqllogictest/cockroach/prepare.slt
+    test/sqllogictest/cockroach/rename_column.slt
+    test/sqllogictest/cockroach/rename_constraint.slt
+    test/sqllogictest/cockroach/rename_database.slt
+    test/sqllogictest/cockroach/rename_table.slt
+    test/sqllogictest/cockroach/rename_view.slt
+    test/sqllogictest/cockroach/returning.slt
+    test/sqllogictest/cockroach/rows_from.slt
+    test/sqllogictest/cockroach/scale.slt
+    test/sqllogictest/cockroach/select_index_flags.slt
+    # test/sqllogictest/cockroach/select_index_span_ranges.slt
+    # test/sqllogictest/cockroach/select_index.slt
+    test/sqllogictest/cockroach/select_search_path.slt
+    # test/sqllogictest/cockroach/select_table_alias.slt
+    # test/sqllogictest/cockroach/select.slt
+    test/sqllogictest/cockroach/shift.slt
+    # test/sqllogictest/cockroach/sqlsmith.slt
+    test/sqllogictest/cockroach/srfs.slt
+    # test/sqllogictest/cockroach/statement_source.slt
+    # test/sqllogictest/cockroach/suboperators.slt
+    # test/sqllogictest/cockroach/subquery_correlated.slt
+    # test/sqllogictest/cockroach/subquery-opt.slt
+    # test/sqllogictest/cockroach/subquery.slt
+    test/sqllogictest/cockroach/table.slt
+    # test/sqllogictest/cockroach/target_names.slt
+    # test/sqllogictest/cockroach/time.slt
+    test/sqllogictest/cockroach/timestamp.slt
+    test/sqllogictest/cockroach/truncate.slt
+    test/sqllogictest/cockroach/tuple.slt
+    # test/sqllogictest/cockroach/typing.slt
+    # test/sqllogictest/cockroach/union-opt.slt
+    test/sqllogictest/cockroach/union.slt
+    test/sqllogictest/cockroach/update.slt
+    test/sqllogictest/cockroach/upsert.slt
+    test/sqllogictest/cockroach/uuid.slt
+    # test/sqllogictest/cockroach/values.slt
+    # test/sqllogictest/cockroach/views.slt
+    # test/sqllogictest/cockroach/where.slt
+    test/sqllogictest/cockroach/window.slt
+    test/sqllogictest/cockroach/with.slt
+    # test/sqllogictest/cockroach/zero.slt
 )
 
 sqllogictest -v "${tests[@]}"

ci/test/testdrive.compose.yml

@@ -13,7 +13,7 @@ services:
       --kafka-addr=kafka:9092
       --schema-registry-url=http://schema-registry:8081
       --materialized-url=postgres://ignored@materialized:6875
-      test/*.td"
+      test/testdrive/*.td"
     environment:
     - TMPDIR=/tmpshare
     volumes:

doc/developer/develop-testing.md

@@ -142,7 +142,7 @@ sqllogictest file.
 Once PostgreSQL is running, you can run a sqllogictest file like so:
 
 ```shell
-$ cargo run --bin sqllogictest --release -- test/TESTFILE.slt
+$ cargo run --bin sqllogictest --release -- test/sqllogictest/TESTFILE.slt
 ```
 
 For larger test files, it is imperative that you compile in release mode, i.e.,
@@ -155,10 +155,12 @@ To add logging for tests, append `-vv`, e.g.:
 $ cargo run --bin sqllogictest --release -- test/TESTFILE.slt -vv
 ```
 
-The offical SQLite test files are in [test/sqllogictest](/test/sqllogictest),
-and some additional test files from CockroachDB are in
-[test/cockroach](/test/cockroach). Some additional Materialize-specific
-sqllogictest files live in [test/](/test) with a filename suffix of `.slt`—feel
+The offical SQLite test files are in
+[test/sqllogictest/sqlite](/test/sqllogictest/sqlite), and some additional test
+files from CockroachDB are in
+[test/sqllogictest/cockroach](/test/sqllogictest/cockroach). Some additional
+Materialize-specific sqllogictest files live in
+[test/sqllogictest](/test/sqllogictest) with a filename suffix of `.slt`—feel
 free to add more!
 
 ### testdrive
@@ -228,7 +230,7 @@ more!
 ## Long-running tests
 
 These are still a work in progress. The beginning of the orchestration has
-begun, though; see the Docker Compose demo in [ex/chbench](/ex/chbench) if
+begun, though; see the Docker Compose demo in [demo/chbench](/demo/chbench) if
 you're curious.
 
 ## What kind of tests should I write?