From 07058213ca861715e032bbf5b5268a4f48dd848b Mon Sep 17 00:00:00 2001 From: Zhiyuan Liang <132966438+Ami11111@users.noreply.github.com> Date: Fri, 30 Aug 2024 19:06:12 +0800 Subject: [PATCH 1/3] Add segment_index_entry and table_index_entry test cases for unit test (#1774) ### What problem does this PR solve? Add segment_index_entry and table_index_entry test cases for unit test ### Type of change - [x] Test cases --- .../meta/entry/segment_index_entry.cpp | 469 ++++++++++++++++++ .../storage/meta/entry/table_index_entry.cpp | 152 ++++++ 2 files changed, 621 insertions(+) create mode 100644 src/unit_test/storage/meta/entry/segment_index_entry.cpp diff --git a/src/unit_test/storage/meta/entry/segment_index_entry.cpp b/src/unit_test/storage/meta/entry/segment_index_entry.cpp new file mode 100644 index 0000000000..edc07b1452 --- /dev/null +++ b/src/unit_test/storage/meta/entry/segment_index_entry.cpp @@ -0,0 +1,469 @@ +#include "unit_test/base_test.h" +#include "type/complex/embedding_type.h" +#include + +import infinity_context; +import infinity_exception; + +import stl; +import global_resource_usage; +import third_party; +import logger; +import table_def; +import value; +import bitmask; + +import data_block; +import default_values; +import txn_manager; +import buffer_manager; +import txn; +import catalog; +import status; +import extra_ddl_info; +import column_def; +import data_type; +import logical_type; +import embedding_info; +import sparse_info; +import index_hnsw; +import index_full_text; +import index_ivfflat; +import index_bmp; +import statement_common; +import data_access_state; +import txn_store; +import column_vector; +import internal_types; +import constant_expr; +import parsed_expr; +import base_table_ref; +import index_base; +import chunk_index_entry; + +import base_entry; +import table_entry; +import table_entry_type; +import segment_entry; +import segment_index_entry; +import block_entry; + +using namespace infinity; + +class SegmentIndexEntryTest : public BaseTestParamStr { + void SetUp() override { + BaseTestParamStr::SetUp(); +#ifdef INFINITY_DEBUG + infinity::GlobalResourceUsage::Init(); +#endif + std::shared_ptr config_path = nullptr; + RemoveDbDirs(); + system(("mkdir -p " + infinity::String(GetFullPersistDir())).c_str()); + system(("mkdir -p " + infinity::String(GetFullDataDir())).c_str()); + system(("mkdir -p " + infinity::String(GetFullTmpDir())).c_str()); + std::string config_path_str = GetParam(); + if (config_path_str != BaseTestParamStr::NULL_CONFIG_PATH) { + config_path = infinity::MakeShared(config_path_str); + } + infinity::InfinityContext::instance().Init(config_path); + } + + void TearDown() override { + infinity::InfinityContext::instance().UnInit(); +#ifdef INFINITY_DEBUG + EXPECT_EQ(infinity::GlobalResourceUsage::GetObjectCount(), 0); + EXPECT_EQ(infinity::GlobalResourceUsage::GetRawMemoryCount(), 0); + infinity::GlobalResourceUsage::UnInit(); +#endif + BaseTestParamStr::TearDown(); + } +}; + +INSTANTIATE_TEST_SUITE_P(TestWithDifferentParams, + SegmentIndexEntryTest, + ::testing::Values(BaseTestParamStr::NULL_CONFIG_PATH, + BaseTestParamStr::VFS_OFF_CONFIG_PATH)); + +void CreateTable(); +void CreateIndex(); +void InsertData(const String& db_name, const String& table_name); +void DropIndex(); +void DropTable(); + +TEST_P(SegmentIndexEntryTest, decode_index_test) { + TxnManager *txn_mgr = infinity::InfinityContext::instance().storage()->txn_manager(); + CreateTable(); + CreateIndex(); + InsertData("default_db", "tbl1"); + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("decode")); + const String &db_name = "default_db"; + const String &table_name = "tbl1"; + const String &index_name = "fulltext_index"; + auto [index_entry, index_status] = txn1->GetIndexByName(db_name, table_name, index_name); + EXPECT_TRUE(index_status.ok()); + SharedPtr segment_index_entry; + EXPECT_FALSE(index_entry->GetOrCreateSegment(0, txn1, segment_index_entry));//get + String encoded_index = SegmentIndexEntry::EncodeIndex(0, index_entry); + std::cout< decoded_index = SegmentIndexEntry::DecodeIndex(encoded_index); + EXPECT_TRUE(decoded_index[0] == "default_db"); + EXPECT_TRUE(decoded_index[1] == "tbl1"); + EXPECT_TRUE(decoded_index[2] == "fulltext_index"); + EXPECT_TRUE(decoded_index[3] == "0"); + EXPECT_THROW(SegmentEntry::DecodeIndex("/default_db/tbl1/fulltext_index/0"), UnrecoverableException); + } + + DropIndex(); + DropTable(); +} + +TEST_P(SegmentIndexEntryTest, create_ivfflat_index_test) { + TxnManager *txn_mgr = infinity::InfinityContext::instance().storage()->txn_manager(); + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("create table")); + Vector> columns; + { + std::set constraints; + constraints.insert(ConstraintType::kNotNull); + i64 column_id = 0; + auto embeddingInfo = MakeShared(EmbeddingDataType::kElemFloat, 128); + auto column_def_ptr = + MakeShared(column_id, MakeShared(LogicalType::kEmbedding, embeddingInfo), "col1", constraints); + columns.emplace_back(column_def_ptr); + } + auto tbl1_def = MakeUnique(MakeShared("default_db"), MakeShared("tbl1"), columns); + auto status = txn1->CreateTable("default_db", std::move(tbl1_def), ConflictType::kError); + EXPECT_TRUE(status.ok()); + txn_mgr->CommitTxn(txn1); + } + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("create index")); + Vector columns{"col1"}; + Vector parameters; + parameters.emplace_back(new InitParameter("centroids_count", "100")); + parameters.emplace_back(new InitParameter("metric", "l2")); + + auto index_base = IndexIVFFlat::Make(MakeShared("idx1"), "tbl1_idx1", columns, parameters); + // std::cout << "index_base: " << index_base->ToString() << std::endl; + for (auto parameter : parameters) { + delete parameter; + } + + const String &db_name = "default_db"; + const String &table_name = "tbl1"; + auto [table_entry, table_status] = txn1->GetTableByName(db_name, table_name); + EXPECT_EQ(table_status.ok(), true); + + auto [index_entry, index_status] = txn1->CreateIndexDef(table_entry, index_base, ConflictType::kInvalid); + EXPECT_EQ(index_status.ok(), true); + auto table_ref = BaseTableRef::FakeTableRef(table_entry, txn1); + auto [_, status3] = txn1->CreateIndexPrepare(index_entry, table_ref.get(), true, true); + txn1->CreateIndexFinish(table_entry, index_entry); + EXPECT_TRUE(status3.ok()); + + txn_mgr->CommitTxn(txn1); + } + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("create_file_worker")); + const String &db_name = "default_db"; + const String &table_name = "tbl1"; + const String &index_name = "idx1"; + auto [index_entry, index_status] = txn1->GetIndexByName(db_name, table_name, index_name); + EXPECT_TRUE(index_status.ok()); + SharedPtr segment_index_entry; + EXPECT_TRUE(index_entry->GetOrCreateSegment(0, txn1, segment_index_entry));//create + txn_mgr->CommitTxn(txn1); + } + + DropTable(); +} + +TEST_P(SegmentIndexEntryTest, index_file_name_test) { + TxnManager *txn_mgr = infinity::InfinityContext::instance().storage()->txn_manager(); + CreateTable(); + CreateIndex(); + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("name")); + const String &db_name = "default_db"; + const String &table_name = "tbl1"; + const String &index_name = "fulltext_index"; + auto [index_entry, index_status] = txn1->GetIndexByName(db_name, table_name, index_name); + EXPECT_TRUE(index_status.ok()); + SharedPtr segment_index_entry; + EXPECT_TRUE(index_entry->GetOrCreateSegment(0, txn1, segment_index_entry));//create + String index_file_name = segment_index_entry->IndexFileName(0); + std::cout<txn_manager(); + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("create table")); + Vector> columns; + { + std::set constraints; + constraints.insert(ConstraintType::kNotNull); + i64 column_id = 0; + auto embeddingInfo = MakeShared(EmbeddingDataType::kElemFloat, 128); + auto column_def_ptr = + MakeShared(column_id, MakeShared(LogicalType::kEmbedding, embeddingInfo), "col1", constraints); + columns.emplace_back(column_def_ptr); + } + auto tbl1_def = MakeUnique(MakeShared("default_db"), MakeShared("tbl1"), columns); + auto status = txn1->CreateTable("default_db", std::move(tbl1_def), ConflictType::kError); + EXPECT_TRUE(status.ok()); + txn_mgr->CommitTxn(txn1); + } + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("create index")); + Vector columns{"col1"}; + Vector parameters; + parameters.emplace_back(new InitParameter("metric", "l2")); + parameters.emplace_back(new InitParameter("m", "16")); + parameters.emplace_back(new InitParameter("ef_construction", "200")); + parameters.emplace_back(new InitParameter("encode", "plain")); + + auto index_base = IndexHnsw::Make(MakeShared("idx1"), "tbl1_idx1", columns, parameters); + // std::cout << "index_base: " << index_base->ToString() << std::endl; + + for (auto parameter : parameters) { + delete parameter; + } + + const String &db_name = "default_db"; + const String &table_name = "tbl1"; + auto [table_entry, table_status] = txn1->GetTableByName(db_name, table_name); + EXPECT_EQ(table_status.ok(), true); + + auto [index_entry, index_status] = txn1->CreateIndexDef(table_entry, index_base, ConflictType::kInvalid); + EXPECT_EQ(index_status.ok(), true); + auto table_ref = BaseTableRef::FakeTableRef(table_entry, txn1); + auto [_, status3] = txn1->CreateIndexPrepare(index_entry, table_ref.get(), true, true); + txn1->CreateIndexFinish(table_entry, index_entry); + EXPECT_TRUE(status3.ok()); + + txn_mgr->CommitTxn(txn1); + } + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("opt")); + const String &db_name = "default_db"; + const String &table_name = "tbl1"; + const String &index_name = "idx1"; + auto [index_entry, index_status] = txn1->GetIndexByName(db_name, table_name, index_name); + EXPECT_TRUE(index_status.ok()); + auto [table_entry, table_status] = txn1->GetTableByName(db_name, table_name); + EXPECT_TRUE(table_status.ok()); + Vector> opt_params; + opt_params.emplace_back(MakeUnique("compress_to_lvq", "")); + TxnTableStore *txn_table_store = txn1->GetTxnTableStore(table_entry); + SharedPtr segment_index_entry; + EXPECT_TRUE(index_entry->GetOrCreateSegment(0, txn1, segment_index_entry)); + segment_index_entry->OptIndex(const_cast(index_entry->index_base()), txn_table_store, opt_params, false); + + opt_params.clear(); + opt_params.emplace_back(MakeUnique("lvq_avg", "")); + segment_index_entry->OptIndex(const_cast(index_entry->index_base()), txn_table_store, opt_params, false); + + opt_params.clear(); + opt_params.emplace_back(MakeUnique("compress_to_lvq", "")); + opt_params.emplace_back(MakeUnique("lvq_avg", "")); + EXPECT_THROW(segment_index_entry->OptIndex(const_cast(index_entry->index_base()), txn_table_store, opt_params, false), RecoverableException); + txn_mgr->CommitTxn(txn1); + } + + DropTable(); +} + +TEST_P(SegmentIndexEntryTest, opt_bmp_index_test){ + TxnManager *txn_mgr = infinity::InfinityContext::instance().storage()->txn_manager(); + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("create table")); + Vector> columns; + { + std::set constraints; + constraints.insert(ConstraintType::kNotNull); + i64 column_id = 0; + auto sparseInfo = MakeShared(EmbeddingDataType::kElemFloat, EmbeddingDataType::kElemInt32, 30000, SparseStoreType::kSort); + auto column_def_ptr = + MakeShared(column_id, MakeShared(LogicalType::kSparse, sparseInfo), "col1", constraints); + columns.emplace_back(column_def_ptr); + } + auto tbl1_def = MakeUnique(MakeShared("default_db"), MakeShared("tbl1"), columns); + auto status = txn1->CreateTable("default_db", std::move(tbl1_def), ConflictType::kError); + EXPECT_TRUE(status.ok()); + txn_mgr->CommitTxn(txn1); + } + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("create index")); + Vector columns{"col1"}; + Vector parameters; + parameters.emplace_back(new InitParameter("block_size", "16")); + parameters.emplace_back(new InitParameter("compress_type", "compress")); + + auto index_base = IndexBMP::Make(MakeShared("idx1"), "tbl1_idx1", columns, parameters); + // std::cout << "index_base: " << index_base->ToString() << std::endl; + + for (auto parameter : parameters) { + delete parameter; + } + + const String &db_name = "default_db"; + const String &table_name = "tbl1"; + auto [table_entry, table_status] = txn1->GetTableByName(db_name, table_name); + EXPECT_EQ(table_status.ok(), true); + + auto [index_entry, index_status] = txn1->CreateIndexDef(table_entry, index_base, ConflictType::kInvalid); + EXPECT_EQ(index_status.ok(), true); + auto table_ref = BaseTableRef::FakeTableRef(table_entry, txn1); + auto [_, status3] = txn1->CreateIndexPrepare(index_entry, table_ref.get(), true, true); + txn1->CreateIndexFinish(table_entry, index_entry); + EXPECT_TRUE(status3.ok()); + + txn_mgr->CommitTxn(txn1); + } + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("opt")); + const String &db_name = "default_db"; + const String &table_name = "tbl1"; + const String &index_name = "idx1"; + auto [index_entry, index_status] = txn1->GetIndexByName(db_name, table_name, index_name); + EXPECT_TRUE(index_status.ok()); + auto [table_entry, table_status] = txn1->GetTableByName(db_name, table_name); + EXPECT_TRUE(table_status.ok()); + Vector> opt_params; + opt_params.emplace_back(MakeUnique("topk", "10")); + opt_params.emplace_back(MakeUnique("bp_reorder", "")); + TxnTableStore *txn_table_store = txn1->GetTxnTableStore(table_entry); + SharedPtr segment_index_entry; + EXPECT_TRUE(index_entry->GetOrCreateSegment(0, txn1, segment_index_entry)); + segment_index_entry->OptIndex(const_cast(index_entry->index_base()), txn_table_store, opt_params, false); + txn_mgr->CommitTxn(txn1); + } + + DropTable(); +} + +TEST_P(SegmentIndexEntryTest, flush_fulltext_test) { + TxnManager *txn_mgr = infinity::InfinityContext::instance().storage()->txn_manager(); + CreateTable(); + CreateIndex(); + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("flush")); + const String &db_name = "default_db"; + const String &table_name = "tbl1"; + const String &index_name = "fulltext_index"; + auto [index_entry, index_status] = txn1->GetIndexByName(db_name, table_name, index_name); + EXPECT_TRUE(index_status.ok()); + SharedPtr segment_index_entry; + EXPECT_TRUE(index_entry->GetOrCreateSegment(0, txn1, segment_index_entry));//create + EXPECT_FALSE(segment_index_entry->Flush(txn1->BeginTS())); + } + + DropIndex(); + DropTable(); +} + +TEST_P(SegmentIndexEntryTest, flush_bmp_test) { + TxnManager *txn_mgr = infinity::InfinityContext::instance().storage()->txn_manager(); + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("create table")); + Vector> columns; + { + std::set constraints; + constraints.insert(ConstraintType::kNotNull); + i64 column_id = 0; + auto sparseInfo = MakeShared(EmbeddingDataType::kElemFloat, EmbeddingDataType::kElemInt32, 30000, SparseStoreType::kSort); + auto column_def_ptr = + MakeShared(column_id, MakeShared(LogicalType::kSparse, sparseInfo), "col1", constraints); + columns.emplace_back(column_def_ptr); + } + auto tbl1_def = MakeUnique(MakeShared("default_db"), MakeShared("tbl1"), columns); + auto status = txn1->CreateTable("default_db", std::move(tbl1_def), ConflictType::kError); + EXPECT_TRUE(status.ok()); + txn_mgr->CommitTxn(txn1); + } + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("create index")); + Vector columns{"col1"}; + Vector parameters; + parameters.emplace_back(new InitParameter("block_size", "16")); + parameters.emplace_back(new InitParameter("compress_type", "compress")); + + auto index_base = IndexBMP::Make(MakeShared("idx1"), "tbl1_idx1", columns, parameters); + // std::cout << "index_base: " << index_base->ToString() << std::endl; + + for (auto parameter : parameters) { + delete parameter; + } + + const String &db_name = "default_db"; + const String &table_name = "tbl1"; + auto [table_entry, table_status] = txn1->GetTableByName(db_name, table_name); + EXPECT_EQ(table_status.ok(), true); + + auto [index_entry, index_status] = txn1->CreateIndexDef(table_entry, index_base, ConflictType::kInvalid); + EXPECT_EQ(index_status.ok(), true); + auto table_ref = BaseTableRef::FakeTableRef(table_entry, txn1); + auto [_, status3] = txn1->CreateIndexPrepare(index_entry, table_ref.get(), true, true); + txn1->CreateIndexFinish(table_entry, index_entry); + EXPECT_TRUE(status3.ok()); + + txn_mgr->CommitTxn(txn1); + } + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("flush")); + const String &db_name = "default_db"; + const String &table_name = "tbl1"; + const String &index_name = "idx1"; + auto [index_entry, index_status] = txn1->GetIndexByName(db_name, table_name, index_name); + EXPECT_TRUE(index_status.ok()); + SharedPtr segment_index_entry; + EXPECT_TRUE(index_entry->GetOrCreateSegment(0, txn1, segment_index_entry));//create + EXPECT_TRUE(segment_index_entry->Flush(txn1->BeginTS())); + } + + DropTable(); +} + +TEST_P(SegmentIndexEntryTest, cleanup_test) { + TxnManager *txn_mgr = infinity::InfinityContext::instance().storage()->txn_manager(); + CreateTable(); + CreateIndex(); + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("flush")); + const String &db_name = "default_db"; + const String &table_name = "tbl1"; + const String &index_name = "fulltext_index"; + auto [index_entry, index_status] = txn1->GetIndexByName(db_name, table_name, index_name); + EXPECT_TRUE(index_status.ok()); + SharedPtr segment_index_entry; + EXPECT_TRUE(index_entry->GetOrCreateSegment(0, txn1, segment_index_entry));//create + segment_index_entry->Cleanup(); + } + + DropIndex(); + DropTable(); +} diff --git a/src/unit_test/storage/meta/entry/table_index_entry.cpp b/src/unit_test/storage/meta/entry/table_index_entry.cpp index 51d23e7164..d8ad99e000 100644 --- a/src/unit_test/storage/meta/entry/table_index_entry.cpp +++ b/src/unit_test/storage/meta/entry/table_index_entry.cpp @@ -23,8 +23,10 @@ import column_def; import data_type; import logical_type; import embedding_info; +import sparse_info; import index_hnsw; import index_full_text; +import index_bmp; import statement_common; import data_access_state; import txn_store; @@ -209,5 +211,155 @@ TEST_P(TableIndexEntryTest, get_mem_index_test){ } DropIndex(); + DropTable(); +} + +TEST_P(TableIndexEntryTest, opt_hnsw_index_test){ + TxnManager *txn_mgr = infinity::InfinityContext::instance().storage()->txn_manager(); + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("create table")); + Vector> columns; + { + std::set constraints; + constraints.insert(ConstraintType::kNotNull); + i64 column_id = 0; + auto embeddingInfo = MakeShared(EmbeddingDataType::kElemFloat, 128); + auto column_def_ptr = + MakeShared(column_id, MakeShared(LogicalType::kEmbedding, embeddingInfo), "col1", constraints); + columns.emplace_back(column_def_ptr); + } + auto tbl1_def = MakeUnique(MakeShared("default_db"), MakeShared("tbl1"), columns); + auto status = txn1->CreateTable("default_db", std::move(tbl1_def), ConflictType::kError); + EXPECT_TRUE(status.ok()); + txn_mgr->CommitTxn(txn1); + } + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("create index")); + Vector columns{"col1"}; + Vector parameters; + parameters.emplace_back(new InitParameter("metric", "l2")); + parameters.emplace_back(new InitParameter("m", "16")); + parameters.emplace_back(new InitParameter("ef_construction", "200")); + parameters.emplace_back(new InitParameter("encode", "plain")); + + auto index_base = IndexHnsw::Make(MakeShared("idx1"), "tbl1_idx1", columns, parameters); + // std::cout << "index_base: " << index_base->ToString() << std::endl; + + for (auto parameter : parameters) { + delete parameter; + } + + const String &db_name = "default_db"; + const String &table_name = "tbl1"; + auto [table_entry, table_status] = txn1->GetTableByName(db_name, table_name); + EXPECT_EQ(table_status.ok(), true); + + auto [index_entry, index_status] = txn1->CreateIndexDef(table_entry, index_base, ConflictType::kInvalid); + EXPECT_EQ(index_status.ok(), true); + auto table_ref = BaseTableRef::FakeTableRef(table_entry, txn1); + auto [_, status3] = txn1->CreateIndexPrepare(index_entry, table_ref.get(), true, true); + txn1->CreateIndexFinish(table_entry, index_entry); + EXPECT_TRUE(status3.ok()); + + txn_mgr->CommitTxn(txn1); + } + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("opt")); + const String &db_name = "default_db"; + const String &table_name = "tbl1"; + const String &index_name = "idx1"; + auto [index_entry, index_status] = txn1->GetIndexByName(db_name, table_name, index_name); + EXPECT_TRUE(index_status.ok()); + auto [table_entry, table_status] = txn1->GetTableByName(db_name, table_name); + EXPECT_TRUE(table_status.ok()); + Vector> opt_params; + opt_params.emplace_back(MakeUnique("compress_to_lvq", "")); + TxnTableStore *txn_table_store = txn1->GetTxnTableStore(table_entry); + index_entry->OptIndex(txn_table_store, opt_params, false); + + opt_params.clear(); + opt_params.emplace_back(MakeUnique("lvq_avg", "")); + index_entry->OptIndex(txn_table_store, opt_params, false); + + opt_params.clear(); + opt_params.emplace_back(MakeUnique("compress_to_lvq", "")); + opt_params.emplace_back(MakeUnique("lvq_avg", "")); + EXPECT_THROW(index_entry->OptIndex(txn_table_store, opt_params, false), RecoverableException); + txn_mgr->CommitTxn(txn1); + } + + DropTable(); +} + +TEST_P(TableIndexEntryTest, opt_bmp_index_test){ + TxnManager *txn_mgr = infinity::InfinityContext::instance().storage()->txn_manager(); + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("create table")); + Vector> columns; + { + std::set constraints; + constraints.insert(ConstraintType::kNotNull); + i64 column_id = 0; + auto sparseInfo = MakeShared(EmbeddingDataType::kElemFloat, EmbeddingDataType::kElemInt32, 30000, SparseStoreType::kSort); + auto column_def_ptr = + MakeShared(column_id, MakeShared(LogicalType::kSparse, sparseInfo), "col1", constraints); + columns.emplace_back(column_def_ptr); + } + auto tbl1_def = MakeUnique(MakeShared("default_db"), MakeShared("tbl1"), columns); + auto status = txn1->CreateTable("default_db", std::move(tbl1_def), ConflictType::kError); + EXPECT_TRUE(status.ok()); + txn_mgr->CommitTxn(txn1); + } + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("create index")); + Vector columns{"col1"}; + Vector parameters; + parameters.emplace_back(new InitParameter("block_size", "16")); + parameters.emplace_back(new InitParameter("compress_type", "compress")); + + auto index_base = IndexBMP::Make(MakeShared("idx1"), "tbl1_idx1", columns, parameters); + // std::cout << "index_base: " << index_base->ToString() << std::endl; + + for (auto parameter : parameters) { + delete parameter; + } + + const String &db_name = "default_db"; + const String &table_name = "tbl1"; + auto [table_entry, table_status] = txn1->GetTableByName(db_name, table_name); + EXPECT_EQ(table_status.ok(), true); + + auto [index_entry, index_status] = txn1->CreateIndexDef(table_entry, index_base, ConflictType::kInvalid); + EXPECT_EQ(index_status.ok(), true); + auto table_ref = BaseTableRef::FakeTableRef(table_entry, txn1); + auto [_, status3] = txn1->CreateIndexPrepare(index_entry, table_ref.get(), true, true); + txn1->CreateIndexFinish(table_entry, index_entry); + EXPECT_TRUE(status3.ok()); + + txn_mgr->CommitTxn(txn1); + } + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("opt")); + const String &db_name = "default_db"; + const String &table_name = "tbl1"; + const String &index_name = "idx1"; + auto [index_entry, index_status] = txn1->GetIndexByName(db_name, table_name, index_name); + EXPECT_TRUE(index_status.ok()); + auto [table_entry, table_status] = txn1->GetTableByName(db_name, table_name); + EXPECT_TRUE(table_status.ok()); + Vector> opt_params; + opt_params.emplace_back(MakeUnique("topk", "10")); + opt_params.emplace_back(MakeUnique("bp_reorder", "")); + TxnTableStore *txn_table_store = txn1->GetTxnTableStore(table_entry); + index_entry->OptIndex(txn_table_store, opt_params, false); + txn_mgr->CommitTxn(txn1); + } + DropTable(); } \ No newline at end of file From d0b774e58dd406b3a87ade3bc6828fca904d999b Mon Sep 17 00:00:00 2001 From: yangzq50 <58433399+yangzq50@users.noreply.github.com> Date: Fri, 30 Aug 2024 20:40:56 +0800 Subject: [PATCH 2/3] Fix ShowTable (#1775) ### What problem does this PR solve? Fix ShowTable for tables in databases except "default_db" ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue) --- src/planner/logical_planner.cpp | 50 ++++++++++++++++----------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/src/planner/logical_planner.cpp b/src/planner/logical_planner.cpp index 8900480c7a..5e1f3ce24c 100644 --- a/src/planner/logical_planner.cpp +++ b/src/planner/logical_planner.cpp @@ -1375,7 +1375,7 @@ Status LogicalPlanner::BuildShowIndexes(const ShowStatement *statement, SharedPt Status LogicalPlanner::BuildShowColumns(const ShowStatement *statement, SharedPtr &bind_context_ptr) { SharedPtr logical_show = MakeShared(bind_context_ptr->GetNewLogicalNodeId(), ShowType::kShowColumn, - query_context_ptr_->schema_name(), + statement->schema_name_, statement->table_name_, bind_context_ptr->GenerateTableIndex()); @@ -1386,7 +1386,7 @@ Status LogicalPlanner::BuildShowColumns(const ShowStatement *statement, SharedPt Status LogicalPlanner::BuildShowSegments(const ShowStatement *statement, SharedPtr &bind_context_ptr) { SharedPtr logical_show = MakeShared(bind_context_ptr->GetNewLogicalNodeId(), ShowType::kShowSegments, - query_context_ptr_->schema_name(), + statement->schema_name_, statement->table_name_, bind_context_ptr->GenerateTableIndex()); @@ -1397,7 +1397,7 @@ Status LogicalPlanner::BuildShowSegments(const ShowStatement *statement, SharedP Status LogicalPlanner::BuildShowSegment(const ShowStatement *statement, SharedPtr &bind_context_ptr) { SharedPtr logical_show = MakeShared(bind_context_ptr->GetNewLogicalNodeId(), ShowType::kShowSegment, - query_context_ptr_->schema_name(), + statement->schema_name_, statement->table_name_, bind_context_ptr->GenerateTableIndex(), statement->segment_id_); @@ -1409,7 +1409,7 @@ Status LogicalPlanner::BuildShowSegment(const ShowStatement *statement, SharedPt Status LogicalPlanner::BuildShowBlocks(const ShowStatement *statement, SharedPtr &bind_context_ptr) { SharedPtr logical_show = MakeShared(bind_context_ptr->GetNewLogicalNodeId(), ShowType::kShowBlocks, - query_context_ptr_->schema_name(), + statement->schema_name_, statement->table_name_, bind_context_ptr->GenerateTableIndex(), statement->segment_id_); @@ -1421,7 +1421,7 @@ Status LogicalPlanner::BuildShowBlocks(const ShowStatement *statement, SharedPtr Status LogicalPlanner::BuildShowBlock(const ShowStatement *statement, SharedPtr &bind_context_ptr) { SharedPtr logical_show = MakeShared(bind_context_ptr->GetNewLogicalNodeId(), ShowType::kShowBlock, - query_context_ptr_->schema_name(), + statement->schema_name_, statement->table_name_, bind_context_ptr->GenerateTableIndex(), statement->segment_id_, @@ -1434,7 +1434,7 @@ Status LogicalPlanner::BuildShowBlock(const ShowStatement *statement, SharedPtr< Status LogicalPlanner::BuildShowBlockColumn(const ShowStatement *statement, SharedPtr &bind_context_ptr) { SharedPtr logical_show = MakeShared(bind_context_ptr->GetNewLogicalNodeId(), ShowType::kShowBlockColumn, - query_context_ptr_->schema_name(), + statement->schema_name_, statement->table_name_, bind_context_ptr->GenerateTableIndex(), statement->segment_id_, @@ -1460,11 +1460,11 @@ Status LogicalPlanner::BuildShowTables(const ShowStatement *statement, SharedPtr return Status::OK(); } -Status LogicalPlanner::BuildShowViews(const ShowStatement *, SharedPtr &bind_context_ptr) { +Status LogicalPlanner::BuildShowViews(const ShowStatement *statement, SharedPtr &bind_context_ptr) { String object_name; SharedPtr logical_show = MakeShared(bind_context_ptr->GetNewLogicalNodeId(), ShowType::kShowViews, - query_context_ptr_->schema_name(), + statement->schema_name_, object_name, bind_context_ptr->GenerateTableIndex()); this->logical_plan_ = logical_show; @@ -1485,7 +1485,7 @@ Status LogicalPlanner::BuildShowDatabase(const ShowStatement *statement, SharedP Status LogicalPlanner::BuildShowTable(const ShowStatement *statement, SharedPtr &bind_context_ptr) { SharedPtr logical_show = MakeShared(bind_context_ptr->GetNewLogicalNodeId(), ShowType::kShowTable, - query_context_ptr_->schema_name(), + statement->schema_name_, statement->table_name_, bind_context_ptr->GenerateTableIndex()); this->logical_plan_ = logical_show; @@ -1495,7 +1495,7 @@ Status LogicalPlanner::BuildShowTable(const ShowStatement *statement, SharedPtr< Status LogicalPlanner::BuildShowIndex(const ShowStatement *statement, SharedPtr &bind_context_ptr) { SharedPtr logical_show = MakeShared(bind_context_ptr->GetNewLogicalNodeId(), ShowType::kShowIndex, - query_context_ptr_->schema_name(), + statement->schema_name_, statement->table_name_, bind_context_ptr->GenerateTableIndex(), None, @@ -1510,7 +1510,7 @@ Status LogicalPlanner::BuildShowIndex(const ShowStatement *statement, SharedPtr< Status LogicalPlanner::BuildShowIndexSegment(const ShowStatement *statement, SharedPtr &bind_context_ptr) { SharedPtr logical_show = MakeShared(bind_context_ptr->GetNewLogicalNodeId(), ShowType::kShowIndexSegment, - query_context_ptr_->schema_name(), + statement->schema_name_, statement->table_name_, bind_context_ptr->GenerateTableIndex(), statement->segment_id_, @@ -1525,7 +1525,7 @@ Status LogicalPlanner::BuildShowIndexSegment(const ShowStatement *statement, Sha Status LogicalPlanner::BuildShowIndexChunk(const ShowStatement *statement, SharedPtr &bind_context_ptr) { SharedPtr logical_show = MakeShared(bind_context_ptr->GetNewLogicalNodeId(), ShowType::kShowIndexChunk, - query_context_ptr_->schema_name(), + statement->schema_name_, statement->table_name_, bind_context_ptr->GenerateTableIndex(), statement->segment_id_, @@ -1537,11 +1537,11 @@ Status LogicalPlanner::BuildShowIndexChunk(const ShowStatement *statement, Share return Status::OK(); } -Status LogicalPlanner::BuildShowDatabases(const ShowStatement *, SharedPtr &bind_context_ptr) { +Status LogicalPlanner::BuildShowDatabases(const ShowStatement *statement, SharedPtr &bind_context_ptr) { String object_name; SharedPtr logical_show = MakeShared(bind_context_ptr->GetNewLogicalNodeId(), ShowType::kShowDatabases, - query_context_ptr_->schema_name(), + statement->schema_name_, object_name, bind_context_ptr->GenerateTableIndex()); this->logical_plan_ = logical_show; @@ -1551,7 +1551,7 @@ Status LogicalPlanner::BuildShowDatabases(const ShowStatement *, SharedPtr &bind_context_ptr) { SharedPtr logical_show = MakeShared(bind_context_ptr->GetNewLogicalNodeId(), ShowType::kShowSessionVariable, - query_context_ptr_->schema_name(), + statement->schema_name_, statement->var_name_, bind_context_ptr->GenerateTableIndex()); @@ -1562,7 +1562,7 @@ Status LogicalPlanner::BuildShowSessionVariable(const ShowStatement *statement, Status LogicalPlanner::BuildShowSessionVariables(const ShowStatement *statement, SharedPtr &bind_context_ptr) { SharedPtr logical_show = MakeShared(bind_context_ptr->GetNewLogicalNodeId(), ShowType::kShowSessionVariables, - query_context_ptr_->schema_name(), + statement->schema_name_, statement->var_name_, bind_context_ptr->GenerateTableIndex()); @@ -1573,7 +1573,7 @@ Status LogicalPlanner::BuildShowSessionVariables(const ShowStatement *statement, Status LogicalPlanner::BuildShowGlobalVariable(const ShowStatement *statement, SharedPtr &bind_context_ptr) { SharedPtr logical_show = MakeShared(bind_context_ptr->GetNewLogicalNodeId(), ShowType::kShowGlobalVariable, - query_context_ptr_->schema_name(), + statement->schema_name_, statement->var_name_, bind_context_ptr->GenerateTableIndex()); @@ -1584,7 +1584,7 @@ Status LogicalPlanner::BuildShowGlobalVariable(const ShowStatement *statement, S Status LogicalPlanner::BuildShowGlobalVariables(const ShowStatement *statement, SharedPtr &bind_context_ptr) { SharedPtr logical_show = MakeShared(bind_context_ptr->GetNewLogicalNodeId(), ShowType::kShowGlobalVariables, - query_context_ptr_->schema_name(), + statement->schema_name_, statement->var_name_, bind_context_ptr->GenerateTableIndex()); @@ -1595,7 +1595,7 @@ Status LogicalPlanner::BuildShowGlobalVariables(const ShowStatement *statement, Status LogicalPlanner::BuildShowConfig(const ShowStatement *statement, SharedPtr &bind_context_ptr) { SharedPtr logical_show = MakeShared(bind_context_ptr->GetNewLogicalNodeId(), ShowType::kShowConfig, - query_context_ptr_->schema_name(), + statement->schema_name_, statement->var_name_, bind_context_ptr->GenerateTableIndex()); @@ -1606,7 +1606,7 @@ Status LogicalPlanner::BuildShowConfig(const ShowStatement *statement, SharedPtr Status LogicalPlanner::BuildShowBuffer(const ShowStatement *statement, SharedPtr &bind_context_ptr) { SharedPtr logical_show = MakeShared(bind_context_ptr->GetNewLogicalNodeId(), ShowType::kShowBuffer, - query_context_ptr_->schema_name(), + statement->schema_name_, statement->var_name_, bind_context_ptr->GenerateTableIndex()); @@ -1617,7 +1617,7 @@ Status LogicalPlanner::BuildShowBuffer(const ShowStatement *statement, SharedPtr Status LogicalPlanner::BuildShowMemIndex(const ShowStatement *statement, SharedPtr &bind_context_ptr) { SharedPtr logical_show = MakeShared(bind_context_ptr->GetNewLogicalNodeId(), ShowType::kShowMemIndex, - query_context_ptr_->schema_name(), + statement->schema_name_, statement->var_name_, bind_context_ptr->GenerateTableIndex()); this->logical_plan_ = logical_show; @@ -1627,7 +1627,7 @@ Status LogicalPlanner::BuildShowMemIndex(const ShowStatement *statement, SharedP Status LogicalPlanner::BuildShowLogs(const ShowStatement *statement, SharedPtr &bind_context_ptr) { SharedPtr logical_show = MakeShared(bind_context_ptr->GetNewLogicalNodeId(), ShowType::kShowLogs, - query_context_ptr_->schema_name(), + statement->schema_name_, statement->var_name_, bind_context_ptr->GenerateTableIndex()); @@ -1638,7 +1638,7 @@ Status LogicalPlanner::BuildShowLogs(const ShowStatement *statement, SharedPtr &bind_context_ptr) { SharedPtr logical_show = MakeShared(bind_context_ptr->GetNewLogicalNodeId(), ShowType::kShowDeltaLogs, - query_context_ptr_->schema_name(), + statement->schema_name_, statement->var_name_, bind_context_ptr->GenerateTableIndex()); @@ -1649,7 +1649,7 @@ Status LogicalPlanner::BuildShowDeltaLogs(const ShowStatement *statement, Shared Status LogicalPlanner::BuildShowCatalogs(const ShowStatement *statement, SharedPtr &bind_context_ptr) { SharedPtr logical_show = MakeShared(bind_context_ptr->GetNewLogicalNodeId(), ShowType::kShowCatalogs, - query_context_ptr_->schema_name(), + statement->schema_name_, statement->var_name_, bind_context_ptr->GenerateTableIndex()); @@ -1759,7 +1759,7 @@ Status LogicalPlanner::BuildFlushBuffer(const FlushStatement *, SharedPtr &bind_context_ptr) { BindSchemaName(statement->schema_name_); SharedPtr logical_optimize = MakeShared(bind_context_ptr->GetNewLogicalNodeId(), - query_context_ptr_->schema_name(), + statement->schema_name_, statement->table_name_, std::move(statement->index_name_), std::move(statement->opt_params_)); From c8868a741e7b1ccf19c7414228564b0100b43f8e Mon Sep 17 00:00:00 2001 From: writinwaters <93570324+writinwaters@users.noreply.github.com> Date: Fri, 30 Aug 2024 20:42:36 +0800 Subject: [PATCH 3/3] Updated RESTful sample code (#1777) ### What problem does this PR solve? ### Type of change - [x] Documentation Update --- docs/references/http_api_reference.mdx | 45 +++++++++++---- docs/references/pysdk_api_reference.md | 77 +++++++++++++++++++++++++- 2 files changed, 111 insertions(+), 11 deletions(-) diff --git a/docs/references/http_api_reference.mdx b/docs/references/http_api_reference.mdx index 6e07c7a37d..56d5463134 100644 --- a/docs/references/http_api_reference.mdx +++ b/docs/references/http_api_reference.mdx @@ -324,31 +324,38 @@ curl --request POST \ }, { "name": "score", - "type": "float" + "type": "float", + "default": 3.0 }, { "name": "dense_column", - "type": "vector,8,float" + "type": "vector,8,float", + "default`: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] }, { - "name": "body", - "type": "varchar" + "name": "fulltext_column", + "type": "varchar", + "default": "" }, { "name": "sparse_column", - "type": "sparse,8,float,int" + "type": "sparse,128,float,int", + "default": {"10":1.1, "20":2.2, "30": 3.3} }, { "name": "tensor_column", - "type": "tensor,8,float" + "type": "tensor,4,float", + "default": [[1.0, 0.0, 0.0, 0.0], [1.1, 0.0, 0.0, 0.0]] }, { "name": "multivector_column", - "type": "multivector,8,float" + "type": "multivector,4,float", + "default": [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]] }, { "name": "tensorarray_column", - "type": "tensorarray,8,float" + "type": "tensorarray,2,float", + "default": [[[1.0, 1.0], [1.0, 1.0]], [[1.0, 1.0]]] } ] } ' @@ -402,6 +409,8 @@ curl --request POST \ - `"int64"` - `"float"`/`"float32"` - `"double"`/`"float64"` + - `"float16"` + - `"bfloat16"` - Sparse vector: e.g., `"sparse,128,float,int"` - `sparse`: The column is a sparse vector column. - The second item in the string: The dimension of the sparse vector. @@ -412,6 +421,8 @@ curl --request POST \ - `"int64"` - `"float"`/`"float32"` - `"double"`/`"float64"` + - `"float16"` + - `"bfloat16"` - The fourth item in the string: The data type of the sparse vector indices. Can be: - `int8` - `int16` @@ -427,6 +438,8 @@ curl --request POST \ - `"int64"` - `"float"`/`"float32"` - `"double"`/`"float64"` + - `"float16"` + - `"bfloat16"` - Tensor array: e.g., `"tensorarray,6,float"` - `tensorarray`: The column is a tensor-array column. - The second item in the string: The dimension of each vector unit in the tensor arrays. @@ -437,6 +450,8 @@ curl --request POST \ - `"int64"` - `"float"`/`"float32"` - `"double"`/`"float64"` + - `"float16"` + - `"bfloat16"` - Multivector: e.g., `"multivector,128,float"` - `multivector`: The column is a multi-vector column. - The second item in the string: The dimension of each vector unit. @@ -447,6 +462,10 @@ curl --request POST \ - `"int64"` - `"float"`/`"float32"` - `"double"`/`"float64"` + - `"float16"` + - `"bfloat16"` + - `"default"`: `Any`, *Optional* + The default value for unspecified cells in that column. ### Response @@ -1081,7 +1100,7 @@ The response includes a JSON object like the following: "indexes": [ { - "columns": "vector_column", + "columns": "vector_column_1", "index_name": "idx1", "index_type": "HNSW" }, @@ -1418,7 +1437,13 @@ curl --request POST \ }, { "name": "Neil", - "score": 2.0 + "score": 2.0, + "fulltext_column": "You're absolutely lucky.", + "dense_column": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], + "sparse_column": {"10":1.1, "20":2.2, "30": 3.3}, + "tensor_column": [[1.0, 0.0, 0.0, 0.0], [1.1, 0.0, 0.0, 0.0]], + "tensorarray_column": [[[1.0, 1.0], [1.0, 1.0]], [[1.0, 1.0]]], + "multivector_column": [[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0]] } ] ' ``` diff --git a/docs/references/pysdk_api_reference.md b/docs/references/pysdk_api_reference.md index d3cbab170c..ae01158dcb 100644 --- a/docs/references/pysdk_api_reference.md +++ b/docs/references/pysdk_api_reference.md @@ -332,7 +332,82 @@ A non-empty string indicating the name of the table, which must adhere to the fo Definitions for all table columns as a dictionary. Each key in the dictionary is a column name (`str`), with a corresponding 'value' dictionary defining the column's data type and default value information in key-value pairs: - **Data type** (`"type"`) - The data type of the column. + The data type of the column. + - Numeric: + - `"int8"` + - `"int16"` + - `"int"`/`"int32"`/`"integer"` + - `"int64"` + - `"float"`/`"float32"` + - `"double"`/`"float64"` + - `"float16"` + - `"bfloat16"` +- String: `"varchar"` +- Dense vector: e.g., `"vector,128,float"` + - `vector`: The column is a dense vector column. + - The second item in the string: The dimension of the dense vector. + - The third item in the string: The element type of the dense vector. Can be: + - `"int8"` + - `"int16"` + - `"int"`/`"int32"`/`"integer"` + - `"int64"` + - `"float"`/`"float32"` + - `"double"`/`"float64"` + - `"float16"` + - `"bfloat16"` +- Sparse vector: e.g., `"sparse,128,float,int"` + - `sparse`: The column is a sparse vector column. + - The second item in the string: The dimension of the sparse vector. + - The third item in the string: The element type of the sparse vector. Can be: + - `"int8"` + - `"int16"` + - `"int"`/`"int32"`/`"integer"` + - `"int64"` + - `"float"`/`"float32"` + - `"double"`/`"float64"` + - `"float16"` + - `"bfloat16"` + - The fourth item in the string: The data type of the sparse vector indices. Can be: + - `int8` + - `int16` + - `int`/`int32`/`integer` + - `int64` +- Tensor vector: e.g., `"tensor,4,float"` + - `tensor`: The column is a tensor column. + - The second item in the string: The dimension of each vector unit in the tensor. + - The third item in the string: The element type of the tensors. Can be: + - `"int8"` + - `"int16"` + - `"int"`/`"int32"`/`"integer"` + - `"int64"` + - `"float"`/`"float32"` + - `"double"`/`"float64"` + - `"float16"` + - `"bfloat16"` +- Tensor array: e.g., `"tensorarray,6,float"` + - `tensorarray`: The column is a tensor-array column. + - The second item in the string: The dimension of each vector unit in the tensor arrays. + - The third item in the string: The element type of the tensors. Can be: + - `"int8"` + - `"int16"` + - `"int"`/`"int32"`/`"integer"` + - `"int64"` + - `"float"`/`"float32"` + - `"double"`/`"float64"` + - `"float16"` + - `"bfloat16"` +- Multivector: e.g., `"multivector,128,float"` + - `multivector`: The column is a multi-vector column. + - The second item in the string: The dimension of each vector unit. + - The third item in the string: The element type of the tensors. Can be: + - `"int8"` + - `"int16"` + - `"int"`/`"int32"`/`"integer"` + - `"int64"` + - `"float"`/`"float32"` + - `"double"`/`"float64"` + - `"float16"` + - `"bfloat16"` - **Default value** (`"default"`) The default value for unspecified cells in that column.