From 8e487dc2e16b8c282aed65cbccf54fbc17073aad Mon Sep 17 00:00:00 2001 From: writinwaters <93570324+writinwaters@users.noreply.github.com> Date: Fri, 30 Aug 2024 12:31:09 +0800 Subject: [PATCH 1/3] Miscellaneous updates to the RESTful APIs (#1771) ### What problem does this PR solve? ### Type of change - [x] Documentation Update --- docs/references/http_api_reference.mdx | 223 ++++++++++++++----------- 1 file changed, 127 insertions(+), 96 deletions(-) diff --git a/docs/references/http_api_reference.mdx b/docs/references/http_api_reference.mdx index e3e47c8103..6e07c7a37d 100644 --- a/docs/references/http_api_reference.mdx +++ b/docs/references/http_api_reference.mdx @@ -976,7 +976,6 @@ Deletes an index by its name. - `content-Type: application/json` - Body: - `"drop_option"`: `enum` - ``` #### Request example @@ -1431,7 +1430,7 @@ curl --request POST \ - `table_name`: (*Path parameter*) The name of the table. - `"file_path"`: (*Body parameter*), `string`, *Required* -- `data`: (*Body parameter*), `object[]`, *Required* +- `Body`: (*Body parameter*), `object[]`, *Required* Data to insert. Infinity supports inserting multiple rows to a table at one time in the form of `object[]`. Each JSON object corresponds to a row to insert, and each key-value pair it corresponds to a column name and the table cell value. :::tip NOTE @@ -1507,16 +1506,16 @@ Deletes rows from a table based on the specified condition. curl --request DELETE \ --url http://localhost:23820/databases/{database_name}/tables/{table_name}/docs \ --header 'accept: application/json' \ - --header 'content-type: application/json' \ - --data ' {"filter": "score >= 4.0 and score <= 5.0"} ' + --header 'content-type: application/json' \ + --data ' {"filter": "score >= 4.0 and score <= 5.0"} ' ``` ```shell curl --request DELETE \ --url http://localhost:23820/databases/{database_name}/tables/{table_name}/docs \ --header 'accept: application/json' \ - --header 'content-type: application/json' \ - --data ' {"filter": "name = '"'Toby'"'"} ' + --header 'content-type: application/json' \ + --data ' {"filter": "name = '"'Toby'"'"} ' ``` #### Request parameters @@ -1525,7 +1524,7 @@ curl --request DELETE \ The name of the database. - `table_name`: (*Path parameter*) The name of the table. -- `"filter": (*Body parameter*), `string`, *Optional* +- `"filter"`: (*Body parameter*), `string`, *Optional* A string that defines the condition for selecting rows to delete. The parameter can be an expression, a function, or any other form of conditional logic that evaluates to `True` for the rows that should be deleted. If `"filter"` is not specified or set to `Null`, the method will delete all rows in the table. :::tip NOTE @@ -1615,8 +1614,8 @@ curl --request PUT \ The name of the database. - `table_name`: (*Path parameter*) The name of the table. -- `"filter": (*Body parameter*), `string`, *Optional* - A string that defines the condition for selecting rows to update. The parameter can be an expression, a function, or any other form of conditional logic that evaluates to `true` for the rows that should be deleted. If `"filter"` is not specified or set to `"Null"`, the method will update all rows in the table. +- `"filter"`: (*Body parameter*), `string`, *Optional* + A string that defines the condition for selecting rows to update. The parameter can be an expression, a function, or any other form of conditional logic that evaluates to `true` for the rows that should be updated. If `"filter"` is not specified or set to `"Null"`, the method will update all rows in the table. ### Response @@ -1677,7 +1676,7 @@ Searches for data in a specified table. The search can range from a simple vecto - `accept: application/json` - `content-Type: application/json` - Body: - - `"output"`: `list[string]` + - `"output"`: `string[]` - `"filter"`: `string` - `"fusion"`: `object` @@ -1693,9 +1692,9 @@ curl --request GET \ "output": [ "name", - "age" + "score" ], - "filter": "age > 15", + "filter": "score >= 4.0", "fusion": { "method": "rrf", @@ -1717,6 +1716,10 @@ curl --request GET \ } ' ``` +#### Request parameters + + + ### Response + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 0 +} +``` + +- `"error_code"`: `integer` + `0`: The operation succeeds. + + + + +The response includes a JSON object like the following: + +```shell +{ + "error_code": 3076, + "error_message": "Invalid command: unknown global variable {variable_name}" +} +``` + +- `"error_code"`: `integer` + A non-zero value indicates a specific error condition. +- `"error_message"`: `string` + When `error_code` is non-zero, `"error_message"` provides additional details about the error. + + + + +--- + ## Show variables **GET** `/variables` @@ -1839,7 +1923,8 @@ curl --request GET \ #### Request parameters - +- `variable_name`: (*Path parameter*) + The name of the variable. ### Response @@ -1885,32 +1970,46 @@ The response includes a JSON object like the following: --- -## Set variable +## Set config -**POST** `/variables/{variable_name}` +**PUT** `/configs/{config_name}` -Assigns a value to a global variable. +Assigns a value to a configuration parameter. ### Request -- Method: POST -- URL: `/variables/{variable_name}` +- Method: PUT +- URL: `/config/{config_name}` - Headers: - `accept: application/json` - `content-Type: application/json` -- Body: - - `"profile_record_capacity"`: `integer` +- Body: `object` #### Request example ```shell curl --request POST \ - --url http://localhost:23820/variables/{variable_name} \ + --url http://localhost:23820/configs/{config_name} \ --header 'accept: application/json' \ --header 'content-type: application/json' \ - --data ' { "profile_record_capacity" : 120 } ' + --data ' { "log_level": "trace" } ' ``` +#### Request parameter + +- `config_name`: (*Path parameter*) + The name of the configuration parameter, which must adhere to the following requirements: + - Permitted characters include: + - English letters (a-z, A-Z) + - Digits (0-9) + - "_" (underscore) + - Must begin with an English letter or underscore. + - Maximum 65,535 characters. + - Case-insensitive. +- `Body`: (*Body parameter*), `object`, *Required* + The configuration parameters to update, where each key (`string`) in the JSON object represents an entry, and the corresponding value (`string`) represents its value. + + ### Response ---- - -## Set config - -**PUT** `/configs/{config_name}` - -Assigns a value to a config. - -### Request - -- Method: PUT -- URL: `/config/{config_name}` -- Headers: - - `accept: application/json` - - `content-Type: application/json` -- Body: - - `"log_level"`: `enum` - -#### Request example - -```shell -curl --request POST \ - --url http://localhost:23820/configs/{config_name} \ - --header 'accept: application/json' \ - --header 'content-type: application/json' \ - --data ' { "log_level": "trace" } ' -``` - -### Response - - - - -The response includes a JSON object like the following: - -```shell -{ - "error_code": 0 -} -``` - -- `"error_code"`: `integer` - `0`: The operation succeeds. - - - - -The response includes a JSON object like the following: - -```shell -{ - "error_code": 3028, - "error_message": "log level value range is trace, debug, info, warning, error, critical" -} -``` - -- `"error_code"`: `integer` - A non-zero value indicates a specific error condition. -- `"error_message"`: `string` - When `error_code` is non-zero, `"error_message"` provides additional details about the error. - - - - --- \ No newline at end of file From 4b10067d8a1d8f87ee7916fe152b8e3e985d9bd6 Mon Sep 17 00:00:00 2001 From: Zhiyuan Liang <132966438+Ami11111@users.noreply.github.com> Date: Fri, 30 Aug 2024 13:41:41 +0800 Subject: [PATCH 2/3] Add segment_entry test cases for unit test (#1772) ### What problem does this PR solve? Add segment_entry test cases for unit test ### Type of change - [x] Test cases --- .../storage/meta/entry/segment_entry.cpp | 314 ++++++++++++++++++ 1 file changed, 314 insertions(+) create mode 100644 src/unit_test/storage/meta/entry/segment_entry.cpp diff --git a/src/unit_test/storage/meta/entry/segment_entry.cpp b/src/unit_test/storage/meta/entry/segment_entry.cpp new file mode 100644 index 0000000000..357e545530 --- /dev/null +++ b/src/unit_test/storage/meta/entry/segment_entry.cpp @@ -0,0 +1,314 @@ +#include "unit_test/base_test.h" +#include "type/complex/embedding_type.h" +#include + +import infinity_context; +import infinity_exception; + +import stl; +import global_resource_usage; +import third_party; +import logger; +import table_def; +import value; +import bitmask; + +import data_block; +import default_values; +import txn_manager; +import buffer_manager; +import txn; +import catalog; +import status; +import extra_ddl_info; +import column_def; +import data_type; +import logical_type; +import embedding_info; +import index_hnsw; +import index_full_text; +import statement_common; +import data_access_state; +import txn_store; +import column_vector; +import internal_types; +import constant_expr; +import parsed_expr; + +import base_entry; +import table_entry; +import table_entry_type; +import segment_entry; +import block_entry; + +using namespace infinity; + +class SegmentEntryTest : public BaseTestParamStr { + void SetUp() override { + BaseTestParamStr::SetUp(); +#ifdef INFINITY_DEBUG + infinity::GlobalResourceUsage::Init(); +#endif + std::shared_ptr config_path = nullptr; + RemoveDbDirs(); + system(("mkdir -p " + infinity::String(GetFullPersistDir())).c_str()); + system(("mkdir -p " + infinity::String(GetFullDataDir())).c_str()); + system(("mkdir -p " + infinity::String(GetFullTmpDir())).c_str()); + std::string config_path_str = GetParam(); + if (config_path_str != BaseTestParamStr::NULL_CONFIG_PATH) { + config_path = infinity::MakeShared(config_path_str); + } + infinity::InfinityContext::instance().Init(config_path); + } + + void TearDown() override { + infinity::InfinityContext::instance().UnInit(); +#ifdef INFINITY_DEBUG + EXPECT_EQ(infinity::GlobalResourceUsage::GetObjectCount(), 0); + EXPECT_EQ(infinity::GlobalResourceUsage::GetRawMemoryCount(), 0); + infinity::GlobalResourceUsage::UnInit(); +#endif + BaseTestParamStr::TearDown(); + } +}; + +INSTANTIATE_TEST_SUITE_P(TestWithDifferentParams, + SegmentEntryTest, + ::testing::Values(BaseTestParamStr::NULL_CONFIG_PATH, + BaseTestParamStr::VFS_CONFIG_PATH)); + +void CreateTable(); +void CreateIndex(); +void InsertData(const String& db_name, const String& table_name); +void DropIndex(); +void DropTable(); + +TEST_P(SegmentEntryTest, decode_index_test) { + TxnManager *txn_mgr = infinity::InfinityContext::instance().storage()->txn_manager(); + CreateTable(); + InsertData("default_db", "tbl1"); + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("decode")); + const String &db_name = "default_db"; + const String &table_name = "tbl1"; + auto [table_entry, table_status] = txn1->GetTableByName(db_name, table_name); + EXPECT_TRUE(table_status.ok()); + auto segment_entry = table_entry->GetSegmentEntry(0); + EXPECT_TRUE(segment_entry != nullptr); + String encoded_index = SegmentEntry::EncodeIndex(0, table_entry); + std::cout<<"encoded_index: "< decoded_index = SegmentEntry::DecodeIndex(encoded_index); + EXPECT_TRUE(decoded_index[0] == "default_db"); + EXPECT_TRUE(decoded_index[1] == "tbl1"); + EXPECT_TRUE(decoded_index[2] == "0"); + EXPECT_THROW(SegmentEntry::DecodeIndex("/default_db/tbl1/0"), UnrecoverableException); + } + + DropTable(); +} + +TEST(ToStringTest, segment_status_test) { + EXPECT_STREQ(ToString(SegmentStatus::kUnsealed).c_str(), "Unsealed"); + EXPECT_STREQ(ToString(SegmentStatus::kSealed).c_str(), "Sealed"); + EXPECT_STREQ(ToString(SegmentStatus::kCompacting).c_str(), "Compacting"); + EXPECT_STREQ(ToString(SegmentStatus::kNoDelete).c_str(), "NoDelete"); + EXPECT_STREQ(ToString(SegmentStatus::kDeprecated).c_str(), "Deprecated"); +} + +TEST(ToStringTest, segment_status_to_string_test) { + EXPECT_STREQ(SegmentEntry::SegmentStatusToString(SegmentStatus::kUnsealed).c_str(), "Unsealed"); + EXPECT_STREQ(SegmentEntry::SegmentStatusToString(SegmentStatus::kSealed).c_str(), "Sealed"); + EXPECT_STREQ(SegmentEntry::SegmentStatusToString(SegmentStatus::kCompacting).c_str(), "Compacting"); + EXPECT_STREQ(SegmentEntry::SegmentStatusToString(SegmentStatus::kNoDelete).c_str(), "NoDelete"); + EXPECT_STREQ(SegmentEntry::SegmentStatusToString(SegmentStatus::kDeprecated).c_str(), "Deprecated"); + EXPECT_STREQ(SegmentEntry::SegmentStatusToString(static_cast(5)).c_str(), "Invalid Status"); +} + +TEST_P(SegmentEntryTest, segment_entry_to_string_test) { + TxnManager *txn_mgr = infinity::InfinityContext::instance().storage()->txn_manager(); + CreateTable(); + InsertData("default_db", "tbl1"); + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("set")); + const String &db_name = "default_db"; + const String &table_name = "tbl1"; + auto [table_entry, table_status] = txn1->GetTableByName(db_name, table_name); + EXPECT_TRUE(table_status.ok()); + auto segment_entry = table_entry->GetSegmentEntry(0); + String segment_path = segment_entry->ToString(); + std::cout<txn_manager(); + CreateTable(); + InsertData("default_db", "tbl1"); + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("set")); + const String &db_name = "default_db"; + const String &table_name = "tbl1"; + auto [table_entry, table_status] = txn1->GetTableByName(db_name, table_name); + EXPECT_TRUE(table_status.ok()); + auto segment_entry = table_entry->GetSegmentEntry(0); + std::cout<<"segment_entry_status: "<status())<SetSealed()); + } + + DropTable(); +} + +TEST_P(SegmentEntryTest, set_compacting_test) { + TxnManager *txn_mgr = infinity::InfinityContext::instance().storage()->txn_manager(); + CreateTable(); + InsertData("default_db", "tbl1"); + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("set")); + const String &db_name = "default_db"; + const String &table_name = "tbl1"; + auto [table_entry, table_status] = txn1->GetTableByName(db_name, table_name); + EXPECT_TRUE(table_status.ok()); + auto segment_entry = table_entry->GetSegmentEntry(0); + std::cout<<"segment_entry_status: "<status())<TrySetCompacting(nullptr)); + EXPECT_FALSE(segment_entry->TrySetCompacting(nullptr)); + } + + DropTable(); +} + +TEST_P(SegmentEntryTest, set_no_delete_test) { + TxnManager *txn_mgr = infinity::InfinityContext::instance().storage()->txn_manager(); + CreateTable(); + InsertData("default_db", "tbl1"); + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("set")); + const String &db_name = "default_db"; + const String &table_name = "tbl1"; + auto [table_entry, table_status] = txn1->GetTableByName(db_name, table_name); + EXPECT_TRUE(table_status.ok()); + auto segment_entry = table_entry->GetSegmentEntry(0); + std::cout<<"segment_entry_status: "<status())<SetNoDelete(), UnrecoverableException); + EXPECT_TRUE(segment_entry->TrySetCompacting(nullptr)); + EXPECT_TRUE(segment_entry->SetNoDelete()); + } + + DropTable(); +} + +TEST_P(SegmentEntryTest, set_deprecated_test) { + TxnManager *txn_mgr = infinity::InfinityContext::instance().storage()->txn_manager(); + CreateTable(); + InsertData("default_db", "tbl1"); + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("set")); + const String &db_name = "default_db"; + const String &table_name = "tbl1"; + auto [table_entry, table_status] = txn1->GetTableByName(db_name, table_name); + EXPECT_TRUE(table_status.ok()); + auto segment_entry = table_entry->GetSegmentEntry(0); + std::cout<<"segment_entry_status: "<status())<SetDeprecated(txn1->BeginTS()), UnrecoverableException); + EXPECT_TRUE(segment_entry->TrySetCompacting(nullptr)); + EXPECT_TRUE(segment_entry->SetNoDelete()); + segment_entry->SetDeprecated(txn1->BeginTS()); + } + + DropTable(); +} + +TEST_P(SegmentEntryTest, roll_back_compact_test) { + TxnManager *txn_mgr = infinity::InfinityContext::instance().storage()->txn_manager(); + CreateTable(); + InsertData("default_db", "tbl1"); + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("set")); + const String &db_name = "default_db"; + const String &table_name = "tbl1"; + auto [table_entry, table_status] = txn1->GetTableByName(db_name, table_name); + EXPECT_TRUE(table_status.ok()); + auto segment_entry = table_entry->GetSegmentEntry(0); + std::cout<<"segment_entry_status: "<status())<RollbackCompact(), UnrecoverableException); + EXPECT_TRUE(segment_entry->TrySetCompacting(nullptr)); + EXPECT_THROW(segment_entry->RollbackCompact(), UnrecoverableException); + EXPECT_TRUE(segment_entry->SetNoDelete()); + segment_entry->RollbackCompact(); + } + + DropTable(); +} + +TEST_P(SegmentEntryTest, check_row_visible_test) { + TxnManager *txn_mgr = infinity::InfinityContext::instance().storage()->txn_manager(); + CreateTable(); + InsertData("default_db", "tbl1"); + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("set")); + const String &db_name = "default_db"; + const String &table_name = "tbl1"; + auto [table_entry, table_status] = txn1->GetTableByName(db_name, table_name); + EXPECT_TRUE(table_status.ok()); + auto segment_entry = table_entry->GetSegmentEntry(0); + EXPECT_TRUE(segment_entry->CheckRowVisible(0, txn1->BeginTS(), true)); + } + + DropTable(); +} + +TEST_P(SegmentEntryTest, check_rows_visible_test) { + TxnManager *txn_mgr = infinity::InfinityContext::instance().storage()->txn_manager(); + CreateTable(); + InsertData("default_db", "tbl1"); + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("set")); + const String &db_name = "default_db"; + const String &table_name = "tbl1"; + auto [table_entry, table_status] = txn1->GetTableByName(db_name, table_name); + EXPECT_TRUE(table_status.ok()); + auto segment_entry = table_entry->GetSegmentEntry(0); + Vector segment_offsets = {0}; + segment_entry->CheckRowsVisible(segment_offsets, txn1->BeginTS()); + SharedPtr offsets = Bitmask::Make(4); + segment_entry->CheckRowsVisible(*offsets, txn1->BeginTS()); + } + + DropTable(); +} + +TEST_P(SegmentEntryTest, append_data_test) { + TxnManager *txn_mgr = infinity::InfinityContext::instance().storage()->txn_manager(); + BufferManager *buffer_mgr = infinity::InfinityContext::instance().storage()->buffer_manager(); + CreateTable(); + InsertData("default_db", "tbl1"); + + { + auto *txn1 = txn_mgr->BeginTxn(MakeUnique("set")); + const String &db_name = "default_db"; + const String &table_name = "tbl1"; + auto [table_entry, table_status] = txn1->GetTableByName(db_name, table_name); + EXPECT_TRUE(table_status.ok()); + auto segment_entry = table_entry->GetSegmentEntry(0); + auto txn_store = txn1->GetTxnTableStore(table_entry); + txn_store->SetAppendState(MakeUnique(txn_store->GetBlocks())); + auto append_state = txn_store->GetAppendState(); + EXPECT_THROW(segment_entry->AppendData(txn1->TxnID(), txn1->BeginTS(), append_state, buffer_mgr, txn1), UnrecoverableException); + } + + DropTable(); +} \ No newline at end of file From da427c239db27f07224a3bed2905546690946d36 Mon Sep 17 00:00:00 2001 From: yangzq50 <58433399+yangzq50@users.noreply.github.com> Date: Fri, 30 Aug 2024 13:44:49 +0800 Subject: [PATCH 3/3] Update python test check_data (#1773) ### What problem does this PR solve? Update python test check_data function Ignore conflict if multiple test jobs create the same dir repeatedly ### Type of change - [x] Test cases --- python/parallel_test/conftest.py | 2 +- python/test_http_api/httputils.py | 2 +- python/test_pysdk/conftest.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/parallel_test/conftest.py b/python/parallel_test/conftest.py index decd326ba7..e57c7e6da0 100644 --- a/python/parallel_test/conftest.py +++ b/python/parallel_test/conftest.py @@ -59,7 +59,7 @@ def check_data(request): data_dir = request.param["data_dir"] # path not exists if not os.path.exists(data_dir): - os.makedirs(data_dir) + os.makedirs(data_dir, exist_ok=True) return False if not os.path.exists(data_dir + file_name): return False diff --git a/python/test_http_api/httputils.py b/python/test_http_api/httputils.py index 43732c809e..319b283082 100644 --- a/python/test_http_api/httputils.py +++ b/python/test_http_api/httputils.py @@ -124,5 +124,5 @@ def generate_commas_enwiki(in_filename, out_filename, is_embedding): def check_data(data_dir): # path not exists if not os.path.exists(data_dir): - os.makedirs(data_dir) + os.makedirs(data_dir, exist_ok=True) return False diff --git a/python/test_pysdk/conftest.py b/python/test_pysdk/conftest.py index 46a2c46132..2a56933077 100644 --- a/python/test_pysdk/conftest.py +++ b/python/test_pysdk/conftest.py @@ -107,7 +107,7 @@ def check_data(request): data_dir = request.param["data_dir"] # path not exists if not os.path.exists(data_dir): - os.makedirs(data_dir) + os.makedirs(data_dir, exist_ok=True) return False if not os.path.exists(data_dir + file_name): return False