Merge remote-tracking branch 'upstream/main' into unit_test

.github/workflows/tests.yml

@@ -169,7 +169,7 @@ jobs:
       - name: Start builder container
         if: ${{ !cancelled() && !failure() }}
         run: |
-          BUILDER_CONTAINER=infinity_build_$(od -An -N4 -t u8 < /dev/urandom | tr -d ' ')
+          BUILDER_CONTAINER=infinity_build_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
           CPUS=${CPUS:-$(nproc)}
           echo "BUILDER_CONTAINER=${BUILDER_CONTAINER}" >> $GITHUB_ENV
           echo "CPUS=${CPUS}" >> $GITHUB_ENV
@@ -261,23 +261,30 @@ jobs:
         if: ${{ !cancelled() && !failure() }}
         run: |
           RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX:-$HOME}
+          echo "RUNNER_WORKSPACE_PREFIX=${RUNNER_WORKSPACE_PREFIX}" >> $GITHUB_ENV
           sudo chmod +x ./tools/ci_tools/check_benchmark_result.py && sudo mkdir -p test/data/benchmark && sudo ln -s ${RUNNER_WORKSPACE_PREFIX}/benchmark/sift1M test/data/benchmark/sift_1m
 
-      - name: Generate config file
-        if: ${{ !cancelled() && !failure() }}
-        run: mkdir -p $PWD/db_tmp && cat conf/infinity_conf.toml | sed -e "s|/var/infinity|$PWD/db_tmp|g" > $PWD/db_tmp/infinity_conf.toml
-
-      - name: Import sift dataset
-        if: ${{ !cancelled() && !failure() }}
-        run: ./cmake-build-release/benchmark/local_infinity/knn_import_benchmark sift $PWD/test/data $PWD/db_tmp
-
-      - name: Benchmark sift search 1 thread repeat 50 times
+      - name: Golden benchmark sift
         if: ${{ !cancelled() && !failure() }}
-        run: rm -f benchmark_sift_1_thread_latest_output.log ; echo "1 50" | ./cmake-build-release/benchmark/local_infinity/knn_query_benchmark sift 200 false $PWD/test/data $PWD/db_tmp | tee benchmark_sift_1_thread_latest_output.log ; cat benchmark_sift_1_thread_latest_output.log | ./tools/ci_tools/collect_benchmark_result.py benchmark_sift_1_thread.log ; ./tools/ci_tools/check_benchmark_result.py sift_1 benchmark_sift_1_thread.log
-
-      - name: Benchmark sift search 8 threads repeat 50 times
+        run: |
+            if [[ -f "${RUNNER_WORKSPACE_PREFIX}/benchmark/golden_benchmark_sift_1_thread.log" ]]; then
+                echo "golden benchmark result already exists!"
+                exit 0
+            fi
+            rm -fr $PWD/db_tmp && mkdir -p $PWD/db_tmp && cat ${RUNNER_WORKSPACE_PREFIX}/benchmark/infinity_conf.toml | sed -e "s|/var/infinity|$PWD/db_tmp|g" > $PWD/db_tmp/infinity_conf.toml && \
+            ${RUNNER_WORKSPACE_PREFIX}/benchmark/knn_import_benchmark sift $PWD/test/data $PWD/db_tmp && \
+            echo "1 50" | ${RUNNER_WORKSPACE_PREFIX}/benchmark/knn_query_benchmark sift 200 false $PWD/test/data $PWD/db_tmp | sudo tee ${RUNNER_WORKSPACE_PREFIX}/benchmark/golden_benchmark_sift_1_thread.log && \
+            echo "8 50" | ${RUNNER_WORKSPACE_PREFIX}/benchmark/knn_query_benchmark sift 200 false $PWD/test/data $PWD/db_tmp | sudo tee ${RUNNER_WORKSPACE_PREFIX}/benchmark/golden_benchmark_sift_8_threads.log
+
+      - name: Latest benchmark sift
         if: ${{ !cancelled() && !failure() }}
-        run: rm -f benchmark_sift_8_threads_latest_output.log ; echo "8 50" | ./cmake-build-release/benchmark/local_infinity/knn_query_benchmark sift 200 false $PWD/test/data $PWD/db_tmp | tee benchmark_sift_8_threads_latest_output.log ; cat benchmark_sift_8_threads_latest_output.log | ./tools/ci_tools/collect_benchmark_result.py benchmark_sift_8_threads.log ; ./tools/ci_tools/check_benchmark_result.py sift_8 benchmark_sift_8_threads.log
+        run: |
+            rm -fr $PWD/db_tmp && mkdir -p $PWD/db_tmp && cat conf/infinity_conf.toml | sed -e "s|/var/infinity|$PWD/db_tmp|g" > $PWD/db_tmp/infinity_conf.toml && \
+            ./cmake-build-release/benchmark/local_infinity/knn_import_benchmark sift $PWD/test/data $PWD/db_tmp && \
+            echo "1 50" | ./cmake-build-release/benchmark/local_infinity/knn_query_benchmark sift 200 false $PWD/test/data $PWD/db_tmp | tee benchmark_sift_1_thread.log && \
+            echo "8 50" | ./cmake-build-release/benchmark/local_infinity/knn_query_benchmark sift 200 false $PWD/test/data $PWD/db_tmp | tee benchmark_sift_8_threads.log && \
+            ./tools/ci_tools/check_benchmark_result.py ${RUNNER_WORKSPACE_PREFIX}/benchmark/golden_benchmark_sift_1_thread.log benchmark_sift_1_thread.log && \
+            ./tools/ci_tools/check_benchmark_result.py ${RUNNER_WORKSPACE_PREFIX}/benchmark/golden_benchmark_sift_8_threads.log benchmark_sift_8_threads.log
 
       - name: Destroy builder container
         if: always()  # always run this step even if previous steps failed

README.md

@@ -60,7 +60,7 @@ Supports a wide range of data types including strings, numerics, vectors, and mo
 Infinity, also available as a Python module, eliminates the need for a separate back-end server and all the complex communication settings. Using `pip install` and `import infinity`, you can quickly build a local AI application in Python, leveraging the world's fastest and the most powerful RAG database:
 
    ```bash
-   pip install infinity-sdk==0.3.0.dev6
+   pip install infinity-sdk==0.3.0.dev7
    ```
 
    ```python

benchmark/remote_infinity/remote_query_benchmark.cpp

@@ -51,7 +51,7 @@ struct InfinityClient {
         transport->open();
         CommonResponse response;
         ConnectRequest request;
-        request.__set_client_version(15); // 0.3.0.dev6
+        request.__set_client_version(16); // 0.3.0.dev7
         client->Connect(response, request);
         session_id = response.session_id;
     }

client/cpp/infinity_client.cpp

@@ -25,7 +25,7 @@ Client Client::Connect(const std::string &ip_address, uint16_t port) {
     transport->open();
     CommonResponse response;
     ConnectRequest request;
-    request.__set_client_version(15); // 0.3.0.dev6
+    request.__set_client_version(16); // 0.3.0.dev7
     client->Connect(response, request);
     return {socket, transport, protocol, std::move(client), response.session_id};
 }

docs/getstarted/deploy_infinity_server.mdx

@@ -30,7 +30,7 @@ This approach allows you to embed Infinity as a module in a Python application.
 ### Install Infinity as a module
 
 ```
-pip install infinity-sdk==0.3.0.dev6
+pip install infinity-sdk==0.3.0.dev7
 ```
 
 ### Create an Infinity object
@@ -98,7 +98,7 @@ If you are on Windows 10+, you must enable WSL or WSL2 to deploy Infinity using
 ### Install Infinity client
 
 ```
-pip install infinity-sdk==0.3.0.dev6
+pip install infinity-sdk==0.3.0.dev7
 ```
 
 ### Connect to Infinity Server 
@@ -140,7 +140,7 @@ This section provides instructions on deploying Infinity using binary package on
 
 Fedora/RHEL/CentOS/OpenSUSE
 ```bash
-sudo rpm -i infinity-0.3.0.dev6-x86_64.rpm
+sudo rpm -i infinity-0.3.0.dev7-x86_64.rpm
 ```
 
 ```bash
@@ -151,7 +151,7 @@ sudo systemctl start infinity
   <TabItem value="ubuntu">
 
 ```bash
-sudo dpkg -i infinity-0.3.0.dev6-x86_64.deb
+sudo dpkg -i infinity-0.3.0.dev7-x86_64.deb
 ```
 
 ```bash
@@ -172,7 +172,7 @@ sudo systemctl start infinity
 ### Install Infinity client
 
 ```
-pip install infinity-sdk==0.3.0.dev6
+pip install infinity-sdk==0.3.0.dev7
 ```
 
 ### Connect to Infinity Server 

docs/getstarted/quickstart.md

@@ -17,7 +17,7 @@ Infinity, also available as a Python module, eliminates the need for a separate
 
 1. Install Infinity as a Python module:
    ```bash
-   pip install infinity-sdk==0.3.0.dev6
+   pip install infinity-sdk==0.3.0.dev7
    ```
 2. Use Infinity to conduct a KNN search:
    ```python

docs/references/configurations.md

@@ -0,0 +1,147 @@
+---
+sidebar_position: 5
+slug: /configurations
+---
+
+# Configurations
+
+If you need to change or customize the behavior of the Infinity runtime, you can use a configuration file. By default, Infinity uses the default configuration. To specify your own configuration file, you can use the `--config path/to/config.toml` option.
+
+When Infinity starts up, it determines and loads each item in the configuration in a specific order, and merges these configurations from different sources into a single configuration used by Infinity.
+
+## Configuration Loading Order and Priority
+
+When Infinity starts up, configurations are loaded and determined in the following order:
+
+1. Default values embedded in the code.
+
+2. Values from the configuration file provided with --config PATH (if exists).
+
+The loading methods listed above have increasing priority, so a higher-priority source will override values loaded from a lower-priority source. If the provided configuration file is not found, Infinity will display a warning at startup. The whole configuration is also displayed upon startup.
+
+## Configuration file example
+
+
+```toml
+[general]
+# version of Infinity Server, defaults to version in source code
+version                  = "0.3.0"
+# timezone
+time_zone                = "utc-8"
+# number of worker threads, defaults to concurrent threads supported by compiler implementation
+# range : [1, 16384]
+cpu_limit                = 8
+
+# Network-related configuration
+[network]
+# IP address of infinity server
+server_address           = "0.0.0.0"
+
+# Range for ports : [1024, 65535]
+# Port of postgres server
+postgres_port            = 5432
+# Port of HTTP server
+http_port                = 23820
+# Port of Python SDK server
+client_port              = 23817
+# the maximum number of connections, defaults to 256
+# range : [1, 65536]
+connection_pool_size     = 128
+
+# Log related configurations
+[log]
+# filename of log files
+log_filename             = "infinity.log"
+# directory of log files
+log_dir                  = "/var/infinity/log"
+# controls if the log should be printed to standard output
+# range : {true|false} 
+log_to_stdout            = false
+# The maximum size of a single log file, defaults to 1GB
+# range : at least 1GB
+log_file_max_size        = "10GB"
+# The number of rotating log files, defaults to 8
+# range : [1, 655636]
+log_file_rotate_count    = 10
+# The level of output log, defaults to info
+# range : {trace/debug/info/warning/error/critical}
+log_level               = "info"
+
+# storage-related configuration
+[storage]
+# The directory of data files
+data_dir                 = "/var/infinity/data"
+
+# periodically activates garbage collection:
+# use "number + unit of time" to specify intervals
+# s means seconds, for example "60s", 60 seconds
+# m means minutes, for example "60m", 60 minutes
+# h means hours, for example "1h", 1 hour
+# set to "0s" to disable corresponding periodic tasks
+
+# interval between index optimization tasks
+# range : ["10s", "720h"]
+optimize_interval        = "10s"
+# interval between cleanup tasks
+# range : ["10s", "720h"]
+cleanup_interval         = "60s"
+# interval between compaction tasks
+# range : ["10s", "720h"]
+compact_interval         = "120s"
+# The threshold for flushing a single in-memory index to disk,
+# when the number of rows the index includes exceeds this threshold,
+# the system performs a flush operation for that in-memory index.
+# range : [8192, 8388608]
+mem_index_capacity       = 1048576
+
+# buffer manager-related configurations
+[buffer]
+# the size of memory that buffer manager holds, defaults to 8GB
+buffer_manager_size      = "4GB"
+# the number of LRU caches in buffer manager
+# range : [1, 100]
+lru_num                  = 7
+# When the required memory exceeds the size of the free memory in the buffer manager,
+# the buffer manager dumps some of the its in-use memory to a temporary storage path specified by temp_dir.
+temp_dir                 = "/var/infinity/tmp"
+# The global memory index threshold triggers a flush to disk.
+# When the memory used by all existing in-memory indices in the system exceeds this threshold
+# the system performs a flush operation for all in-memory indices.
+memindex_memory_quota   = "1GB"
+
+# WAL related configuration
+[wal]
+# directory of WAL files
+wal_dir                       = "/var/infinity/wal"
+# interval between periodically full checkpoints
+# range : ["0s", "720h"]
+full_checkpoint_interval      = "86400s"
+# interval between periodically delta(incremental) checkpoints
+# range : ["0s", "720h"]
+delta_checkpoint_interval     = "60s"
+
+# the size threshold of WAL files for triggering compaction
+# after a WAL file grows bigger than this threshold, the system will perform a compaction to it
+# range : ["1KB", "1024GB"]
+wal_compact_threshold         = "1GB"
+
+# controls how WAL manager flush its logs to the disk
+# flush_at_once: write and flush log each commit
+# only_write: write log, OS control when to flush the log, default
+# flush_per_second: logs are written after each commit and flushed to disk per second.
+wal_flush                     = "only_write"
+
+[resource]
+# The resource files used by Infinity (such as the dictionary files used by the analyzer)
+resource_dir                  = "/var/infinity/resource"
+
+[persistence]
+# Persistent File Storage Path
+# When set to "" or not provided, persistent manager is diabled, and files are stored directly on the local file system.
+# When not empty, persistent manager is enabled. The generated files will be stored by the persistence manager on the path specified by persistence_dir.
+# For example, if the user adds three files: A, B, and C.
+# When persistence manager is disabled, the three files are stored on the local file system in their specified raw paths: A, B, C.
+# When persistence manager is enabled, the persistence manager determines how to store these files on the specified persistent path.
+# toggle persistence on to reduce possible file fragments in your local filesystem
+persistence_dir          = "/var/infinity/persistence"
+```