Merge remote-tracking branch 'origin/unstable' into hfe

.github/workflows/kvrocks.yaml

@@ -58,7 +58,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Check typos
-        uses: crate-ci/[email protected].4
+        uses: crate-ci/[email protected].5
         with:
           config: .github/config/typos.toml
 
@@ -390,7 +390,10 @@ jobs:
     name: Check Docker image
     needs: [precondition, check-and-lint, check-typos]
     if: ${{ needs.precondition.outputs.docs_only != 'true' }}
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-24.04, ubuntu-24.04-arm]
     steps:
       - uses: actions/checkout@v4
       - name: Get core numbers
@@ -440,6 +443,10 @@ jobs:
           - name: Debian 12
             image: debian:12
             compiler: gcc
+          - name: Alpine 3
+            image: alpine:3
+            compiler: gcc
+            disable_jemalloc: -DDISABLE_JEMALLOC=ON
 
     runs-on: ubuntu-22.04
     container:
@@ -495,6 +502,13 @@ jobs:
           apt install -y bash build-essential cmake curl git libssl-dev libtool python3 python3-pip wget
           echo "NPROC=$(nproc)" >> $GITHUB_ENV
 
+      - name: Setup Alpine
+        if: ${{ startsWith(matrix.image, 'alpine') }}
+        run: |
+          apk update
+          apk add bash cmake curl git python3 wget make gcc g++ autoconf linux-headers py3-pip py3-redis
+          echo "NPROC=$(nproc)" >> $GITHUB_ENV
+
       - name: Cache redis
         id: cache-redis
         uses: actions/cache@v4
@@ -530,7 +544,7 @@ jobs:
 
       - name: Build Kvrocks
         run: |
-          ./x.py build -j$NPROC --unittest --compiler ${{ matrix.compiler }}
+          ./x.py build -j$NPROC --unittest --compiler ${{ matrix.compiler }} ${{ matrix.disable_jemalloc }}
 
       - name: Run Unit Test
         run: |
@@ -543,7 +557,7 @@ jobs:
           ./x.py test go build $GOCASE_RUN_ARGS
 
       - name: Install redis-py for openSUSE and Rocky
-        if: ${{ !startsWith(matrix.image, 'archlinux') && !startsWith(matrix.image, 'debian') }}
+        if: ${{ !startsWith(matrix.image, 'archlinux') && !startsWith(matrix.image, 'debian') && !startsWith(matrix.image, 'alpine') }}
         run: pip3 install redis==4.3.6
 
       - name: Install redis-py for Debian

.github/workflows/nightly.yaml

@@ -20,19 +20,40 @@ name: Nightly
 on:
   push:
     branches: [unstable]
+    tags: ['v2.**']
   pull_request:
     paths: ['.github/workflows/nightly.yaml']
 
 jobs:
-  publish-nightly-docker-image:
-    name: Publish nightly Docker image
-    runs-on: ubuntu-20.04
-    if: github.repository_owner == 'apache'
+  build:
+    name: "Build nightly docker images"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux/amd64
+            os: ubuntu-24.04
+          - platform: linux/arm64
+            os: ubuntu-24.04-arm
     steps:
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
 
       - uses: actions/checkout@v4
-      - name: Login Docker Hub
-        if: (github.event_name != 'pull_request')
+        with:
+          fetch-depth: 0
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: apache/kvrocks
+
+      - name: Login to Docker Hub
+        if: ${{ github.event_name != 'pull_request' }}
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
@@ -42,17 +63,59 @@ jobs:
         uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        id: buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Available platforms
-        run: echo ${{ steps.buildx.outputs.platforms }}
-
-      - name: Get core numbers
-        run: echo "NPROC=$(nproc)" >> $GITHUB_ENV
+      - name: Build and push by digest
+        id: build
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          platforms: ${{ matrix.platform }}
+          push: ${{ github.event_name != 'pull_request' }}
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,"name=apache/kvrocks",push-by-digest=true,name-canonical=true
 
-      - name: Docker meta
+      - name: Export digest
+        run: |
+          mkdir -p ${{ runner.temp }}/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "${{ runner.temp }}/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        if: ${{ github.event_name != 'pull_request' }}
+        with:
+          name: digests-${{ env.PLATFORM_PAIR }}
+          path: ${{ runner.temp }}/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge:
+    name: Merge and push nightly docker images
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name != 'pull_request' }}
+    needs:
+      - build
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ runner.temp }}/digests
+          pattern: digests-*
+          merge-multiple: true
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Docker meta (for unstable)
         id: meta
+        if: ${{ github.ref_name == 'unstable' }}
         uses: docker/metadata-action@v5
         with:
           images: apache/kvrocks
@@ -61,12 +124,28 @@ jobs:
             type=sha,prefix=nightly-{{date 'YYYYMMDD'}}-,format=short
             type=raw,value=nightly
 
-      - uses: docker/build-push-action@v6
+      - name: Docker meta (for tags)
+        id: meta_tag
+        if: ${{ github.ref_name != 'unstable' }}
+        uses: docker/metadata-action@v5
         with:
-          context: .
-          platforms: linux/amd64, linux/arm64
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          build-args: |
-            MORE_BUILD_ARGS=-j${{ env.NPROC }}
+          images: apache/kvrocks
+          flavor: latest=false
+          tags: |
+            type=sha,prefix=nightly-{{date 'YYYYMMDD'}}-{{tag}}-,format=short
+
+      - name: Create manifest list and push
+        working-directory: ${{ runner.temp }}/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf 'apache/kvrocks@sha256:%s ' *)
+
+      - name: Inspect image (for unstable)
+        if: ${{ github.ref_name == 'unstable' }}
+        run: |
+          docker buildx imagetools inspect apache/kvrocks:${{ steps.meta.outputs.version }}
+
+      - name: Inspect image (for tags)
+        if: ${{ github.ref_name != 'unstable' }}
+        run: |
+          docker buildx imagetools inspect apache/kvrocks:${{ steps.meta_tag.outputs.version }}

CMakeLists.txt

@@ -38,8 +38,6 @@ if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0")
     cmake_policy(SET CMP0135 NEW)
 endif()
 
-find_package(Backtrace REQUIRED)
-
 if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
     if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8)
         message(FATAL_ERROR "It is expected to build kvrocks with GCC 8 or above")

cmake/gtest.cmake

@@ -20,8 +20,8 @@ include_guard()
 include(cmake/utils.cmake)
 
 FetchContent_DeclareGitHubWithMirror(gtest
-  google/googletest v1.15.2
-  MD5=eb1c5c237d13ed12bf492d3997ca6b0d
+  google/googletest v1.16.0
+  MD5=92a5fd39c0952595b0ceea41805dd79d
 )
 
 FetchContent_MakeAvailableWithArgs(gtest

cmake/jsoncons.cmake

@@ -20,8 +20,8 @@ include_guard()
 include(cmake/utils.cmake)
 
 FetchContent_DeclareGitHubWithMirror(jsoncons
-  danielaparker/jsoncons v1.1.0
-  MD5=6aa6c2aec8876c49cab90be75bc545a9
+  danielaparker/jsoncons v1.2.0
+  MD5=6aa3784462eb4df848d7cdd4631d8ae9
 )
 
 FetchContent_MakeAvailableWithArgs(jsoncons

kvrocks.conf

@@ -342,10 +342,9 @@ max-bitmap-to-string-mb 16
 redis-cursor-compatible yes
 
 # Whether to enable the RESP3 protocol.
-# NOTICE: RESP3 is still under development, don't enable it in production environment.
 #
-# Default: no
-# resp3-enabled no
+# Default: yes
+# resp3-enabled yes
 
 # Maximum nesting depth allowed when parsing and serializing
 # JSON documents while using JSON commands like JSON.SET.

src/cli/signal_util.h

@@ -20,7 +20,6 @@
 
 #pragma once
 
-#include <execinfo.h>
 #include <glog/logging.h>
 #include <signal.h>
 

src/commands/cmd_hash.cc

@@ -258,6 +258,36 @@ class CommandHMSet : public Commander {
   std::vector<FieldValue> field_values_;
 };
 
+class CommandHSetExpire : public Commander {
+ public:
+  Status Parse(const std::vector<std::string> &args) override {
+    ttl_ = GET_OR_RET(ParseInt<uint64_t>(args[2], 10));
+    if ((args.size() - 3) % 2 != 0) {
+      return {Status::RedisParseErr, "Invalid number of arguments: field-value pairs must be complete"};
+    }
+    for (size_t i = 3; i < args_.size(); i += 2) {
+      field_values_.emplace_back(args_[i], args_[i + 1]);
+    }
+    return Commander::Parse(args);
+  }
+
+  Status Execute(engine::Context &ctx, Server *srv, Connection *conn, std::string *output) override {
+    uint64_t ret = 0;
+    redis::Hash hash_db(srv->storage, conn->GetNamespace());
+
+    auto s = hash_db.MSet(ctx, args_[1], field_values_, false, &ret, ttl_ * 1000 + util::GetTimeStampMS());
+    if (!s.ok()) {
+      return {Status::RedisExecErr, s.ToString()};
+    }
+    *output = redis::RESP_OK;
+    return Status::OK();
+  }
+
+ private:
+  std::vector<FieldValue> field_values_;
+  uint64_t ttl_ = 0;
+};
+
 class CommandHKeys : public Commander {
  public:
   Status Execute(engine::Context &ctx, Server *srv, Connection *conn, std::string *output) override {
@@ -680,6 +710,7 @@ REDIS_REGISTER_COMMANDS(Hash, MakeCmdAttr<CommandHGet>("hget", 3, "read-only", 1
                         MakeCmdAttr<CommandHIncrBy>("hincrby", 4, "write", 1, 1, 1),
                         MakeCmdAttr<CommandHIncrByFloat>("hincrbyfloat", 4, "write", 1, 1, 1),
                         MakeCmdAttr<CommandHMSet>("hset", -4, "write", 1, 1, 1),
+                        MakeCmdAttr<CommandHSetExpire>("hsetexpire", -5, "write", 1, 1, 1),
                         MakeCmdAttr<CommandHSetNX>("hsetnx", -4, "write", 1, 1, 1),
                         MakeCmdAttr<CommandHDel>("hdel", -3, "write no-dbsize-check", 1, 1, 1),
                         MakeCmdAttr<CommandHStrlen>("hstrlen", 3, "read-only", 1, 1, 1),

src/commands/cmd_replication.cc

@@ -240,10 +240,10 @@ class CommandFetchMeta : public Commander {
         return;
       }
       // Send full data file info
-      if (util::SockSend(repl_fd, files + CRLF, bev).IsOK()) {
+      if (auto s = util::SockSend(repl_fd, files + CRLF, bev)) {
         LOG(INFO) << "[replication] Succeed sending full data file info to " << ip;
       } else {
-        LOG(WARNING) << "[replication] Fail to send full data file info " << ip << ", error: " << strerror(errno);
+        LOG(WARNING) << "[replication] Fail to send full data file info " << ip << ", error: " << s.Msg();
       }
       auto now_secs = static_cast<time_t>(util::GetTimeStamp());
       srv->storage->SetCheckpointAccessTimeSecs(now_secs);
@@ -295,11 +295,14 @@ class CommandFetchFile : public Commander {
         if (!fd) break;
 
         // Send file size and content
-        if (util::SockSend(repl_fd, std::to_string(file_size) + CRLF, bev).IsOK() &&
-            util::SockSendFile(repl_fd, *fd, file_size, bev).IsOK()) {
+        auto s = util::SockSend(repl_fd, std::to_string(file_size) + CRLF, bev);
+        if (s) {
+          s = util::SockSendFile(repl_fd, *fd, file_size, bev);
+        }
+        if (s) {
           LOG(INFO) << "[replication] Succeed sending file " << file << " to " << ip;
         } else {
-          LOG(WARNING) << "[replication] Fail to send file " << file << " to " << ip << ", error: " << strerror(errno);
+          LOG(WARNING) << "[replication] Fail to send file " << file << " to " << ip << ", error: " << s.Msg();
           break;
         }
         fd.Close();

src/commands/cmd_server.cc

@@ -235,14 +235,11 @@ class CommandConfig : public Commander {
 class CommandInfo : public Commander {
  public:
   Status Execute([[maybe_unused]] engine::Context &ctx, Server *srv, Connection *conn, std::string *output) override {
-    std::string section = "all";
-    if (args_.size() == 2) {
-      section = util::ToLower(args_[1]);
-    } else if (args_.size() > 2) {
-      return {Status::RedisParseErr, errInvalidSyntax};
-    }
-    std::string info;
-    srv->GetInfo(conn->GetNamespace(), section, &info);
+    std::vector<std::string> sections;
+    for (size_t i = 1; i < args_.size(); ++i) {
+      sections.push_back(args_[i]);
+    }
+    auto info = srv->GetInfo(conn->GetNamespace(), sections);
     *output = conn->VerbatimString("txt", info);
     return Status::OK();
   }
@@ -285,7 +282,7 @@ class CommandRole : public Commander {
  public:
   Status Execute([[maybe_unused]] engine::Context &ctx, Server *srv, [[maybe_unused]] Connection *conn,
                  std::string *output) override {
-    srv->GetRoleInfo(output);
+    *output = srv->GetRoleInfo();
     return Status::OK();
   }
 };

src/commands/cmd_string.cc

@@ -221,7 +221,7 @@ class CommandSubStr : public CommandGetRange {
 class CommandSetRange : public Commander {
  public:
   Status Parse(const std::vector<std::string> &args) override {
-    auto parse_result = ParseInt<int>(args[2], 10);
+    auto parse_result = ParseInt<int>(args[2], {0, INT32_MAX}, 10);
     if (!parse_result) {
       return {Status::RedisParseErr, errValueNotInteger};
     }
@@ -234,6 +234,11 @@ class CommandSetRange : public Commander {
     uint64_t ret = 0;
     redis::String string_db(srv->storage, conn->GetNamespace());
 
+    auto total = offset_ + args_[3].size();
+    if (total > srv->GetConfig()->proto_max_bulk_len) {
+      return {Status::RedisExecErr, "string exceeds maximum allowed size"};
+    }
+
     auto s = string_db.SetRange(ctx, args_[1], offset_, args_[3], &ret);
     if (!s.ok()) {
       return {Status::RedisExecErr, s.ToString()};