*: fix client log (tikv#8060)

.github/workflows/pd-tests.yaml

@@ -25,9 +25,19 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        worker_id: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
+        include:
+          - worker_id: 1
+            name: 'Unit Test'
+          - worker_id: 2
+            name: 'Tools Test'
+          - worker_id: 3
+            name: 'Client Integration Test'
+          - worker_id: 4
+            name: 'TSO Integration Test'
+          - worker_id: 5
+            name: 'MicroService Integration Test'
     outputs:
-      job-total: 13
+      job-total: 5
     steps:
       - uses: actions/setup-go@v3
         with:
@@ -43,15 +53,13 @@ jobs:
             **/.tools
             **/.dashboard_download_cache
           key: ${{ runner.os }}-go-${{ matrix.worker_id }}-${{ hashFiles('**/go.sum') }}
-      - name: Make Test
+      - name: ${{ matrix.name }}
         env:
           WORKER_ID: ${{ matrix.worker_id }}
-          WORKER_COUNT: 13
-          JOB_COUNT: 9 # 10 is tools test, 11, 12, 13 are for other integrations jobs
+          WORKER_COUNT: 10
         run: |
-          make ci-test-job JOB_COUNT=$(($JOB_COUNT)) JOB_INDEX=$WORKER_ID
+          make ci-test-job JOB_INDEX=1
           mv covprofile covprofile_$WORKER_ID
-          sed -i "/failpoint_binding/d" covprofile_$WORKER_ID
       - name: Upload coverage result ${{ matrix.worker_id }}
         uses: actions/upload-artifact@v2
         with:
@@ -70,7 +78,11 @@ jobs:
       - name: Merge
         env:
           TOTAL_JOBS: ${{needs.chunks.outputs.job-total}}
-        run: for i in $(seq 1 $TOTAL_JOBS); do cat covprofile_$i >> covprofile; done
+        run: |
+          for i in $(seq 1 $TOTAL_JOBS); do cat covprofile_$i >> covprofile; done
+          sed -i "/failpoint_binding/d" covprofile
+          # only keep the first line(`mode: aomic`) of the coverage profile
+          sed -i '2,${/mode: atomic/d;}' covprofile
       - name: Send coverage
         uses: codecov/[email protected]
         with:

Makefile

@@ -127,7 +127,7 @@ regions-dump:
 stores-dump:
 	cd tools && CGO_ENABLED=0 go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/stores-dump stores-dump/main.go
 pd-ut: pd-xprog
-	cd tools && GOEXPERIMENT=$(BUILD_GOEXPERIMENT) CGO_ENABLED=$(BUILD_TOOL_CGO_ENABLED) go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-ut pd-ut/ut.go
+	cd tools && GOEXPERIMENT=$(BUILD_GOEXPERIMENT) CGO_ENABLED=$(BUILD_TOOL_CGO_ENABLED) go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-ut pd-ut/ut.go pd-ut/coverProfile.go
 pd-xprog:
 	cd tools && GOEXPERIMENT=$(BUILD_GOEXPERIMENT) CGO_ENABLED=$(BUILD_TOOL_CGO_ENABLED) go build -tags xprog -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/xprog pd-ut/xprog.go
 
@@ -251,9 +251,10 @@ basic-test: install-tools
 	go test $(BASIC_TEST_PKGS) || { $(FAILPOINT_DISABLE); exit 1; }
 	@$(FAILPOINT_DISABLE)
 
-ci-test-job: install-tools dashboard-ui
+ci-test-job: install-tools dashboard-ui pd-ut
 	@$(FAILPOINT_ENABLE)
-	./scripts/ci-subtask.sh $(JOB_COUNT) $(JOB_INDEX) || { $(FAILPOINT_DISABLE); exit 1; }
+	./scripts/ci-subtask.sh $(JOB_INDEX) || { $(FAILPOINT_DISABLE); exit 1; }
+	@$(CLEAN_UT_BINARY)
 	@$(FAILPOINT_DISABLE)
 
 TSO_INTEGRATION_TEST_PKGS := $(PD_PKG)/tests/server/tso

codecov.yml

@@ -24,9 +24,3 @@ flag_management:
         target: 74% # increase it if you want to enforce higher coverage for project, current setting as 74% is for do not let the error be reported and lose the meaning of warning.
       - type: patch
         target: 74% # increase it if you want to enforce higher coverage for project, current setting as 74% is for do not let the error be reported and lose the meaning of warning.
-
-ignore:
-  # Ignore the tool tests
-  - tests/dashboard
-  - tests/pdbackup
-  - tests/pdctl

go.mod

@@ -18,7 +18,7 @@ require (
 	github.com/brianvoe/gofakeit/v6 v6.26.3
 	github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5
 	github.com/coreos/go-semver v0.3.1
-	github.com/docker/go-units v0.4.0
+	github.com/docker/go-units v0.5.0
 	github.com/elliotchance/pie/v2 v2.1.0
 	github.com/gin-contrib/cors v1.4.0
 	github.com/gin-contrib/gzip v0.0.1
@@ -33,7 +33,7 @@ require (
 	github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d
 	github.com/pingcap/errcode v0.3.0
 	github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c
-	github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00
+	github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c
 	github.com/pingcap/kvproto v0.0.0-20240403065636-c699538f7aa1
 	github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3
 	github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21

go.sum

@@ -99,8 +99,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk=
 github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE=
-github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
-github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
+github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o=
@@ -368,8 +368,8 @@ github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTw
 github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4=
 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg=
-github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0=
-github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew=
+github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c h1:CgbKAHto5CQgWM9fSBIvaxsJHuGP0uM74HXtv3MyyGQ=
+github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew=
 github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w=
 github.com/pingcap/kvproto v0.0.0-20240403065636-c699538f7aa1 h1:vDWWJKU6ztczn24XixahtLwcnJ15DOtSRIRM3jVtZNU=
 github.com/pingcap/kvproto v0.0.0-20240403065636-c699538f7aa1/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8=

scripts/ci-subtask.sh

@@ -1,65 +1,31 @@
 #!/usr/bin/env bash
 
-# ./ci-subtask.sh <TOTAL_TASK_N> <TASK_INDEX>
+# ./ci-subtask.sh <TASK_INDEX>
 
 ROOT_PATH_COV=$(pwd)/covprofile
-
-if [[ $2 -gt 9 ]]; then
-    # run tools tests
-    if [[ $2 -eq 10 ]]; then
+# Currently, we only have 3 integration tests, so we can hardcode the task index.
+integrations_dir=$(pwd)/tests/integrations
+
+case $1 in
+    1)
+        # unit tests
+        ./bin/pd-ut run --race --coverprofile $ROOT_PATH_COV  || exit 1
+        ;;
+    2)
+        # tools tests
         cd ./tools && make ci-test-job && cat covprofile >> $ROOT_PATH_COV || exit 1
-        exit
-    fi
-
-    # Currently, we only have 3 integration tests, so we can hardcode the task index.
-    integrations_dir=$(pwd)/tests/integrations
-    integrations_tasks=($(find "$integrations_dir" -mindepth 1 -maxdepth 1 -type d))
-    for t in "${integrations_tasks[@]}"; do
-        if [[ "$t" = "$integrations_dir/client" && $2 -eq 11 ]]; then
-            cd ./client && make ci-test-job && cat covprofile >> $ROOT_PATH_COV || exit 1
-            cd $integrations_dir && make ci-test-job test_name=client && cat ./client/covprofile >> $ROOT_PATH_COV || exit 1
-        elif [[ "$t" = "$integrations_dir/tso" && $2 -eq 12 ]]; then
-            cd $integrations_dir && make ci-test-job test_name=tso && cat ./tso/covprofile >> $ROOT_PATH_COV || exit 1
-        elif [[ "$t" = "$integrations_dir/mcs" && $2 -eq 13 ]]; then
-            cd $integrations_dir && make ci-test-job test_name=mcs && cat ./mcs/covprofile >> $ROOT_PATH_COV || exit 1
-        fi
-    done
-else
-    # Get package test list.
-    packages=($(go list ./...))
-    dirs=($(find . -iname "*_test.go" -exec dirname {} \; | sort -u | sed -e "s/^\./github.com\/tikv\/pd/"))
-    tasks=($(comm -12 <(printf "%s\n" "${packages[@]}") <(printf "%s\n" "${dirs[@]}")))
-
-    weight() {
-        [[ $1 == "github.com/tikv/pd/server/api" ]] && return 30
-        [[ $1 == "github.com/tikv/pd/pkg/schedule" ]] && return 30
-        [[ $1 == "github.com/tikv/pd/pkg/core" ]] && return 30
-        [[ $1 == "github.com/tikv/pd/tests/server/api" ]] && return 30
-        [[ $1 =~ "pd/tests" ]] && return 5
-        return 1
-    }
-
-    # Create an associative array to store the weight of each task.
-    declare -A task_weights
-    for t in ${tasks[@]}; do
-        weight $t
-        task_weights[$t]=$?
-    done
-
-    # Sort tasks by weight in descending order.
-    tasks=($(printf "%s\n" "${tasks[@]}" | sort -rn))
-
-    scores=($(seq "$1" | xargs -I{} echo 0))
-
-    res=()
-    for t in ${tasks[@]}; do
-        min_i=0
-        for i in ${!scores[@]}; do
-            [[ ${scores[i]} -lt ${scores[$min_i]} ]] && min_i=$i
-        done
-        scores[$min_i]=$((${scores[$min_i]} + ${task_weights[$t]}))
-        [[ $(($min_i + 1)) -eq $2 ]] && res+=($t)
-    done
-
-    CGO_ENABLED=1 go test -timeout=15m -tags deadlock -race -cover -covermode=atomic -coverprofile=$ROOT_PATH_COV -coverpkg=./... ${res[@]}
-fi
+        ;;
+    3)
+        # integration test client
+        cd ./client && make ci-test-job && cat covprofile >> $ROOT_PATH_COV  || exit 1
+        cd $integrations_dir && make ci-test-job test_name=client && cat ./client/covprofile >> $ROOT_PATH_COV || exit 1
+        ;;
+    4)
+        # integration test tso
+        cd $integrations_dir && make ci-test-job test_name=tso && cat ./tso/covprofile >> $ROOT_PATH_COV || exit 1
+        ;;
+    5)
+        # integration test mcs
+        cd $integrations_dir && make ci-test-job test_name=mcs && cat ./mcs/covprofile >> $ROOT_PATH_COV || exit 1
+        ;;
+esac

server/cluster/cluster_test.go

@@ -3482,8 +3482,13 @@ func TestStoreOverloadedWithReplace(t *testing.T) {
 	re.Empty(ops)
 	// sleep 2 seconds to make sure that token is filled up
 	time.Sleep(2 * time.Second)
-	ops, _ = lb.Schedule(tc, false /* dryRun */)
-	re.NotEmpty(ops)
+	testutil.Eventually(re, func() bool {
+		ops, _ = lb.Schedule(tc, false /* dryRun */)
+		if len(ops) == 0 {
+			return false
+		}
+		return true
+	})
 }
 
 func TestDownStoreLimit(t *testing.T) {

tests/scheduling_cluster.go

@@ -113,7 +113,7 @@ func (tc *TestSchedulingCluster) WaitForPrimaryServing(re *require.Assertions) *
 			}
 		}
 		return false
-	}, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond))
+	}, testutil.WithWaitFor(10*time.Second), testutil.WithTickInterval(50*time.Millisecond))
 
 	return primary
 }