⚡ Improve data sync performance siyuan-note/siyuan#13216

go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
 	github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4
 	github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97
-	github.com/siyuan-note/filelock v0.0.0-20240724034355-d1ed7bf21d04
+	github.com/siyuan-note/filelock v0.0.0-20241121021809-1cf0626a7d64
 	github.com/siyuan-note/httpclient v0.0.0-20241113084556-839baaab03f6
 	github.com/siyuan-note/logging v0.0.0-20240505035402-6430d57006a2
 	github.com/studio-b12/gowebdav v0.9.0
@@ -36,7 +36,7 @@ require (
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/imroc/req/v3 v3.48.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/onsi/ginkgo/v2 v2.21.0 // indirect
+	github.com/onsi/ginkgo/v2 v2.22.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/quic-go/qpack v0.5.1 // indirect
 	github.com/quic-go/quic-go v0.48.1 // indirect

go.sum

@@ -65,8 +65,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
-github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM=
-github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
+github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg=
+github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
 github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8=
 github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc=
 github.com/panjf2000/ants/v2 v2.10.0 h1:zhRg1pQUtkyRiOFo2Sbqwjp0GfBNo9cUY2/Grpx1p+8=
@@ -98,8 +98,8 @@ github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4 h1:kJaw5L/e
 github.com/siyuan-note/encryption v0.0.0-20231219001248-1e028a4d13b4/go.mod h1:UYcCCY+0wh+GmUoDOaO63j1sV5lgy7laLAk1XhEiUis=
 github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97 h1:lM5v8BfNtbOL5jYwhCdMYBcYtr06IYBKjjSLAPMKTM8=
 github.com/siyuan-note/eventbus v0.0.0-20240627125516-396fdb0f0f97/go.mod h1:1/nGgthl89FPA7GzAcEWKl6zRRnfgyTjzLZj9bW7kuw=
-github.com/siyuan-note/filelock v0.0.0-20240724034355-d1ed7bf21d04 h1:aoXvEO6BMqm6L0EnTjRhB4ynQIyJvHpqz+Ue3g0D3a0=
-github.com/siyuan-note/filelock v0.0.0-20240724034355-d1ed7bf21d04/go.mod h1:iqFhf4EDKy4MjQgT6RQJ6Z6NSW662NS0PR40K1qdSpE=
+github.com/siyuan-note/filelock v0.0.0-20241121021809-1cf0626a7d64 h1:w4i1AumKqsUgMcuNKw5FkJjAWK8jXVKpvW1BkAJvB+0=
+github.com/siyuan-note/filelock v0.0.0-20241121021809-1cf0626a7d64/go.mod h1:QUxfb/zE/lMrpiGBV9MBT5c5NKQanGvpdBXtMwWjTD0=
 github.com/siyuan-note/httpclient v0.0.0-20241113084556-839baaab03f6 h1:IzvOUPUc6YbtLgQuj3rVVIdbn1HzDOGMsTLLb9yN0Wo=
 github.com/siyuan-note/httpclient v0.0.0-20241113084556-839baaab03f6/go.mod h1:cMXV74/pjFOvH3zIwQW/SyH3gZw8YzTBbqt7dxof69k=
 github.com/siyuan-note/logging v0.0.0-20240505035402-6430d57006a2 h1:/2+tlOThVB86RxSLeW0JFw2ISUrH2ZFRg15ULGAUGAE=

repo.go

@@ -20,6 +20,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"math"
 	"os"
 	"path"
@@ -417,7 +418,13 @@ func (repo *Repo) Checkout(id string, context map[string]interface{}) (upserts,
 	var files []*entity.File
 	ignoreMatcher := repo.ignoreMatcher()
 	eventbus.Publish(eventbus.EvtCheckoutBeforeWalkData, context, repo.DataPath)
-	err = filelock.Walk(repo.DataPath, func(path string, info os.FileInfo, err error) error {
+	err = filelock.Walk(repo.DataPath, func(path string, d fs.DirEntry, err error) error {
+		if nil != err {
+			logging.LogErrorf("walk data failed: %s", err)
+			return err
+		}
+
+		info, err := d.Info()
 		if nil != err {
 			logging.LogErrorf("walk data failed: %s", err)
 			return err
@@ -618,7 +625,7 @@ func (repo *Repo) index0(memo string, context map[string]interface{}) (ret *enti
 	ignoreMatcher := repo.ignoreMatcher()
 	eventbus.Publish(eventbus.EvtIndexBeforeWalkData, context, repo.DataPath)
 	start := time.Now()
-	err = filelock.Walk(repo.DataPath, func(path string, info os.FileInfo, err error) error {
+	err = filelock.Walk(repo.DataPath, func(path string, d fs.DirEntry, err error) error {
 		if nil != err {
 			if isNoSuchFileOrDirErr(err) {
 				// An error `Failed to create data snapshot` is occasionally reported during automatic data sync https://github.com/siyuan-note/siyuan/issues/8998
@@ -628,6 +635,12 @@ func (repo *Repo) index0(memo string, context map[string]interface{}) (ret *enti
 			logging.LogErrorf("walk data failed: %s", err)
 			return err
 		}
+
+		info, err := d.Info()
+		if nil != err {
+			logging.LogErrorf("walk data failed: %s", err)
+			return err
+		}
 		if ignored, ignoreErr := repo.builtInIgnore(info, path); ignored || nil != ignoreErr {
 			return ignoreErr
 		}