From ddf8b736c3eae885d9ce058b60589d718ea64b4c Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 12 Dec 2024 17:03:34 +0800 Subject: [PATCH 01/27] pb --- internal/rpc/msg/delete.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/internal/rpc/msg/delete.go b/internal/rpc/msg/delete.go index e19bba867e..371c50e2ea 100644 --- a/internal/rpc/msg/delete.go +++ b/internal/rpc/msg/delete.go @@ -138,9 +138,16 @@ func (m *msgServer) clearConversation(ctx context.Context, conversationIDs []str } isSyncSelf, isSyncOther := m.validateDeleteSyncOpt(deleteSyncOpt) if !isSyncOther { - if err := m.MsgDatabase.SetUserConversationsMinSeqs(ctx, userID, m.getMinSeqs(maxSeqs)); err != nil { + setSeqs := m.getMinSeqs(maxSeqs) + if err := m.MsgDatabase.SetUserConversationsMinSeqs(ctx, userID, setSeqs); err != nil { return err } + ownerUserIDs := []string{userID} + for conversationID, seq := range setSeqs { + if err := m.Conversation.SetConversationMinSeq(ctx, ownerUserIDs, conversationID, seq); err != nil { + return err + } + } // notification 2 self if isSyncSelf { tips := &sdkws.ClearConversationTips{UserID: userID, ConversationIDs: existConversationIDs} From 6f169ec9690f62e8ea8edb697c87bb74cd3c17c3 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 17 Dec 2024 11:58:06 +0800 Subject: [PATCH 02/27] fix: Modifying other fields while setting IsPrivateChat does not take effect --- internal/rpc/conversation/conversation.go | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index 7345c965db..8ed1323d56 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -354,7 +354,15 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver needUpdateUsersList = append(needUpdateUsersList, userID) } } + if len(m) != 0 && len(needUpdateUsersList) != 0 { + if err := c.conversationDatabase.SetUsersConversationFieldTx(ctx, needUpdateUsersList, &conversation, m); err != nil { + return nil, err + } + for _, v := range needUpdateUsersList { + c.conversationNotificationSender.ConversationChangeNotification(ctx, v, []string{req.Conversation.ConversationID}) + } + } if req.Conversation.IsPrivateChat != nil && req.Conversation.ConversationType != constant.ReadGroupChatType { var conversations []*dbModel.Conversation for _, ownerUserID := range req.UserIDs { @@ -372,16 +380,6 @@ func (c *conversationServer) SetConversations(ctx context.Context, req *pbconver c.conversationNotificationSender.ConversationSetPrivateNotification(ctx, userID, req.Conversation.UserID, req.Conversation.IsPrivateChat.Value, req.Conversation.ConversationID) } - } else { - if len(m) != 0 && len(needUpdateUsersList) != 0 { - if err := c.conversationDatabase.SetUsersConversationFieldTx(ctx, needUpdateUsersList, &conversation, m); err != nil { - return nil, err - } - - for _, v := range needUpdateUsersList { - c.conversationNotificationSender.ConversationChangeNotification(ctx, v, []string{req.Conversation.ConversationID}) - } - } } return &pbconversation.SetConversationsResp{}, nil From d15414d8851c37b938e5d795fd40e91a836ff5f7 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 17 Dec 2024 18:28:16 +0800 Subject: [PATCH 03/27] fix: quote message error revoke --- pkg/common/storage/controller/msg.go | 52 +++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 8d82d85433..3c3cd9671b 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -269,7 +269,6 @@ func (db *commonMsgDatabase) getMsgBySeqs(ctx context.Context, userID, conversat } return totalMsgs, nil } - func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][]*model.MsgInfoModel, userID, conversationID string, msg *model.MsgInfoModel) { if msg.IsRead { msg.Msg.IsRead = true @@ -280,16 +279,53 @@ func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][ if msg.Msg.Content == "" { return } + type MsgData struct { + SendID string `json:"sendID"` + RecvID string `json:"recvID"` + GroupID string `json:"groupID"` + ClientMsgID string `json:"clientMsgID"` + ServerMsgID string `json:"serverMsgID"` + SenderPlatformID int32 `json:"senderPlatformID"` + SenderNickname string `json:"senderNickname"` + SenderFaceURL string `json:"senderFaceURL"` + SessionType int32 `json:"sessionType"` + MsgFrom int32 `json:"msgFrom"` + ContentType int32 `json:"contentType"` + Content string `json:"content"` + Seq int64 `json:"seq"` + SendTime int64 `json:"sendTime"` + CreateTime int64 `json:"createTime"` + Status int32 `json:"status"` + IsRead bool `json:"isRead"` + Options map[string]bool `json:"options,omitempty"` + OfflinePushInfo *sdkws.OfflinePushInfo `json:"offlinePushInfo"` + AtUserIDList []string `json:"atUserIDList"` + AttachedInfo string `json:"attachedInfo"` + Ex string `json:"ex"` + KeyVersion int32 `json:"keyVersion"` + DstUserIDs []string `json:"dstUserIDs"` + } var quoteMsg struct { Text string `json:"text,omitempty"` - QuoteMessage *sdkws.MsgData `json:"quoteMessage,omitempty"` + QuoteMessage *MsgData `json:"quoteMessage,omitempty"` MessageEntityList json.RawMessage `json:"messageEntityList,omitempty"` } if err := json.Unmarshal([]byte(msg.Msg.Content), "eMsg); err != nil { log.ZError(ctx, "json.Unmarshal", err) return } - if quoteMsg.QuoteMessage == nil || quoteMsg.QuoteMessage.ContentType == constant.MsgRevokeNotification { + if quoteMsg.QuoteMessage == nil || quoteMsg.QuoteMessage.Content == "" { + return + } + if quoteMsg.QuoteMessage.Content == "e30=" { + quoteMsg.QuoteMessage.Content = "{}" + data, err := json.Marshal("eMsg) + if err != nil { + return + } + msg.Msg.Content = string(data) + } + if quoteMsg.QuoteMessage.Seq <= 0 && quoteMsg.QuoteMessage.ContentType == constant.MsgRevokeNotification { return } var msgs []*model.MsgInfoModel @@ -311,9 +347,9 @@ func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][ } quoteMsg.QuoteMessage.ContentType = constant.MsgRevokeNotification if len(msgs) > 0 { - quoteMsg.QuoteMessage.Content = []byte(msgs[0].Msg.Content) + quoteMsg.QuoteMessage.Content = msgs[0].Msg.Content } else { - quoteMsg.QuoteMessage.Content = []byte("{}") + quoteMsg.QuoteMessage.Content = "{}" } data, err := json.Marshal("eMsg) if err != nil { @@ -321,9 +357,9 @@ func (db *commonMsgDatabase) handlerDBMsg(ctx context.Context, cache map[int64][ return } msg.Msg.Content = string(data) - if _, err := db.msgDocDatabase.UpdateMsg(ctx, db.msgTable.GetDocID(conversationID, msg.Msg.Seq), db.msgTable.GetMsgIndex(msg.Msg.Seq), "msg", msg.Msg); err != nil { - log.ZError(ctx, "UpdateMsgContent", err) - } + //if _, err := db.msgDocDatabase.UpdateMsg(ctx, db.msgTable.GetDocID(conversationID, msg.Msg.Seq), db.msgTable.GetMsgIndex(msg.Msg.Seq), "msg", msg.Msg); err != nil { + // log.ZError(ctx, "UpdateMsgContent", err) + //} } func (db *commonMsgDatabase) findMsgInfoBySeq(ctx context.Context, userID, docID string, conversationID string, seqs []int64) (totalMsgs []*model.MsgInfoModel, err error) { From f03cbec6532cfadface6a55b365d36df879e8069 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 18 Dec 2024 17:41:49 +0800 Subject: [PATCH 04/27] refactoring scheduled tasks --- internal/rpc/third/s3.go | 93 +++------------- internal/rpc/third/third.go | 14 +-- internal/tools/cron_task.go | 125 ++++++++-------------- internal/tools/msg.go | 23 ++++ internal/tools/s3.go | 34 ++++++ internal/tools/user_msg.go | 31 ++++++ pkg/common/storage/controller/s3.go | 20 +--- pkg/common/storage/database/mgo/object.go | 29 +++-- pkg/common/storage/database/object.go | 6 +- 9 files changed, 175 insertions(+), 200 deletions(-) create mode 100644 internal/tools/msg.go create mode 100644 internal/tools/s3.go create mode 100644 internal/tools/user_msg.go diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index b4b064d7f4..d4146fdf48 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -23,13 +23,9 @@ import ( "strconv" "time" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "go.mongodb.org/mongo-driver/mongo" - "github.com/google/uuid" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" - "github.com/openimsdk/protocol/sdkws" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/protocol/third" "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" @@ -288,87 +284,30 @@ func (t *thirdServer) apiAddress(prefix, name string) string { } func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { - var conf config.Third + engine := t.config.RpcConfig.Object.Enable expireTime := time.UnixMilli(req.ExpireTime) - - findPagination := &sdkws.RequestPagination{ - PageNumber: 1, - ShowNumber: 500, - } - // Find all expired data in S3 database - total, models, err := t.s3dataBase.FindNeedDeleteObjectByDB(ctx, expireTime, req.ObjectGroup, findPagination) - if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments { - return nil, errs.Wrap(err) - } - - if total == 0 { - log.ZDebug(ctx, "Not have OutdatedData", "delete Total", total) - return &third.DeleteOutdatedDataResp{Count: int32(total)}, nil - } - - needDelObjectKeys := make([]string, len(models)) - for _, model := range models { - needDelObjectKeys = append(needDelObjectKeys, model.Key) + models, err := t.s3dataBase.FindExpirationObject(ctx, engine, expireTime, req.ObjectGroup, int64(req.Count)) + if err != nil { + return nil, err } - - // Remove duplicate keys, have the same key use in different models - needDelObjectKeys = datautil.Distinct(needDelObjectKeys) - - for _, key := range needDelObjectKeys { - // Find all models by key - keyModels, err := t.s3dataBase.FindModelsByKey(ctx, key) - if err != nil && errs.Unwrap(err) != mongo.ErrNoDocuments { + if len(models) > 0 { + names := datautil.Batch(func(o *model.Object) string { + return o.Name + }, models) + if err := t.s3dataBase.DeleteSpecifiedData(ctx, engine, names); err != nil { return nil, errs.Wrap(err) } - - // check keyModels, if all keyModels. - needDelKey := true // Default can delete - for _, keymodel := range keyModels { - // If group is empty or CreateTime is after expireTime, can't delete this key - if keymodel.Group == "" || keymodel.CreateTime.After(expireTime) { - needDelKey = false - break - } + if err := t.s3dataBase.DelS3Key(ctx, engine, names...); err != nil { + return nil, err } - - // If this object is not referenced by not expire data, delete it - if needDelKey && t.minio != nil { - // If have a thumbnail, delete it - thumbnailKey, _ := t.getMinioImageThumbnailKey(ctx, key) - if thumbnailKey != "" { - err := t.s3dataBase.DeleteObject(ctx, thumbnailKey) - if err != nil { - log.ZWarn(ctx, "Delete thumbnail object is error:", errs.Wrap(err), "thumbnailKey", thumbnailKey) - } - } - - // Delete object - err = t.s3dataBase.DeleteObject(ctx, key) - if err != nil { - log.ZWarn(ctx, "Delete object is error", errs.Wrap(err), "object key", key) - } - - // Delete cache key - err = t.s3dataBase.DelS3Key(ctx, conf.Object.Enable, key) - if err != nil { - log.ZWarn(ctx, "Delete cache key is error:", errs.Wrap(err), "cache S3 key:", key) + for _, object := range models { + if err := t.s3.DeleteObject(ctx, object.Key); err != nil { + return nil, err } } } - - // handle delete data in S3 database - for _, model := range models { - // Delete all expired data row in S3 database - err := t.s3dataBase.DeleteSpecifiedData(ctx, model.Engine, model.Name) - if err != nil { - return nil, errs.Wrap(err) - } - } - - log.ZDebug(ctx, "DeleteOutdatedData", "delete Total", total) - - return &third.DeleteOutdatedDataResp{Count: int32(total)}, nil + return &third.DeleteOutdatedDataResp{Count: int32(len(models))}, nil } type FormDataMate struct { diff --git a/internal/rpc/third/third.go b/internal/rpc/third/third.go index 77e6d459fa..dc964fdb15 100644 --- a/internal/rpc/third/third.go +++ b/internal/rpc/third/third.go @@ -44,7 +44,7 @@ type thirdServer struct { s3dataBase controller.S3Database defaultExpire time.Duration config *Config - minio *minio.Minio + s3 s3.Interface } type Config struct { @@ -79,13 +79,11 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg // Select the oss method according to the profile policy enable := config.RpcConfig.Object.Enable var ( - o s3.Interface - minioCli *minio.Minio + o s3.Interface ) switch enable { case "minio": - minioCli, err = minio.NewMinio(ctx, redis.NewMinioCache(rdb), *config.MinioConfig.Build()) - o = minioCli + o, err = minio.NewMinio(ctx, redis.NewMinioCache(rdb), *config.MinioConfig.Build()) case "cos": o, err = cos.NewCos(*config.RpcConfig.Object.Cos.Build()) case "oss": @@ -106,15 +104,11 @@ func Start(ctx context.Context, config *Config, client discovery.SvcDiscoveryReg s3dataBase: controller.NewS3Database(rdb, o, s3db), defaultExpire: time.Hour * 24 * 7, config: config, - minio: minioCli, + s3: o, }) return nil } -func (t *thirdServer) getMinioImageThumbnailKey(ctx context.Context, name string) (string, error) { - return t.minio.GetImageThumbnailKey(ctx, name) -} - func (t *thirdServer) FcmUpdateToken(ctx context.Context, req *third.FcmUpdateTokenReq) (resp *third.FcmUpdateTokenResp, err error) { err = t.thirdDatabase.FcmUpdateToken(ctx, req.Account, int(req.PlatformID), req.FcmToken, req.ExpireTime) if err != nil { diff --git a/internal/tools/cron_task.go b/internal/tools/cron_task.go index 2fe7d0e395..049a6199cf 100644 --- a/internal/tools/cron_task.go +++ b/internal/tools/cron_task.go @@ -2,10 +2,6 @@ package tools import ( "context" - "fmt" - "os" - "time" - "github.com/openimsdk/open-im-server/v3/pkg/common/config" kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" pbconversation "github.com/openimsdk/protocol/conversation" @@ -60,87 +56,58 @@ func Start(ctx context.Context, config *CronTaskConfig) error { return err } - msgClient := msg.NewMsgClient(msgConn) - conversationClient := pbconversation.NewConversationClient(conversationConn) - thirdClient := third.NewThirdClient(thirdConn) - - crontab := cron.New() - - // scheduled hard delete outdated Msgs in specific time. - destructMsgsFunc := func() { - now := time.Now() - deltime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.RetainChatRecords)) - ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deltime.UnixMilli())) - log.ZDebug(ctx, "Destruct chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) - - if _, err := msgClient.DestructMsgs(ctx, &msg.DestructMsgsReq{Timestamp: deltime.UnixMilli()}); err != nil { - log.ZError(ctx, "cron destruct chat records failed", err, "deltime", deltime, "cont", time.Since(now)) - return - } - log.ZDebug(ctx, "cron destruct chat records success", "deltime", deltime, "cont", time.Since(now)) - } - if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, destructMsgsFunc); err != nil { - return errs.Wrap(err) + srv := &cronServer{ + ctx: ctx, + config: config, + cron: cron.New(), + msgClient: msg.NewMsgClient(msgConn), + conversationClient: pbconversation.NewConversationClient(conversationConn), + thirdClient: third.NewThirdClient(thirdConn), } - // scheduled soft delete outdated Msgs in specific time when user set `is_msg_destruct` feature. - clearMsgFunc := func() { - now := time.Now() - ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), now.UnixMilli())) - log.ZDebug(ctx, "clear msg cron start", "now", now) - - conversations, err := conversationClient.GetConversationsNeedClearMsg(ctx, &pbconversation.GetConversationsNeedClearMsgReq{}) - if err != nil { - log.ZError(ctx, "Get conversation need Destruct msgs failed.", err) - return - } - - _, err = msgClient.ClearMsg(ctx, &msg.ClearMsgReq{Conversations: conversations.Conversations}) - if err != nil { - log.ZError(ctx, "Clear Msg failed.", err) - return - } - - log.ZDebug(ctx, "clear msg cron task completed", "cont", time.Since(now)) - } - if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, clearMsgFunc); err != nil { - return errs.Wrap(err) + if err := srv.registerClearS3(); err != nil { + return err } - - // scheduled delete outdated file Objects and their datas in specific time. - deleteObjectFunc := func() { - now := time.Now() - executeNum := 5 - // number of pagination. if need modify, need update value in third.DeleteOutdatedData - pageShowNumber := 500 - deleteTime := now.Add(-time.Hour * 24 * time.Duration(config.CronTask.FileExpireTime)) - ctx := mcontext.SetOperationID(ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli())) - log.ZDebug(ctx, "deleteoutDatedData", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) - - if len(config.CronTask.DeleteObjectType) == 0 { - log.ZDebug(ctx, "cron deleteoutDatedData not type need delete", "deletetime", deleteTime, "DeleteObjectType", config.CronTask.DeleteObjectType, "cont", time.Since(now)) - return - } - - for i := 0; i < executeNum; i++ { - resp, err := thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli(), ObjectGroup: config.CronTask.DeleteObjectType}) - if err != nil { - log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(now)) - return - } - if resp.Count == 0 || resp.Count < int32(pageShowNumber) { - break - } - } - - log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(now)) + if err := srv.registerDeleteMsg(); err != nil { + return err } - if _, err := crontab.AddFunc(config.CronTask.CronExecuteTime, deleteObjectFunc); err != nil { - return errs.Wrap(err) + if err := srv.registerClearUserMsg(); err != nil { + return err } - log.ZDebug(ctx, "start cron task", "CronExecuteTime", config.CronTask.CronExecuteTime) - crontab.Start() + srv.cron.Start() <-ctx.Done() return nil } + +type cronServer struct { + ctx context.Context + config *CronTaskConfig + cron *cron.Cron + msgClient msg.MsgClient + conversationClient pbconversation.ConversationClient + thirdClient third.ThirdClient +} + +func (c *cronServer) registerClearS3() error { + if c.config.CronTask.FileExpireTime <= 0 || len(c.config.CronTask.DeleteObjectType) == 0 { + log.ZInfo(c.ctx, "disable scheduled cleanup of s3", "fileExpireTime", c.config.CronTask.FileExpireTime, "deleteObjectType", c.config.CronTask.DeleteObjectType) + return nil + } + _, err := c.cron.AddFunc(c.config.CronTask.CronExecuteTime, c.clearS3) + return errs.WrapMsg(err, "failed to register clear s3 cron task") +} + +func (c *cronServer) registerDeleteMsg() error { + if c.config.CronTask.RetainChatRecords <= 0 { + log.ZInfo(c.ctx, "disable scheduled cleanup of chat records", "retainChatRecords", c.config.CronTask.RetainChatRecords) + return nil + } + _, err := c.cron.AddFunc(c.config.CronTask.CronExecuteTime, c.deleteMsg) + return errs.WrapMsg(err, "failed to register delete msg cron task") +} + +func (c *cronServer) registerClearUserMsg() error { + _, err := c.cron.AddFunc(c.config.CronTask.CronExecuteTime, c.clearUserMsg) + return errs.WrapMsg(err, "failed to register clear user msg cron task") +} diff --git a/internal/tools/msg.go b/internal/tools/msg.go new file mode 100644 index 0000000000..1149b1f297 --- /dev/null +++ b/internal/tools/msg.go @@ -0,0 +1,23 @@ +package tools + +import ( + "fmt" + "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mcontext" + "os" + "time" +) + +func (c *cronServer) deleteMsg() { + now := time.Now() + deltime := now.Add(-time.Hour * 24 * time.Duration(c.config.CronTask.RetainChatRecords)) + ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deltime.UnixMilli())) + log.ZDebug(ctx, "Destruct chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) + + if _, err := c.msgClient.DestructMsgs(ctx, &msg.DestructMsgsReq{Timestamp: deltime.UnixMilli()}); err != nil { + log.ZError(ctx, "cron destruct chat records failed", err, "deltime", deltime, "cont", time.Since(now)) + return + } + log.ZDebug(ctx, "cron destruct chat records success", "deltime", deltime, "cont", time.Since(now)) +} diff --git a/internal/tools/s3.go b/internal/tools/s3.go new file mode 100644 index 0000000000..38d8badc66 --- /dev/null +++ b/internal/tools/s3.go @@ -0,0 +1,34 @@ +package tools + +import ( + "fmt" + "github.com/openimsdk/protocol/third" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mcontext" + "os" + "time" +) + +func (c *cronServer) clearS3() { + start := time.Now() + executeNum := 10 + // number of pagination. if need modify, need update value in third.DeleteOutdatedData + pageShowNumber := 500 + deleteTime := start.Add(-time.Hour * 24 * time.Duration(c.config.CronTask.FileExpireTime)) + operationID := fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli()) + ctx := mcontext.SetOperationID(c.ctx, operationID) + log.ZDebug(ctx, "deleteoutDatedData", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) + for i := 1; i <= executeNum; i++ { + ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("%s_%d", operationID, i)) + resp, err := c.thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli(), ObjectGroup: c.config.CronTask.DeleteObjectType, Count: int32(pageShowNumber)}) + if err != nil { + log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(start)) + return + } + if resp.Count < int32(pageShowNumber) { + break + } + } + + log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(start)) +} diff --git a/internal/tools/user_msg.go b/internal/tools/user_msg.go new file mode 100644 index 0000000000..9dfaf58c36 --- /dev/null +++ b/internal/tools/user_msg.go @@ -0,0 +1,31 @@ +package tools + +import ( + "fmt" + pbconversation "github.com/openimsdk/protocol/conversation" + "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/tools/log" + "github.com/openimsdk/tools/mcontext" + "os" + "time" +) + +func (c *cronServer) clearUserMsg() { + now := time.Now() + ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), now.UnixMilli())) + log.ZDebug(ctx, "clear msg cron start", "now", now) + + conversations, err := c.conversationClient.GetConversationsNeedClearMsg(ctx, &pbconversation.GetConversationsNeedClearMsgReq{}) + if err != nil { + log.ZError(ctx, "Get conversation need Destruct msgs failed.", err) + return + } + + _, err = c.msgClient.ClearMsg(ctx, &msg.ClearMsgReq{Conversations: conversations.Conversations}) + if err != nil { + log.ZError(ctx, "Clear Msg failed.", err) + return + } + + log.ZDebug(ctx, "clear msg cron task completed", "cont", time.Since(now)) +} diff --git a/pkg/common/storage/controller/s3.go b/pkg/common/storage/controller/s3.go index 4e5ad18b6e..6a0f11b1ce 100644 --- a/pkg/common/storage/controller/s3.go +++ b/pkg/common/storage/controller/s3.go @@ -24,7 +24,6 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache" - "github.com/openimsdk/tools/db/pagination" "github.com/openimsdk/tools/s3" "github.com/openimsdk/tools/s3/cont" "github.com/redis/go-redis/v9" @@ -40,10 +39,8 @@ type S3Database interface { SetObject(ctx context.Context, info *model.Object) error StatObject(ctx context.Context, name string) (*s3.ObjectInfo, error) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) - FindNeedDeleteObjectByDB(ctx context.Context, duration time.Time, needDelType []string, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) - DeleteObject(ctx context.Context, name string) error - DeleteSpecifiedData(ctx context.Context, engine string, name string) error - FindModelsByKey(ctx context.Context, key string) (objects []*model.Object, err error) + FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) + DeleteSpecifiedData(ctx context.Context, engine string, name []string) error DelS3Key(ctx context.Context, engine string, keys ...string) error } @@ -120,21 +117,14 @@ func (s *s3Database) StatObject(ctx context.Context, name string) (*s3.ObjectInf func (s *s3Database) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) { return s.s3.FormData(ctx, name, size, contentType, duration) } -func (s *s3Database) FindNeedDeleteObjectByDB(ctx context.Context, duration time.Time, needDelType []string, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) { - return s.db.FindNeedDeleteObjectByDB(ctx, duration, needDelType, pagination) +func (s *s3Database) FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) { + return s.db.FindExpirationObject(ctx, engine, expiration, needDelType, count) } -func (s *s3Database) DeleteObject(ctx context.Context, name string) error { - return s.s3.DeleteObject(ctx, name) -} -func (s *s3Database) DeleteSpecifiedData(ctx context.Context, engine string, name string) error { +func (s *s3Database) DeleteSpecifiedData(ctx context.Context, engine string, name []string) error { return s.db.Delete(ctx, engine, name) } -func (s *s3Database) FindModelsByKey(ctx context.Context, key string) (objects []*model.Object, err error) { - return s.db.FindModelsByKey(ctx, key) -} - func (s *s3Database) DelS3Key(ctx context.Context, engine string, keys ...string) error { return s.s3cache.DelS3Key(ctx, engine, keys...) } diff --git a/pkg/common/storage/database/mgo/object.go b/pkg/common/storage/database/mgo/object.go index 5bc329d33b..53f2a6ba43 100644 --- a/pkg/common/storage/database/mgo/object.go +++ b/pkg/common/storage/database/mgo/object.go @@ -22,7 +22,6 @@ import ( "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "github.com/openimsdk/tools/db/mongoutil" - "github.com/openimsdk/tools/db/pagination" "github.com/openimsdk/tools/errs" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" @@ -91,21 +90,21 @@ func (o *S3Mongo) Take(ctx context.Context, engine string, name string) (*model. return mongoutil.FindOne[*model.Object](ctx, o.coll, bson.M{"name": name, "engine": engine}) } -func (o *S3Mongo) Delete(ctx context.Context, engine string, name string) error { - return mongoutil.DeleteOne(ctx, o.coll, bson.M{"name": name, "engine": engine}) -} - -// Find Expires object -func (o *S3Mongo) FindNeedDeleteObjectByDB(ctx context.Context, duration time.Time, needDelType []string, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) { - return mongoutil.FindPage[*model.Object](ctx, o.coll, bson.M{ - "create_time": bson.M{"$lt": duration}, - "group": bson.M{"$in": needDelType}, - }, pagination) +func (o *S3Mongo) Delete(ctx context.Context, engine string, name []string) error { + if len(name) == 0 { + return nil + } + return mongoutil.DeleteOne(ctx, o.coll, bson.M{"engine": engine, "name": bson.M{"$in": name}}) } -// Find object by key -func (o *S3Mongo) FindModelsByKey(ctx context.Context, key string) (objects []*model.Object, err error) { +func (o *S3Mongo) FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) { + opt := options.Find() + if count > 0 { + opt.SetLimit(count) + } return mongoutil.Find[*model.Object](ctx, o.coll, bson.M{ - "key": key, - }) + "engine": engine, + "create_time": bson.M{"$lt": expiration}, + "group": bson.M{"$in": needDelType}, + }, opt) } diff --git a/pkg/common/storage/database/object.go b/pkg/common/storage/database/object.go index c741e39a60..86d47a2a6b 100644 --- a/pkg/common/storage/database/object.go +++ b/pkg/common/storage/database/object.go @@ -19,13 +19,11 @@ import ( "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/tools/db/pagination" ) type ObjectInfo interface { SetObject(ctx context.Context, obj *model.Object) error Take(ctx context.Context, engine string, name string) (*model.Object, error) - Delete(ctx context.Context, engine string, name string) error - FindNeedDeleteObjectByDB(ctx context.Context, duration time.Time, needDelType []string, pagination pagination.Pagination) (total int64, objects []*model.Object, err error) - FindModelsByKey(ctx context.Context, key string) (objects []*model.Object, err error) + Delete(ctx context.Context, engine string, name []string) error + FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) } From 705bc37f9942b3059e2ec1bf82c5fb227329b271 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 19 Dec 2024 11:28:49 +0800 Subject: [PATCH 05/27] refactoring scheduled tasks --- internal/rpc/msg/clear.go | 71 ++++++------- internal/rpc/third/s3.go | 6 +- internal/tools/msg.go | 25 +++-- internal/tools/s3.go | 11 ++- internal/tools/user_msg.go | 6 +- pkg/common/storage/controller/msg.go | 104 +++++++++----------- pkg/common/storage/database/mgo/msg.go | 20 ++-- pkg/common/storage/database/mgo/msg_test.go | 78 ++++++++++----- pkg/common/storage/database/msg.go | 4 +- 9 files changed, 179 insertions(+), 146 deletions(-) diff --git a/internal/rpc/msg/clear.go b/internal/rpc/msg/clear.go index 7d62e7c8fa..8d4bd0c8ee 100644 --- a/internal/rpc/msg/clear.go +++ b/internal/rpc/msg/clear.go @@ -2,6 +2,7 @@ package msg import ( "context" + "strings" "time" "github.com/openimsdk/open-im-server/v3/pkg/authverify" @@ -26,52 +27,42 @@ func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) if req.Timestamp > time.Now().UnixMilli() { return nil, errs.ErrArgs.WrapMsg("request millisecond timestamp error") } - var ( - docNum int - msgNum int - start = time.Now() - getLimit = 5000 - ) - - destructMsg := func(ctx context.Context) (bool, error) { - docIDs, err := m.MsgDatabase.GetDocIDs(ctx) - if err != nil { - return false, err - } - - msgs, err := m.MsgDatabase.GetBeforeMsg(ctx, req.Timestamp, docIDs, getLimit) - if err != nil { - return false, err + if req.Limit <= 0 { + return nil, errs.ErrArgs.WrapMsg("request limit error") + } + docs, err := m.MsgDatabase.GetRandBeforeMsg(ctx, req.Timestamp, int(req.Limit)) + if err != nil { + return nil, err + } + for _, doc := range docs { + if err := m.MsgDatabase.DeleteDoc(ctx, doc.DocID); err != nil { + return nil, err } - if len(msgs) == 0 { - return false, nil + index := strings.LastIndex(doc.DocID, ":") + if index < 0 { + continue } - - for _, msg := range msgs { - index, err := m.MsgDatabase.DeleteDocMsgBefore(ctx, req.Timestamp, msg) - if err != nil { - return false, err + var minSeq int64 + for _, model := range doc.Msg { + if model.Msg == nil { + continue } - if len(index) == 0 { - return false, errs.ErrInternalServer.WrapMsg("delete doc msg failed") + if model.Msg.Seq > minSeq { + minSeq = model.Msg.Seq } - - docNum++ - msgNum += len(index) } - - return true, nil - } - - _, err = destructMsg(ctx) - if err != nil { - log.ZError(ctx, "clear msg failed", err, "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) - return nil, err + if minSeq <= 0 { + continue + } + conversationID := doc.DocID[:index] + if conversationID == "" { + continue + } + if err := m.MsgDatabase.SetMinSeq(ctx, conversationID, minSeq); err != nil { + return nil, err + } } - - log.ZDebug(ctx, "clearing message", "docNum", docNum, "msgNum", msgNum, "cost", time.Since(start)) - - return &msg.DestructMsgsResp{}, nil + return &msg.DestructMsgsResp{Count: int32(len(docs))}, nil } // soft delete for user self diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index d4146fdf48..a541462a28 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -19,6 +19,7 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "path" "strconv" "time" @@ -284,10 +285,13 @@ func (t *thirdServer) apiAddress(prefix, name string) string { } func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteOutdatedDataReq) (*third.DeleteOutdatedDataResp, error) { + if err := authverify.CheckAdmin(ctx, t.config.Share.IMAdminUserID); err != nil { + return nil, err + } engine := t.config.RpcConfig.Object.Enable expireTime := time.UnixMilli(req.ExpireTime) // Find all expired data in S3 database - models, err := t.s3dataBase.FindExpirationObject(ctx, engine, expireTime, req.ObjectGroup, int64(req.Count)) + models, err := t.s3dataBase.FindExpirationObject(ctx, engine, expireTime, req.ObjectGroup, int64(req.Limit)) if err != nil { return nil, err } diff --git a/internal/tools/msg.go b/internal/tools/msg.go index 1149b1f297..46f1573f28 100644 --- a/internal/tools/msg.go +++ b/internal/tools/msg.go @@ -12,12 +12,25 @@ import ( func (c *cronServer) deleteMsg() { now := time.Now() deltime := now.Add(-time.Hour * 24 * time.Duration(c.config.CronTask.RetainChatRecords)) - ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), deltime.UnixMilli())) + operationID := fmt.Sprintf("cron_msg_%d_%d", os.Getpid(), deltime.UnixMilli()) + ctx := mcontext.SetOperationID(c.ctx, operationID) log.ZDebug(ctx, "Destruct chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) - - if _, err := c.msgClient.DestructMsgs(ctx, &msg.DestructMsgsReq{Timestamp: deltime.UnixMilli()}); err != nil { - log.ZError(ctx, "cron destruct chat records failed", err, "deltime", deltime, "cont", time.Since(now)) - return + const ( + deleteCount = 20 + deleteLimit = 50 + ) + var count int + for i := 1; i <= deleteCount; i++ { + ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("%s_%d", operationID, i)) + resp, err := c.msgClient.DestructMsgs(ctx, &msg.DestructMsgsReq{Timestamp: deltime.UnixMilli(), Limit: deleteLimit}) + if err != nil { + log.ZError(ctx, "cron destruct chat records failed", err) + break + } + count += int(resp.Count) + if resp.Count <= deleteLimit { + break + } } - log.ZDebug(ctx, "cron destruct chat records success", "deltime", deltime, "cont", time.Since(now)) + log.ZDebug(ctx, "cron destruct chat records end", "deltime", deltime, "cont", time.Since(now), "deleteDocs", count) } diff --git a/internal/tools/s3.go b/internal/tools/s3.go index 38d8badc66..3b0122c84e 100644 --- a/internal/tools/s3.go +++ b/internal/tools/s3.go @@ -15,20 +15,21 @@ func (c *cronServer) clearS3() { // number of pagination. if need modify, need update value in third.DeleteOutdatedData pageShowNumber := 500 deleteTime := start.Add(-time.Hour * 24 * time.Duration(c.config.CronTask.FileExpireTime)) - operationID := fmt.Sprintf("cron_%d_%d", os.Getpid(), deleteTime.UnixMilli()) + operationID := fmt.Sprintf("cron_s3_%d_%d", os.Getpid(), deleteTime.UnixMilli()) ctx := mcontext.SetOperationID(c.ctx, operationID) log.ZDebug(ctx, "deleteoutDatedData", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) + var count int for i := 1; i <= executeNum; i++ { ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("%s_%d", operationID, i)) - resp, err := c.thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli(), ObjectGroup: c.config.CronTask.DeleteObjectType, Count: int32(pageShowNumber)}) + resp, err := c.thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli(), ObjectGroup: c.config.CronTask.DeleteObjectType, Limit: int32(pageShowNumber)}) if err != nil { - log.ZError(ctx, "cron deleteoutDatedData failed", err, "deleteTime", deleteTime, "cont", time.Since(start)) + log.ZError(ctx, "cron deleteoutDatedData failed", err) return } + count += int(resp.Count) if resp.Count < int32(pageShowNumber) { break } } - - log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(start)) + log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(start), "count", count) } diff --git a/internal/tools/user_msg.go b/internal/tools/user_msg.go index 9dfaf58c36..52a2b6bdce 100644 --- a/internal/tools/user_msg.go +++ b/internal/tools/user_msg.go @@ -12,9 +12,9 @@ import ( func (c *cronServer) clearUserMsg() { now := time.Now() - ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("cron_%d_%d", os.Getpid(), now.UnixMilli())) - log.ZDebug(ctx, "clear msg cron start", "now", now) - + operationID := fmt.Sprintf("cron_user_msg_%d_%d", os.Getpid(), now.UnixMilli()) + ctx := mcontext.SetOperationID(c.ctx, operationID) + log.ZDebug(ctx, "clear msg cron start") conversations, err := c.conversationClient.GetConversationsNeedClearMsg(ctx, &pbconversation.GetConversationsNeedClearMsgReq{}) if err != nil { log.ZError(ctx, "Get conversation need Destruct msgs failed.", err) diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 3c3cd9671b..88089f1a09 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -18,7 +18,6 @@ import ( "context" "encoding/json" "errors" - "strings" "time" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" @@ -69,6 +68,7 @@ type CommonMsgDatabase interface { GetMaxSeqs(ctx context.Context, conversationIDs []string) (map[string]int64, error) GetMaxSeq(ctx context.Context, conversationID string) (int64, error) SetMinSeqs(ctx context.Context, seqs map[string]int64) error + SetMinSeq(ctx context.Context, conversationID string, seq int64) error SetUserConversationsMinSeqs(ctx context.Context, userID string, seqs map[string]int64) (err error) SetHasReadSeq(ctx context.Context, userID string, conversationID string, hasReadSeq int64) error @@ -95,13 +95,14 @@ type CommonMsgDatabase interface { ConvertMsgsDocLen(ctx context.Context, conversationIDs []string) // get Msg when destruct msg before - GetBeforeMsg(ctx context.Context, ts int64, docIds []string, limit int) ([]*model.MsgDocModel, error) - DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) + //DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) - GetDocIDs(ctx context.Context) ([]string, error) + GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) SetUserConversationsMaxSeq(ctx context.Context, conversationID string, userID string, seq int64) error SetUserConversationsMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error + + DeleteDoc(ctx context.Context, docID string) error } func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { @@ -806,9 +807,10 @@ func (db *commonMsgDatabase) GetMaxSeq(ctx context.Context, conversationID strin return db.seqConversation.GetMaxSeq(ctx, conversationID) } -func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error { - return db.seqConversation.SetMinSeq(ctx, conversationID, minSeq) -} +// +//func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID string, minSeq int64) error { +// return db.seqConversation.SetMinSeq(ctx, conversationID, minSeq) +//} func (db *commonMsgDatabase) SetMinSeqs(ctx context.Context, seqs map[string]int64) error { return db.seqConversation.SetMinSeqs(ctx, seqs) @@ -947,56 +949,40 @@ func (db *commonMsgDatabase) ConvertMsgsDocLen(ctx context.Context, conversation db.msgDocDatabase.ConvertMsgsDocLen(ctx, conversationIDs) } -func (db *commonMsgDatabase) GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, limit int) ([]*model.MsgDocModel, error) { - var msgs []*model.MsgDocModel - for i := 0; i < len(docIDs); i += 1000 { - end := i + 1000 - if end > len(docIDs) { - end = len(docIDs) - } - - res, err := db.msgDocDatabase.GetBeforeMsg(ctx, ts, docIDs[i:end], limit) - if err != nil { - return nil, err - } - msgs = append(msgs, res...) - - if len(msgs) >= limit { - return msgs[:limit], nil - } - } - return msgs, nil -} - -func (db *commonMsgDatabase) DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) { - var notNull int - index := make([]int, 0, len(doc.Msg)) - for i, message := range doc.Msg { - if message.Msg != nil { - notNull++ - if message.Msg.SendTime < ts { - index = append(index, i) - } - } - } - if len(index) == 0 { - return index, nil - } - maxSeq := doc.Msg[index[len(index)-1]].Msg.Seq - conversationID := doc.DocID[:strings.LastIndex(doc.DocID, ":")] - if err := db.setMinSeq(ctx, conversationID, maxSeq+1); err != nil { - return index, err - } - if len(index) == notNull { - log.ZDebug(ctx, "Delete db in Doc", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq) - return index, db.msgDocDatabase.DeleteDoc(ctx, doc.DocID) - } else { - log.ZDebug(ctx, "delete db in index", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq) - return index, db.msgDocDatabase.DeleteMsgByIndex(ctx, doc.DocID, index) - } +func (db *commonMsgDatabase) GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) { + return db.msgDocDatabase.GetRandBeforeMsg(ctx, ts, limit) } -func (db *commonMsgDatabase) setMinSeq(ctx context.Context, conversationID string, seq int64) error { +// +//func (db *commonMsgDatabase) DeleteDocMsgBefore(ctx context.Context, ts int64, doc *model.MsgDocModel) ([]int, error) { +// var notNull int +// index := make([]int, 0, len(doc.Msg)) +// for i, message := range doc.Msg { +// if message.Msg != nil { +// notNull++ +// if message.Msg.SendTime < ts { +// index = append(index, i) +// } +// } +// } +// if len(index) == 0 { +// return index, nil +// } +// maxSeq := doc.Msg[index[len(index)-1]].Msg.Seq +// conversationID := doc.DocID[:strings.LastIndex(doc.DocID, ":")] +// if err := db.SetMinSeq(ctx, conversationID, maxSeq+1); err != nil { +// return index, err +// } +// if len(index) == notNull { +// log.ZDebug(ctx, "Delete db in Doc", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq) +// return index, db.msgDocDatabase.DeleteDoc(ctx, doc.DocID) +// } else { +// log.ZDebug(ctx, "delete db in index", "DocID", doc.DocID, "index", index, "maxSeq", maxSeq) +// return index, db.msgDocDatabase.DeleteMsgByIndex(ctx, doc.DocID, index) +// } +//} + +func (db *commonMsgDatabase) SetMinSeq(ctx context.Context, conversationID string, seq int64) error { dbSeq, err := db.seqConversation.GetMinSeq(ctx, conversationID) if err != nil { if errors.Is(errs.Unwrap(err), redis.Nil) { @@ -1010,8 +996,8 @@ func (db *commonMsgDatabase) setMinSeq(ctx context.Context, conversationID strin return db.seqConversation.SetMinSeq(ctx, conversationID, seq) } -func (db *commonMsgDatabase) GetDocIDs(ctx context.Context) ([]string, error) { - return db.msgDocDatabase.GetDocIDs(ctx) +func (db *commonMsgDatabase) GetRandDocIDs(ctx context.Context, limit int) ([]string, error) { + return db.msgDocDatabase.GetRandDocIDs(ctx, limit) } func (db *commonMsgDatabase) GetCacheMaxSeqWithTime(ctx context.Context, conversationIDs []string) (map[string]database.SeqTime, error) { @@ -1026,3 +1012,7 @@ func (db *commonMsgDatabase) GetMaxSeqsWithTime(ctx context.Context, conversatio // todo: only the time in the redis cache will be taken, not the message time return db.seqConversation.GetMaxSeqsWithTime(ctx, conversationIDs) } + +func (db *commonMsgDatabase) DeleteDoc(ctx context.Context, docID string) error { + return db.msgDocDatabase.DeleteDoc(ctx, docID) +} diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index fc1fe47eab..937dbae1dd 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -1227,8 +1227,7 @@ func (m *MsgMgo) ConvertMsgsDocLen(ctx context.Context, conversationIDs []string } } -func (m *MsgMgo) GetDocIDs(ctx context.Context) ([]string, error) { - limit := 5000 +func (m *MsgMgo) GetRandDocIDs(ctx context.Context, limit int) ([]string, error) { var skip int var docIDs []string var offset int @@ -1267,15 +1266,18 @@ func (m *MsgMgo) GetDocIDs(ctx context.Context) ([]string, error) { return docIDs, errs.Wrap(err) } -func (m *MsgMgo) GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, limit int) ([]*model.MsgDocModel, error) { +func (m *MsgMgo) GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) { return mongoutil.Aggregate[*model.MsgDocModel](ctx, m.coll, []bson.M{ { "$match": bson.M{ - "doc_id": bson.M{ - "$in": docIDs, - }, - "msgs.msg.send_time": bson.M{ - "$lt": ts, + "msgs": bson.M{ + "$not": bson.M{ + "$elemMatch": bson.M{ + "msg.send_time": bson.M{ + "$gt": ts, + }, + }, + }, }, }, }, @@ -1288,7 +1290,7 @@ func (m *MsgMgo) GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, li }, }, { - "$limit": limit, + "$sample": limit, }, }) } diff --git a/pkg/common/storage/database/mgo/msg_test.go b/pkg/common/storage/database/mgo/msg_test.go index 5aed4dc511..eff847d262 100644 --- a/pkg/common/storage/database/mgo/msg_test.go +++ b/pkg/common/storage/database/mgo/msg_test.go @@ -3,12 +3,11 @@ package mgo import ( "context" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" - "github.com/openimsdk/protocol/msg" - "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/db/mongoutil" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" + "math" "math/rand" "strconv" "testing" @@ -18,33 +17,43 @@ import ( func TestName1(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) defer cancel() - cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.48:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.66:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) - v := &MsgMgo{ - coll: cli.Database("openim_v3").Collection("msg3"), - } + //v := &MsgMgo{ + // coll: cli.Database("openim_v3").Collection("msg3"), + //} + // + //req := &msg.SearchMessageReq{ + // //RecvID: "3187706596", + // //SendID: "7009965934", + // ContentType: 101, + // //SendTime: "2024-05-06", + // //SessionType: 3, + // Pagination: &sdkws.RequestPagination{ + // PageNumber: 1, + // ShowNumber: 10, + // }, + //} + //total, res, err := v.SearchMessage(ctx, req) + //if err != nil { + // panic(err) + //} + // + //for i, re := range res { + // t.Logf("%d => %d | %+v", i+1, re.Msg.Seq, re.Msg.Content) + //} + // + //t.Log(total) - req := &msg.SearchMessageReq{ - //RecvID: "3187706596", - //SendID: "7009965934", - ContentType: 101, - //SendTime: "2024-05-06", - //SessionType: 3, - Pagination: &sdkws.RequestPagination{ - PageNumber: 1, - ShowNumber: 10, - }, - } - total, res, err := v.SearchMessage(ctx, req) + msg, err := NewMsgMongo(cli.Database("openim_v3")) if err != nil { panic(err) } - - for i, re := range res { - t.Logf("%d => %d | %+v", i+1, re.Msg.Seq, re.Msg.Content) + res, err := msg.GetBeforeMsg(ctx, time.Now().UnixMilli(), []string{"1:0"}, 1000) + if err != nil { + panic(err) } - - t.Log(total) + t.Log(len(res)) } func TestName10(t *testing.T) { @@ -73,3 +82,26 @@ func TestName10(t *testing.T) { } } + +func TestName3(t *testing.T) { + t.Log(uint64(math.MaxUint64)) + t.Log(int64(math.MaxInt64)) + + t.Log(int64(math.MinInt64)) +} + +func TestName4(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) + defer cancel() + cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.66:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + + msg, err := NewMsgMongo(cli.Database("openim_v3")) + if err != nil { + panic(err) + } + res, err := msg.GetBeforeMsg(ctx, time.Now().UnixMilli(), []string{"1:0"}, 1000) + if err != nil { + panic(err) + } + t.Log(len(res)) +} diff --git a/pkg/common/storage/database/msg.go b/pkg/common/storage/database/msg.go index 23a99f5b96..17aaf23429 100644 --- a/pkg/common/storage/database/msg.go +++ b/pkg/common/storage/database/msg.go @@ -45,7 +45,7 @@ type Msg interface { DeleteDoc(ctx context.Context, docID string) error DeleteMsgByIndex(ctx context.Context, docID string, index []int) error - GetBeforeMsg(ctx context.Context, ts int64, docIDs []string, limit int) ([]*model.MsgDocModel, error) + GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) - GetDocIDs(ctx context.Context) ([]string, error) + GetRandDocIDs(ctx context.Context, limit int) ([]string, error) } From 8807597ffbba18544bab40021bbc90e87a6caadd Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 19 Dec 2024 17:15:15 +0800 Subject: [PATCH 06/27] refactoring scheduled tasks --- internal/api/router.go | 16 +++ internal/rpc/conversation/conversation.go | 29 ++++++ internal/rpc/msg/clear.go | 19 ++-- internal/tools/msg.go | 2 +- internal/tools/s3.go | 58 +++++++++-- internal/tools/user_msg.go | 31 +++--- pkg/common/storage/controller/conversation.go | 6 ++ pkg/common/storage/controller/msg.go | 6 ++ pkg/common/storage/database/conversation.go | 1 + .../storage/database/mgo/conversation.go | 32 ++++++ pkg/common/storage/database/mgo/msg.go | 99 ++++++++++--------- pkg/common/storage/database/mgo/msg_test.go | 41 ++++---- pkg/common/storage/database/msg.go | 2 + 13 files changed, 247 insertions(+), 95 deletions(-) diff --git a/internal/api/router.go b/internal/api/router.go index d6bf4e130b..33f16ade32 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -1,7 +1,9 @@ package api import ( + "context" "fmt" + "github.com/openimsdk/protocol/user" "net/http" "strings" @@ -164,6 +166,7 @@ func newGinRouter(disCov discovery.SvcDiscoveryRegistry, config *Config) *gin.En authRouterGroup.POST("/get_user_token", a.GetUserToken) authRouterGroup.POST("/parse_token", a.ParseToken) authRouterGroup.POST("/force_logout", a.ForceLogout) + } // Third service thirdGroup := r.Group("/third") @@ -297,3 +300,16 @@ var Whitelist = []string{ "/auth/get_admin_token", "/auth/parse_token", } + +func init() { + + var uc user.UserClient + var g *gin.Engine + g.POST("/get_admin_token", New(uc, uc.AccountCheck)) + +} + +func New[A, B, C any](c C, fn func(c C, ctx context.Context, req *A, opts ...grpc.CallOption) (*B, error)) func(c *gin.Context) { + + return nil +} diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index f534ca6dea..9fcd5abb1d 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -763,3 +763,32 @@ func (c *conversationServer) GetPinnedConversationIDs(ctx context.Context, req * } return &pbconversation.GetPinnedConversationIDsResp{ConversationIDs: conversationIDs}, nil } + +func (c *conversationServer) ClearUserConversationMsg(ctx context.Context, req *pbconversation.ClearUserConversationMsgReq) (*pbconversation.ClearUserConversationMsgResp, error) { + conversations, err := c.conversationDatabase.FindRandConversation(ctx, req.Timestamp, int(req.Limit)) + if err != nil { + return nil, err + } + for _, conversation := range conversations { + if conversation.IsMsgDestruct == false || conversation.MsgDestructTime == 0 { + continue + } + rcpReq := &pbmsg.GetLastMessageSeqByTimeReq{ConversationID: conversation.ConversationID, Time: req.Timestamp - conversation.MsgDestructTime} + resp, err := pbmsg.GetLastMessageSeqByTime.Invoke(ctx, rcpReq) + if err != nil { + return nil, err + } + if resp.Seq == 0 { + continue + } + _, err = c.SetConversationMinSeq(ctx, &pbconversation.SetConversationMinSeqReq{ + ConversationID: conversation.ConversationID, + OwnerUserID: []string{conversation.OwnerUserID}, + MinSeq: resp.Seq + 1, + }) + if err != nil { + return nil, err + } + } + return &pbconversation.ClearUserConversationMsgResp{}, nil +} diff --git a/internal/rpc/msg/clear.go b/internal/rpc/msg/clear.go index 8d4bd0c8ee..eb7a4b7a60 100644 --- a/internal/rpc/msg/clear.go +++ b/internal/rpc/msg/clear.go @@ -10,7 +10,6 @@ import ( pbconv "github.com/openimsdk/protocol/conversation" "github.com/openimsdk/protocol/msg" "github.com/openimsdk/protocol/wrapperspb" - "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" "github.com/openimsdk/tools/utils/datautil" @@ -20,16 +19,10 @@ import ( ) // hard delete in Database. -func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) (_ *msg.DestructMsgsResp, err error) { +func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) (*msg.DestructMsgsResp, error) { if err := authverify.CheckAdmin(ctx, m.config.Share.IMAdminUserID); err != nil { return nil, err } - if req.Timestamp > time.Now().UnixMilli() { - return nil, errs.ErrArgs.WrapMsg("request millisecond timestamp error") - } - if req.Limit <= 0 { - return nil, errs.ErrArgs.WrapMsg("request limit error") - } docs, err := m.MsgDatabase.GetRandBeforeMsg(ctx, req.Timestamp, int(req.Limit)) if err != nil { return nil, err @@ -66,7 +59,7 @@ func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) } // soft delete for user self -func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg.ClearMsgResp, err error) { +func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (*msg.ClearMsgResp, error) { temp := convert.ConversationsPb2DB(req.Conversations) batchNum := 100 @@ -128,3 +121,11 @@ func (m *msgServer) ClearMsg(ctx context.Context, req *msg.ClearMsgReq) (_ *msg. return nil, nil } + +func (m *msgServer) GetLastMessageSeqByTime(ctx context.Context, req *msg.GetLastMessageSeqByTimeReq) (*msg.GetLastMessageSeqByTimeResp, error) { + seq, err := m.MsgDatabase.GetLastMessageSeqByTime(ctx, req.ConversationID, req.Time) + if err != nil { + return nil, err + } + return &msg.GetLastMessageSeqByTimeResp{Seq: seq}, nil +} diff --git a/internal/tools/msg.go b/internal/tools/msg.go index 46f1573f28..5226496450 100644 --- a/internal/tools/msg.go +++ b/internal/tools/msg.go @@ -16,7 +16,7 @@ func (c *cronServer) deleteMsg() { ctx := mcontext.SetOperationID(c.ctx, operationID) log.ZDebug(ctx, "Destruct chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) const ( - deleteCount = 20 + deleteCount = 200 deleteLimit = 50 ) var count int diff --git a/internal/tools/s3.go b/internal/tools/s3.go index 3b0122c84e..469b0d0cb3 100644 --- a/internal/tools/s3.go +++ b/internal/tools/s3.go @@ -11,25 +11,69 @@ import ( func (c *cronServer) clearS3() { start := time.Now() - executeNum := 10 - // number of pagination. if need modify, need update value in third.DeleteOutdatedData - pageShowNumber := 500 deleteTime := start.Add(-time.Hour * 24 * time.Duration(c.config.CronTask.FileExpireTime)) operationID := fmt.Sprintf("cron_s3_%d_%d", os.Getpid(), deleteTime.UnixMilli()) ctx := mcontext.SetOperationID(c.ctx, operationID) log.ZDebug(ctx, "deleteoutDatedData", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) + const ( + deleteCount = 200 + deleteLimit = 100 + ) + var count int - for i := 1; i <= executeNum; i++ { - ctx := mcontext.SetOperationID(c.ctx, fmt.Sprintf("%s_%d", operationID, i)) - resp, err := c.thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli(), ObjectGroup: c.config.CronTask.DeleteObjectType, Limit: int32(pageShowNumber)}) + for i := 1; i <= deleteCount; i++ { + resp, err := c.thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: deleteTime.UnixMilli(), ObjectGroup: c.config.CronTask.DeleteObjectType, Limit: deleteLimit}) if err != nil { log.ZError(ctx, "cron deleteoutDatedData failed", err) return } count += int(resp.Count) - if resp.Count < int32(pageShowNumber) { + if resp.Count < deleteLimit { break } } log.ZDebug(ctx, "cron deleteoutDatedData success", "deltime", deleteTime, "cont", time.Since(start), "count", count) } + +// var req *third.DeleteOutdatedDataReq +// count1, err := ExtractField(ctx, c.thirdClient.DeleteOutdatedData, req, (*third.DeleteOutdatedDataResp).GetCount) +// +// c.thirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{}) +// msggateway.GetUsersOnlineStatusCaller.Invoke(ctx, &msggateway.GetUsersOnlineStatusReq{}) +// +// var cli ThirdClient +// +// c111, err := cli.DeleteOutdatedData(ctx, 100) +// +// cli.ThirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{}) +// +// cli.AuthSign(ctx, &third.AuthSignReq{}) +// +// cli.SetAppBadge() +// +//} +// +//func extractField[A, B, C any](ctx context.Context, fn func(ctx context.Context, req *A, opts ...grpc.CallOption) (*B, error), req *A, get func(*B) C) (C, error) { +// resp, err := fn(ctx, req) +// if err != nil { +// var c C +// return c, err +// } +// return get(resp), nil +//} +// +//func ignore(_ any, err error) error { +// return err +//} +// +//type ThirdClient struct { +// third.ThirdClient +//} +// +//func (c *ThirdClient) DeleteOutdatedData(ctx context.Context, expireTime int64) (int32, error) { +// return extractField(ctx, c.ThirdClient.DeleteOutdatedData, &third.DeleteOutdatedDataReq{ExpireTime: expireTime}, (*third.DeleteOutdatedDataResp).GetCount) +//} +// +//func (c *ThirdClient) DeleteOutdatedData1(ctx context.Context, expireTime int64) error { +// return ignore(c.ThirdClient.DeleteOutdatedData(ctx, &third.DeleteOutdatedDataReq{ExpireTime: expireTime})) +//} diff --git a/internal/tools/user_msg.go b/internal/tools/user_msg.go index 52a2b6bdce..3d4ca40d3f 100644 --- a/internal/tools/user_msg.go +++ b/internal/tools/user_msg.go @@ -3,7 +3,6 @@ package tools import ( "fmt" pbconversation "github.com/openimsdk/protocol/conversation" - "github.com/openimsdk/protocol/msg" "github.com/openimsdk/tools/log" "github.com/openimsdk/tools/mcontext" "os" @@ -14,18 +13,22 @@ func (c *cronServer) clearUserMsg() { now := time.Now() operationID := fmt.Sprintf("cron_user_msg_%d_%d", os.Getpid(), now.UnixMilli()) ctx := mcontext.SetOperationID(c.ctx, operationID) - log.ZDebug(ctx, "clear msg cron start") - conversations, err := c.conversationClient.GetConversationsNeedClearMsg(ctx, &pbconversation.GetConversationsNeedClearMsgReq{}) - if err != nil { - log.ZError(ctx, "Get conversation need Destruct msgs failed.", err) - return + log.ZDebug(ctx, "clear user msg cron start") + const ( + deleteCount = 200 + deleteLimit = 100 + ) + var count int + for i := 1; i <= deleteCount; i++ { + resp, err := c.conversationClient.ClearUserConversationMsg(ctx, &pbconversation.ClearUserConversationMsgReq{Timestamp: now.UnixMilli(), Limit: deleteLimit}) + if err != nil { + log.ZError(ctx, "ClearUserConversationMsg failed.", err) + return + } + count += int(resp.Count) + if resp.Count < deleteLimit { + break + } } - - _, err = c.msgClient.ClearMsg(ctx, &msg.ClearMsgReq{Conversations: conversations.Conversations}) - if err != nil { - log.ZError(ctx, "Clear Msg failed.", err) - return - } - - log.ZDebug(ctx, "clear msg cron task completed", "cont", time.Since(now)) + log.ZDebug(ctx, "clear user msg cron task completed", "cont", time.Since(now), "count", count) } diff --git a/pkg/common/storage/controller/conversation.go b/pkg/common/storage/controller/conversation.go index bf41cce957..d4088e0c0e 100644 --- a/pkg/common/storage/controller/conversation.go +++ b/pkg/common/storage/controller/conversation.go @@ -74,6 +74,8 @@ type ConversationDatabase interface { GetNotNotifyConversationIDs(ctx context.Context, userID string) ([]string, error) // GetPinnedConversationIDs gets pinned conversationIDs by userID GetPinnedConversationIDs(ctx context.Context, userID string) ([]string, error) + // FindRandConversation finds random conversations based on the specified timestamp and limit. + FindRandConversation(ctx context.Context, ts int64, limit int) ([]*relationtb.Conversation, error) } func NewConversationDatabase(conversation database.Conversation, cache cache.ConversationCache, tx tx.Tx) ConversationDatabase { @@ -401,3 +403,7 @@ func (c *conversationDatabase) GetPinnedConversationIDs(ctx context.Context, use } return conversationIDs, nil } + +func (c *conversationDatabase) FindRandConversation(ctx context.Context, ts int64, limit int) ([]*relationtb.Conversation, error) { + return c.conversationDB.FindRandConversation(ctx, ts, limit) +} diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index 88089f1a09..c29544c333 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -103,6 +103,8 @@ type CommonMsgDatabase interface { SetUserConversationsMinSeq(ctx context.Context, conversationID string, userID string, seq int64) error DeleteDoc(ctx context.Context, docID string) error + + GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) } func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { @@ -1016,3 +1018,7 @@ func (db *commonMsgDatabase) GetMaxSeqsWithTime(ctx context.Context, conversatio func (db *commonMsgDatabase) DeleteDoc(ctx context.Context, docID string) error { return db.msgDocDatabase.DeleteDoc(ctx, docID) } + +func (db *commonMsgDatabase) GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) { + return db.msgDocDatabase.GetLastMessageSeqByTime(ctx, conversationID, time) +} diff --git a/pkg/common/storage/database/conversation.go b/pkg/common/storage/database/conversation.go index 30ca01ee71..1fb53cfed2 100644 --- a/pkg/common/storage/database/conversation.go +++ b/pkg/common/storage/database/conversation.go @@ -42,4 +42,5 @@ type Conversation interface { GetConversationIDsNeedDestruct(ctx context.Context) ([]*model.Conversation, error) GetConversationNotReceiveMessageUserIDs(ctx context.Context, conversationID string) ([]string, error) FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) + FindRandConversation(ctx context.Context, ts int64, limit int) ([]*model.Conversation, error) } diff --git a/pkg/common/storage/database/mgo/conversation.go b/pkg/common/storage/database/mgo/conversation.go index 10e223c893..851ec99c40 100644 --- a/pkg/common/storage/database/mgo/conversation.go +++ b/pkg/common/storage/database/mgo/conversation.go @@ -228,3 +228,35 @@ func (c *ConversationMgo) GetConversationNotReceiveMessageUserIDs(ctx context.Co func (c *ConversationMgo) FindConversationUserVersion(ctx context.Context, userID string, version uint, limit int) (*model.VersionLog, error) { return c.version.FindChangeLog(ctx, userID, version, limit) } + +func (c *ConversationMgo) FindRandConversation(ctx context.Context, ts int64, limit int) ([]*model.Conversation, error) { + pipeline := []bson.M{ + { + "$match": bson.M{ + "is_msg_destruct": true, + "msg_destruct_time": bson.M{"$ne": 0}, + }, + }, + { + "$addFields": bson.M{ + "next_msg_destruct_timestamp": bson.M{ + "$add": []any{ + bson.M{ + "$toLong": "$latest_msg_destruct_time", + }, "$msg_destruct_time"}, + }, + }, + }, + { + "$match": bson.M{ + "next_msg_destruct_timestamp": bson.M{"$lt": ts}, + }, + }, + { + "$sample": bson.M{ + "size": limit, + }, + }, + } + return mongoutil.Aggregate[*model.Conversation](ctx, c.coll, pipeline) +} diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index 937dbae1dd..e014654663 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -1290,7 +1290,9 @@ func (m *MsgMgo) GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]* }, }, { - "$sample": limit, + "$sample": bson.M{ + "size": limit, + }, }, }) } @@ -1307,53 +1309,56 @@ func (m *MsgMgo) DeleteMsgByIndex(ctx context.Context, docID string, index []int return mongoutil.UpdateOne(ctx, m.coll, bson.M{"doc_id": docID}, bson.M{"$set": set}, true) } -//func (m *MsgMgo) ClearMsg(ctx context.Context, t time.Time) (int64, error) { -// ts := t.UnixMilli() -// var count int64 -// for { -// msgs, err := m.GetBeforeMsg(ctx, ts, 100) -// if err != nil { -// return count, err -// } -// if len(msgs) == 0 { -// return count, nil -// } -// for _, msg := range msgs { -// num, err := m.deleteOneMsg(ctx, ts, msg) -// count += num -// if err != nil { -// return count, err -// } -// } -// } -//} - func (m *MsgMgo) DeleteDoc(ctx context.Context, docID string) error { return mongoutil.DeleteOne(ctx, m.coll, bson.M{"doc_id": docID}) } -//func (m *MsgMgo) DeleteDocMsg(ctx context.Context, ts int64, doc *relation.MsgDocModel) (int64, error) { -// var notNull int -// index := make([]int, 0, len(doc.Msg)) -// for i, message := range doc.Msg { -// if message.Msg != nil { -// notNull++ -// if message.Msg.SendTime < ts { -// index = append(index, i) -// } -// } -// } -// if len(index) == 0 { -// return 0, errs.New("no msg to delete").WrapMsg("deleteOneMsg", "docID", doc.DocID) -// } -// if len(index) == notNull { -// if err := m.DeleteDoc(ctx, doc.DocID); err != nil { -// return 0, err -// } -// } else { -// if err := m.setNullMsg(ctx, doc.DocID, index); err != nil { -// return 0, err -// } -// } -// return int64(len(index)), nil -//} +func (m *MsgMgo) GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) { + pipeline := []bson.M{ + { + "$match": bson.M{ + "doc_id": bson.M{ + "$regex": fmt.Sprintf("^%s:", conversationID), + }, + }, + }, + { + "$match": bson.M{ + "msgs.msg.send_time": bson.M{ + "msgs.msg.send_time": bson.M{ + "$lte": time, + }, + }, + }, + }, + { + "$sort": bson.M{ + "_id": -1, + }, + }, + { + "$limit": 1, + }, + { + "$project": bson.M{ + "_id": 0, + "msgs.msg.send_time": 1, + "msgs.msg.seq": 1, + }, + }, + } + res, err := mongoutil.Aggregate[*model.MsgDocModel](ctx, m.coll, pipeline) + if err != nil { + return 0, err + } + if len(res) == 0 { + return 0, nil + } + var seq int64 + for _, v := range res[0].Msg { + if v.Msg.SendTime <= time { + seq = v.Msg.Seq + } + } + return seq, nil +} diff --git a/pkg/common/storage/database/mgo/msg_test.go b/pkg/common/storage/database/mgo/msg_test.go index eff847d262..7218ef1ad1 100644 --- a/pkg/common/storage/database/mgo/msg_test.go +++ b/pkg/common/storage/database/mgo/msg_test.go @@ -15,10 +15,10 @@ import ( ) func TestName1(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) - defer cancel() - cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.66:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) - + //ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) + //defer cancel() + //cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.66:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) + // //v := &MsgMgo{ // coll: cli.Database("openim_v3").Collection("msg3"), //} @@ -44,16 +44,16 @@ func TestName1(t *testing.T) { //} // //t.Log(total) - - msg, err := NewMsgMongo(cli.Database("openim_v3")) - if err != nil { - panic(err) - } - res, err := msg.GetBeforeMsg(ctx, time.Now().UnixMilli(), []string{"1:0"}, 1000) - if err != nil { - panic(err) - } - t.Log(len(res)) + // + //msg, err := NewMsgMongo(cli.Database("openim_v3")) + //if err != nil { + // panic(err) + //} + //res, err := msg.GetBeforeMsg(ctx, time.Now().UnixMilli(), []string{"1:0"}, 1000) + //if err != nil { + // panic(err) + //} + //t.Log(len(res)) } func TestName10(t *testing.T) { @@ -95,13 +95,20 @@ func TestName4(t *testing.T) { defer cancel() cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.66:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) - msg, err := NewMsgMongo(cli.Database("openim_v3")) + msg, err := NewConversationMongo(cli.Database("openim_v3")) if err != nil { panic(err) } - res, err := msg.GetBeforeMsg(ctx, time.Now().UnixMilli(), []string{"1:0"}, 1000) + ts := time.Now().UnixMilli() + t.Log(ts) + res, err := msg.FindRandConversation(ctx, ts, 10) if err != nil { panic(err) } - t.Log(len(res)) + t.Log(res) +} + +func TestName5(t *testing.T) { + var v time.Time + t.Log(v.UnixMilli()) } diff --git a/pkg/common/storage/database/msg.go b/pkg/common/storage/database/msg.go index 17aaf23429..abb2a44c2f 100644 --- a/pkg/common/storage/database/msg.go +++ b/pkg/common/storage/database/msg.go @@ -48,4 +48,6 @@ type Msg interface { GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) GetRandDocIDs(ctx context.Context, limit int) ([]string, error) + + GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) } From 1949d6c75684e7fd0bbd349472fac581c8111725 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 19 Dec 2024 17:34:34 +0800 Subject: [PATCH 07/27] refactoring scheduled tasks --- internal/api/router.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/internal/api/router.go b/internal/api/router.go index 33f16ade32..5e8038068c 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -1,9 +1,7 @@ package api import ( - "context" "fmt" - "github.com/openimsdk/protocol/user" "net/http" "strings" @@ -300,16 +298,3 @@ var Whitelist = []string{ "/auth/get_admin_token", "/auth/parse_token", } - -func init() { - - var uc user.UserClient - var g *gin.Engine - g.POST("/get_admin_token", New(uc, uc.AccountCheck)) - -} - -func New[A, B, C any](c C, fn func(c C, ctx context.Context, req *A, opts ...grpc.CallOption) (*B, error)) func(c *gin.Context) { - - return nil -} From 86d58253c96e0e7ac7a1285df46c816a803c366f Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 19 Dec 2024 18:53:49 +0800 Subject: [PATCH 08/27] refactoring scheduled tasks --- internal/rpc/third/s3.go | 21 +++++--- internal/tools/cron_test.go | 63 +++++++++++++++++++++++ internal/tools/msg.go | 6 +-- internal/tools/s3.go | 2 +- internal/tools/user_msg.go | 2 +- pkg/common/storage/controller/s3.go | 6 +++ pkg/common/storage/database/mgo/object.go | 4 ++ pkg/common/storage/database/object.go | 1 + 8 files changed, 92 insertions(+), 13 deletions(-) create mode 100644 internal/tools/cron_test.go diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index a541462a28..e1b42ca338 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -295,18 +295,23 @@ func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteO if err != nil { return nil, err } - if len(models) > 0 { - names := datautil.Batch(func(o *model.Object) string { - return o.Name - }, models) - if err := t.s3dataBase.DeleteSpecifiedData(ctx, engine, names); err != nil { + keyCount := make(map[string]int) + for _, obj := range models { + keyCount[obj.Key]++ + } + for _, obj := range models { + count, err := t.s3dataBase.GetKeyCount(ctx, engine, obj.Key) + if err != nil { + return nil, err + } + if err := t.s3dataBase.DeleteSpecifiedData(ctx, engine, []string{obj.Name}); err != nil { return nil, errs.Wrap(err) } - if err := t.s3dataBase.DelS3Key(ctx, engine, names...); err != nil { + if err := t.s3dataBase.DelS3Key(ctx, engine, obj.Name); err != nil { return nil, err } - for _, object := range models { - if err := t.s3.DeleteObject(ctx, object.Key); err != nil { + if int(count) <= keyCount[obj.Key] { + if err := t.s3.DeleteObject(ctx, obj.Key); err != nil { return nil, err } } diff --git a/internal/tools/cron_test.go b/internal/tools/cron_test.go new file mode 100644 index 0000000000..8903490698 --- /dev/null +++ b/internal/tools/cron_test.go @@ -0,0 +1,63 @@ +package tools + +import ( + "context" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + kdisc "github.com/openimsdk/open-im-server/v3/pkg/common/discoveryregister" + pbconversation "github.com/openimsdk/protocol/conversation" + "github.com/openimsdk/protocol/msg" + "github.com/openimsdk/protocol/third" + "github.com/openimsdk/tools/mcontext" + "github.com/openimsdk/tools/mw" + "github.com/robfig/cron/v3" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "testing" +) + +func TestName(t *testing.T) { + conf := &config.Discovery{ + Enable: config.ETCD, + Etcd: config.Etcd{ + RootDirectory: "openim", + Address: []string{"localhost:12379"}, + }, + } + client, err := kdisc.NewDiscoveryRegister(conf, "source") + if err != nil { + panic(err) + } + client.AddOption(mw.GrpcClient(), grpc.WithTransportCredentials(insecure.NewCredentials())) + ctx := mcontext.SetOpUserID(context.Background(), "imAdmin") + msgConn, err := client.GetConn(ctx, "msg-rpc-service") + if err != nil { + panic(err) + } + thirdConn, err := client.GetConn(ctx, "third-rpc-service") + if err != nil { + panic(err) + } + + conversationConn, err := client.GetConn(ctx, "conversation-rpc-service") + if err != nil { + panic(err) + } + + srv := &cronServer{ + ctx: ctx, + config: &CronTaskConfig{ + CronTask: config.CronTask{ + RetainChatRecords: 1, + FileExpireTime: 1, + DeleteObjectType: []string{"msg-picture", "msg-file", "msg-voice", "msg-video", "msg-video-snapshot", "sdklog", ""}, + }, + }, + cron: cron.New(), + msgClient: msg.NewMsgClient(msgConn), + conversationClient: pbconversation.NewConversationClient(conversationConn), + thirdClient: third.NewThirdClient(thirdConn), + } + srv.deleteMsg() + //srv.clearS3() + //srv.clearUserMsg() +} diff --git a/internal/tools/msg.go b/internal/tools/msg.go index 5226496450..cc00cc5b83 100644 --- a/internal/tools/msg.go +++ b/internal/tools/msg.go @@ -16,7 +16,7 @@ func (c *cronServer) deleteMsg() { ctx := mcontext.SetOperationID(c.ctx, operationID) log.ZDebug(ctx, "Destruct chat records", "deltime", deltime, "timestamp", deltime.UnixMilli()) const ( - deleteCount = 200 + deleteCount = 10000 deleteLimit = 50 ) var count int @@ -28,9 +28,9 @@ func (c *cronServer) deleteMsg() { break } count += int(resp.Count) - if resp.Count <= deleteLimit { + if resp.Count < deleteLimit { break } } - log.ZDebug(ctx, "cron destruct chat records end", "deltime", deltime, "cont", time.Since(now), "deleteDocs", count) + log.ZDebug(ctx, "cron destruct chat records end", "deltime", deltime, "cont", time.Since(now), "count", count) } diff --git a/internal/tools/s3.go b/internal/tools/s3.go index 469b0d0cb3..9b6b9c4089 100644 --- a/internal/tools/s3.go +++ b/internal/tools/s3.go @@ -16,7 +16,7 @@ func (c *cronServer) clearS3() { ctx := mcontext.SetOperationID(c.ctx, operationID) log.ZDebug(ctx, "deleteoutDatedData", "deletetime", deleteTime, "timestamp", deleteTime.UnixMilli()) const ( - deleteCount = 200 + deleteCount = 10000 deleteLimit = 100 ) diff --git a/internal/tools/user_msg.go b/internal/tools/user_msg.go index 3d4ca40d3f..a4afa769ed 100644 --- a/internal/tools/user_msg.go +++ b/internal/tools/user_msg.go @@ -15,7 +15,7 @@ func (c *cronServer) clearUserMsg() { ctx := mcontext.SetOperationID(c.ctx, operationID) log.ZDebug(ctx, "clear user msg cron start") const ( - deleteCount = 200 + deleteCount = 10000 deleteLimit = 100 ) var count int diff --git a/pkg/common/storage/controller/s3.go b/pkg/common/storage/controller/s3.go index 6a0f11b1ce..6693d2ddea 100644 --- a/pkg/common/storage/controller/s3.go +++ b/pkg/common/storage/controller/s3.go @@ -42,6 +42,7 @@ type S3Database interface { FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) DeleteSpecifiedData(ctx context.Context, engine string, name []string) error DelS3Key(ctx context.Context, engine string, keys ...string) error + GetKeyCount(ctx context.Context, engine string, key string) (int64, error) } func NewS3Database(rdb redis.UniversalClient, s3 s3.Interface, obj database.ObjectInfo) S3Database { @@ -117,10 +118,15 @@ func (s *s3Database) StatObject(ctx context.Context, name string) (*s3.ObjectInf func (s *s3Database) FormData(ctx context.Context, name string, size int64, contentType string, duration time.Duration) (*s3.FormData, error) { return s.s3.FormData(ctx, name, size, contentType, duration) } + func (s *s3Database) FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) { return s.db.FindExpirationObject(ctx, engine, expiration, needDelType, count) } +func (s *s3Database) GetKeyCount(ctx context.Context, engine string, key string) (int64, error) { + return s.db.GetKeyCount(ctx, engine, key) +} + func (s *s3Database) DeleteSpecifiedData(ctx context.Context, engine string, name []string) error { return s.db.Delete(ctx, engine, name) } diff --git a/pkg/common/storage/database/mgo/object.go b/pkg/common/storage/database/mgo/object.go index 53f2a6ba43..624eb84a0d 100644 --- a/pkg/common/storage/database/mgo/object.go +++ b/pkg/common/storage/database/mgo/object.go @@ -108,3 +108,7 @@ func (o *S3Mongo) FindExpirationObject(ctx context.Context, engine string, expir "group": bson.M{"$in": needDelType}, }, opt) } + +func (o *S3Mongo) GetKeyCount(ctx context.Context, engine string, key string) (int64, error) { + return mongoutil.Count(ctx, o.coll, bson.M{"engine": engine, "key": key}) +} diff --git a/pkg/common/storage/database/object.go b/pkg/common/storage/database/object.go index 86d47a2a6b..5541a159b4 100644 --- a/pkg/common/storage/database/object.go +++ b/pkg/common/storage/database/object.go @@ -26,4 +26,5 @@ type ObjectInfo interface { Take(ctx context.Context, engine string, name string) (*model.Object, error) Delete(ctx context.Context, engine string, name []string) error FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) + GetKeyCount(ctx context.Context, engine string, key string) (int64, error) } From 46a8d171049faefba46f0c3cc19a9c4099b35cc7 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 20 Dec 2024 15:26:32 +0800 Subject: [PATCH 09/27] refactoring scheduled tasks --- go.mod | 4 +-- go.sum | 8 ++--- internal/rpc/conversation/conversation.go | 39 ++++++++++++++++----- internal/rpc/msg/clear.go | 5 ++- internal/rpc/third/s3.go | 17 ++++----- pkg/common/storage/database/mgo/msg.go | 10 +++--- pkg/common/storage/database/mgo/msg_test.go | 6 ++-- 7 files changed, 56 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index 4eaa18ccc8..6c1d421c84 100644 --- a/go.mod +++ b/go.mod @@ -14,8 +14,8 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.66 - github.com/openimsdk/tools v0.0.50-alpha.57 + github.com/openimsdk/protocol v0.0.72-alpha.67 + github.com/openimsdk/tools v0.0.50-alpha.58 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 6e283eb668..86ed9ed6e3 100644 --- a/go.sum +++ b/go.sum @@ -347,10 +347,10 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrkXcDACCqtyM= github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.66 h1:5KoDY6M4T+pXg449ScF6hqeQ+WenBwNyUJn/t8W0oBQ= -github.com/openimsdk/protocol v0.0.72-alpha.66/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.57 h1:oIKV6vYhqp7TRmZ6Pe+r9RNl1D5s7aB/kE9yQVEWcSY= -github.com/openimsdk/tools v0.0.50-alpha.57/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= +github.com/openimsdk/protocol v0.0.72-alpha.67 h1:zlLbVkoT0OYsjO2RCutQuDFllcfNvZfdYchvlR6UIe0= +github.com/openimsdk/protocol v0.0.72-alpha.67/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= +github.com/openimsdk/tools v0.0.50-alpha.58 h1:hkFL02Bzzp/l5x+tb7kJ9zes7hilh65EQ4qEIthsQX4= +github.com/openimsdk/tools v0.0.50-alpha.58/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index 9fcd5abb1d..696ada1521 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -769,7 +769,8 @@ func (c *conversationServer) ClearUserConversationMsg(ctx context.Context, req * if err != nil { return nil, err } - for _, conversation := range conversations { + latestMsgDestructTime := time.UnixMilli(req.Timestamp) + for i, conversation := range conversations { if conversation.IsMsgDestruct == false || conversation.MsgDestructTime == 0 { continue } @@ -778,17 +779,37 @@ func (c *conversationServer) ClearUserConversationMsg(ctx context.Context, req * if err != nil { return nil, err } - if resp.Seq == 0 { + if resp.Seq <= 0 { + log.ZDebug(ctx, "ClearUserConversationMsg GetLastMessageSeqByTime seq <= 0", "index", i, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID, "msgDestructTime", conversation.MsgDestructTime, "seq", resp.Seq) + if err := c.setConversationMinSeqAndLatestMsgDestructTime(ctx, conversation.ConversationID, conversation.OwnerUserID, -1, latestMsgDestructTime); err != nil { + return nil, err + } continue } - _, err = c.SetConversationMinSeq(ctx, &pbconversation.SetConversationMinSeqReq{ - ConversationID: conversation.ConversationID, - OwnerUserID: []string{conversation.OwnerUserID}, - MinSeq: resp.Seq + 1, - }) - if err != nil { + resp.Seq++ + if err := c.setConversationMinSeqAndLatestMsgDestructTime(ctx, conversation.ConversationID, conversation.OwnerUserID, resp.Seq, latestMsgDestructTime); err != nil { return nil, err } + log.ZDebug(ctx, "ClearUserConversationMsg set min seq", "index", i, "conversationID", conversation.ConversationID, "ownerUserID", conversation.OwnerUserID, "seq", resp.Seq, "msgDestructTime", conversation.MsgDestructTime) + } + return &pbconversation.ClearUserConversationMsgResp{Count: int32(len(conversations))}, nil +} + +func (c *conversationServer) setConversationMinSeqAndLatestMsgDestructTime(ctx context.Context, conversationID string, ownerUserID string, minSeq int64, latestMsgDestructTime time.Time) error { + update := map[string]any{ + "latest_msg_destruct_time": latestMsgDestructTime, } - return &pbconversation.ClearUserConversationMsgResp{}, nil + if minSeq >= 0 { + req := &pbmsg.SetUserConversationMinSeqReq{ConversationID: conversationID, OwnerUserID: []string{ownerUserID}, MinSeq: minSeq} + if _, err := pbmsg.SetUserConversationMinSeqCaller.Invoke(ctx, req); err != nil { + return err + } + update["min_seq"] = minSeq + } + + if err := c.conversationDatabase.UpdateUsersConversationField(ctx, []string{ownerUserID}, conversationID, update); err != nil { + return err + } + c.conversationNotificationSender.ConversationChangeNotification(ctx, ownerUserID, []string{conversationID}) + return nil } diff --git a/internal/rpc/msg/clear.go b/internal/rpc/msg/clear.go index eb7a4b7a60..7a2d363009 100644 --- a/internal/rpc/msg/clear.go +++ b/internal/rpc/msg/clear.go @@ -27,10 +27,11 @@ func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) if err != nil { return nil, err } - for _, doc := range docs { + for i, doc := range docs { if err := m.MsgDatabase.DeleteDoc(ctx, doc.DocID); err != nil { return nil, err } + log.ZDebug(ctx, "DestructMsgs delete doc", "index", i, "docID", doc.DocID) index := strings.LastIndex(doc.DocID, ":") if index < 0 { continue @@ -51,9 +52,11 @@ func (m *msgServer) DestructMsgs(ctx context.Context, req *msg.DestructMsgsReq) if conversationID == "" { continue } + minSeq++ if err := m.MsgDatabase.SetMinSeq(ctx, conversationID, minSeq); err != nil { return nil, err } + log.ZDebug(ctx, "DestructMsgs delete doc set min seq", "index", i, "docID", doc.DocID, "conversationID", conversationID, "setMinSeq", minSeq) } return &msg.DestructMsgsResp{Count: int32(len(docs))}, nil } diff --git a/internal/rpc/third/s3.go b/internal/rpc/third/s3.go index e1b42ca338..8796fe824e 100644 --- a/internal/rpc/third/s3.go +++ b/internal/rpc/third/s3.go @@ -295,22 +295,19 @@ func (t *thirdServer) DeleteOutdatedData(ctx context.Context, req *third.DeleteO if err != nil { return nil, err } - keyCount := make(map[string]int) - for _, obj := range models { - keyCount[obj.Key]++ - } - for _, obj := range models { - count, err := t.s3dataBase.GetKeyCount(ctx, engine, obj.Key) - if err != nil { - return nil, err - } + for i, obj := range models { if err := t.s3dataBase.DeleteSpecifiedData(ctx, engine, []string{obj.Name}); err != nil { return nil, errs.Wrap(err) } if err := t.s3dataBase.DelS3Key(ctx, engine, obj.Name); err != nil { return nil, err } - if int(count) <= keyCount[obj.Key] { + count, err := t.s3dataBase.GetKeyCount(ctx, engine, obj.Key) + if err != nil { + return nil, err + } + log.ZDebug(ctx, "delete s3 object record", "index", i, "s3", obj, "count", count) + if count == 0 { if err := t.s3.DeleteObject(ctx, obj.Key); err != nil { return nil, err } diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index e014654663..f371766958 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -1318,16 +1318,14 @@ func (m *MsgMgo) GetLastMessageSeqByTime(ctx context.Context, conversationID str { "$match": bson.M{ "doc_id": bson.M{ - "$regex": fmt.Sprintf("^%s:", conversationID), + "$regex": fmt.Sprintf("^%s", conversationID), }, }, }, { "$match": bson.M{ "msgs.msg.send_time": bson.M{ - "msgs.msg.send_time": bson.M{ - "$lte": time, - }, + "$lte": time, }, }, }, @@ -1342,6 +1340,7 @@ func (m *MsgMgo) GetLastMessageSeqByTime(ctx context.Context, conversationID str { "$project": bson.M{ "_id": 0, + "doc_id": 1, "msgs.msg.send_time": 1, "msgs.msg.seq": 1, }, @@ -1356,6 +1355,9 @@ func (m *MsgMgo) GetLastMessageSeqByTime(ctx context.Context, conversationID str } var seq int64 for _, v := range res[0].Msg { + if v.Msg == nil { + continue + } if v.Msg.SendTime <= time { seq = v.Msg.Seq } diff --git a/pkg/common/storage/database/mgo/msg_test.go b/pkg/common/storage/database/mgo/msg_test.go index 7218ef1ad1..992090552e 100644 --- a/pkg/common/storage/database/mgo/msg_test.go +++ b/pkg/common/storage/database/mgo/msg_test.go @@ -95,13 +95,13 @@ func TestName4(t *testing.T) { defer cancel() cli := Result(mongo.Connect(ctx, options.Client().ApplyURI("mongodb://openIM:openIM123@172.16.8.66:37017/openim_v3?maxPoolSize=100").SetConnectTimeout(5*time.Second))) - msg, err := NewConversationMongo(cli.Database("openim_v3")) + msg, err := NewMsgMongo(cli.Database("openim_v3")) if err != nil { panic(err) } - ts := time.Now().UnixMilli() + ts := time.Now().Add(-time.Hour * 24 * 5).UnixMilli() t.Log(ts) - res, err := msg.FindRandConversation(ctx, ts, 10) + res, err := msg.GetLastMessageSeqByTime(ctx, "sg_1523453548", ts) if err != nil { panic(err) } From 2e8ca5f5b938fa929b0123c2a66d246aa3537ed2 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 24 Dec 2024 17:50:05 +0800 Subject: [PATCH 10/27] upgrading pkg tools --- go.mod | 2 +- go.sum | 4 ++-- internal/msggateway/client.go | 4 ++-- internal/msgtransfer/init.go | 2 +- internal/msgtransfer/online_history_msg_handler.go | 2 +- internal/rpc/msg/send.go | 2 +- pkg/rpccache/subscriber.go | 3 ++- 7 files changed, 10 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 26f167a19e..05fe6db15e 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.68 - github.com/openimsdk/tools v0.0.50-alpha.60 + github.com/openimsdk/tools v0.0.50-alpha.61 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index c0385481ca..1fc3c33db3 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrk github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.68 h1:Ekn6S9Ftt12Xs/p9kJ39RDr2gSwIczz+MmSHQE4lAek= github.com/openimsdk/protocol v0.0.72-alpha.68/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.60 h1:dYqYpSdSN5o6CxlEjua2USfwfUiG0tUWFBpqghTjbWE= -github.com/openimsdk/tools v0.0.50-alpha.60/go.mod h1:muCtxguNJv8lFwLei27UASu2Nvg4ERSeN0R4K5tivk0= +github.com/openimsdk/tools v0.0.50-alpha.61 h1:zKEZwrj+fUVuyC6KR3kZp9zFaCCIFgoSbHO0r0mZ6h4= +github.com/openimsdk/tools v0.0.50-alpha.61/go.mod h1:JowL2jYr8tu4vcQe+5hJh4v3BtSx1T0CIS3pgU/Mw+U= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 161300d6d0..1040f2be23 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -131,7 +131,7 @@ func (c *Client) readMessage() { defer func() { if r := recover(); r != nil { c.closedErr = ErrPanic - log.ZPanic(c.ctx, "socket have panic err:", r) + log.ZPanic(c.ctx, "socket have panic err:", errs.ErrPanic(r)) } c.close() }() @@ -376,7 +376,7 @@ func (c *Client) activeHeartbeat(ctx context.Context) { go func() { defer func() { if r := recover(); r != nil { - log.ZPanic(ctx, "activeHeartbeat Panic", r) + log.ZPanic(ctx, "activeHeartbeat Panic", errs.ErrPanic(r)) } }() log.ZDebug(ctx, "server initiative send heartbeat start.") diff --git a/internal/msgtransfer/init.go b/internal/msgtransfer/init.go index 5cb6131238..ee9f066448 100644 --- a/internal/msgtransfer/init.go +++ b/internal/msgtransfer/init.go @@ -195,7 +195,7 @@ func (m *MsgTransfer) Start(index int, config *Config, client discovery.SvcDisco go func() { defer func() { if r := recover(); r != nil { - log.ZPanic(m.ctx, "MsgTransfer Start Panic", r) + log.ZPanic(m.ctx, "MsgTransfer Start Panic", errs.ErrPanic(r)) } }() if err := prommetrics.TransferInit(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { diff --git a/internal/msgtransfer/online_history_msg_handler.go b/internal/msgtransfer/online_history_msg_handler.go index 83b0750613..7b989070b0 100644 --- a/internal/msgtransfer/online_history_msg_handler.go +++ b/internal/msgtransfer/online_history_msg_handler.go @@ -361,7 +361,7 @@ func (och *OnlineHistoryRedisConsumerHandler) handleNotification(ctx context.Con func (och *OnlineHistoryRedisConsumerHandler) HandleUserHasReadSeqMessages(ctx context.Context) { defer func() { if r := recover(); r != nil { - log.ZPanic(ctx, "HandleUserHasReadSeqMessages Panic", r) + log.ZPanic(ctx, "HandleUserHasReadSeqMessages Panic", errs.ErrPanic(r)) } }() diff --git a/internal/rpc/msg/send.go b/internal/rpc/msg/send.go index b9bbf615f3..f01134f8f0 100644 --- a/internal/rpc/msg/send.go +++ b/internal/rpc/msg/send.go @@ -88,7 +88,7 @@ func (m *msgServer) setConversationAtInfo(nctx context.Context, msg *sdkws.MsgDa defer func() { if r := recover(); r != nil { - log.ZPanic(nctx, "setConversationAtInfo Panic", r) + log.ZPanic(nctx, "setConversationAtInfo Panic", errs.ErrPanic(r)) } }() diff --git a/pkg/rpccache/subscriber.go b/pkg/rpccache/subscriber.go index d28d1aa295..0cb35bebfb 100644 --- a/pkg/rpccache/subscriber.go +++ b/pkg/rpccache/subscriber.go @@ -17,6 +17,7 @@ package rpccache import ( "context" "encoding/json" + "github.com/openimsdk/tools/errs" "github.com/openimsdk/tools/log" "github.com/redis/go-redis/v9" ) @@ -24,7 +25,7 @@ import ( func subscriberRedisDeleteCache(ctx context.Context, client redis.UniversalClient, channel string, del func(ctx context.Context, key ...string)) { defer func() { if r := recover(); r != nil { - log.ZPanic(ctx, "subscriberRedisDeleteCache Panic", r) + log.ZPanic(ctx, "subscriberRedisDeleteCache Panic", errs.ErrPanic(r)) } }() for message := range client.Subscribe(ctx, channel).Channel() { From 030c4493dcad2f745e218a3a4e233c31a506f1d5 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 27 Dec 2024 17:19:46 +0800 Subject: [PATCH 11/27] fix --- internal/api/jssdk/jssdk.go | 21 ++++++++++++------- internal/api/router.go | 6 ++++-- .../msgtransfer/online_history_msg_handler.go | 2 +- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/internal/api/jssdk/jssdk.go b/internal/api/jssdk/jssdk.go index 036cb027ab..3c09112077 100644 --- a/internal/api/jssdk/jssdk.go +++ b/internal/api/jssdk/jssdk.go @@ -20,16 +20,23 @@ const ( defaultGetActiveConversation = 100 ) -func NewJSSdkApi() *JSSdk { - return &JSSdk{} +func NewJSSdkApi(userClient *rpcli.UserClient, relationClient *rpcli.RelationClient, groupClient *rpcli.GroupClient, + conversationClient *rpcli.ConversationClient, msgClient *rpcli.MsgClient) *JSSdk { + return &JSSdk{ + userClient: userClient, + relationClient: relationClient, + groupClient: groupClient, + conversationClient: conversationClient, + msgClient: msgClient, + } } type JSSdk struct { - userClient rpcli.UserClient - relationClient rpcli.RelationClient - groupClient rpcli.GroupClient - conversationClient rpcli.ConversationClient - msgClient rpcli.MsgClient + userClient *rpcli.UserClient + relationClient *rpcli.RelationClient + groupClient *rpcli.GroupClient + conversationClient *rpcli.ConversationClient + msgClient *rpcli.MsgClient } func (x *JSSdk) GetActiveConversations(c *gin.Context) { diff --git a/internal/api/router.go b/internal/api/router.go index 5c4fa21370..e516d8ca83 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -100,8 +100,8 @@ func newGinRouter(ctx context.Context, client discovery.SvcDiscoveryRegistry, cf case BestSpeed: r.Use(gzip.Gzip(gzip.BestSpeed)) } - r.Use(prommetricsGin(), gin.RecoveryWithWriter(gin.DefaultErrorWriter, mw.GinPanicErr), mw.CorsHandler(), mw.GinParseOperationID(), GinParseToken(rpcli.NewAuthClient(authConn))) - j := jssdk.NewJSSdkApi() + r.Use(prommetricsGin(), gin.RecoveryWithWriter(gin.DefaultErrorWriter, mw.GinPanicErr), mw.CorsHandler(), + mw.GinParseOperationID(), GinParseToken(rpcli.NewAuthClient(authConn))) u := NewUserApi(user.NewUserClient(userConn), client, cfg.Discovery.RpcService) { @@ -280,6 +280,8 @@ func newGinRouter(ctx context.Context, client discovery.SvcDiscoveryRegistry, cf } { + j := jssdk.NewJSSdkApi(rpcli.NewUserClient(userConn), rpcli.NewRelationClient(friendConn), + rpcli.NewGroupClient(groupConn), rpcli.NewConversationClient(conversationConn), rpcli.NewMsgClient(msgConn)) jssdk := r.Group("/jssdk") jssdk.POST("/get_conversations", j.GetConversations) jssdk.POST("/get_active_conversations", j.GetActiveConversations) diff --git a/internal/msgtransfer/online_history_msg_handler.go b/internal/msgtransfer/online_history_msg_handler.go index 7b989070b0..6334c95fd3 100644 --- a/internal/msgtransfer/online_history_msg_handler.go +++ b/internal/msgtransfer/online_history_msg_handler.go @@ -116,7 +116,7 @@ func NewOnlineHistoryRedisConsumerHandler(ctx context.Context, client discovery. och.redisMessageBatches = b och.historyConsumerGroup = historyConsumerGroup - return &och, err + return &och, nil } func (och *OnlineHistoryRedisConsumerHandler) do(ctx context.Context, channelID int, val *batcher.Msg[sarama.ConsumerMessage]) { ctx = mcontext.WithTriggerIDContext(ctx, val.TriggerID()) From f0d88829d86257443081f11e66363a874ed32a87 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Sun, 29 Dec 2024 14:13:39 +0800 Subject: [PATCH 12/27] fix --- go.mod | 4 ++-- go.sum | 4 ++-- pkg/rpccache/online.go | 47 +++--------------------------------------- pkg/rpcli/group.go | 24 +++++++++++++++++++++ 4 files changed, 31 insertions(+), 48 deletions(-) diff --git a/go.mod b/go.mod index 3cbbff3c9f..387cb73047 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.68 + github.com/openimsdk/protocol v0.0.72-alpha.69 github.com/openimsdk/tools v0.0.50-alpha.62 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 @@ -42,7 +42,6 @@ require ( github.com/spf13/viper v1.18.2 go.etcd.io/etcd/client/v3 v3.5.13 go.uber.org/automaxprocs v1.5.3 - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/sync v0.8.0 k8s.io/api v0.31.2 k8s.io/apimachinery v0.31.2 @@ -187,6 +186,7 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.7.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/image v0.15.0 // indirect golang.org/x/net v0.29.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect diff --git a/go.sum b/go.sum index 846ef721de..d207325e92 100644 --- a/go.sum +++ b/go.sum @@ -347,8 +347,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrkXcDACCqtyM= github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.68 h1:Ekn6S9Ftt12Xs/p9kJ39RDr2gSwIczz+MmSHQE4lAek= -github.com/openimsdk/protocol v0.0.72-alpha.68/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= +github.com/openimsdk/protocol v0.0.72-alpha.69 h1:b22oY2XTdBR/BePqA73KsrM3GDF3Vk8YcBEXZU4ArJc= +github.com/openimsdk/protocol v0.0.72-alpha.69/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= github.com/openimsdk/tools v0.0.50-alpha.62 h1:e/m1XL7+EXbkOoxr/En/612WcOPKOUHPBj0++gG6MuQ= github.com/openimsdk/tools v0.0.50-alpha.62/go.mod h1:JowL2jYr8tu4vcQe+5hJh4v3BtSx1T0CIS3pgU/Mw+U= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index 8f53234775..32b2f28894 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -206,16 +206,6 @@ func (o *OnlineCache) GetUserOnlinePlatform(ctx context.Context, userID string) return platformIDs, nil } -// func (o *OnlineCache) GetUserOnlinePlatformBatch(ctx context.Context, userIDs []string) (map[string]int32, error) { -// platformIDs, err := o.getUserOnlinePlatform(ctx, userIDs) -// if err != nil { -// return nil, err -// } -// tmp := make([]int32, len(platformIDs)) -// copy(tmp, platformIDs) -// return platformIDs, nil -// } - func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, error) { platformIDs, err := o.getUserOnlinePlatform(ctx, userID) if err != nil { @@ -225,6 +215,9 @@ func (o *OnlineCache) GetUserOnline(ctx context.Context, userID string) (bool, e } func (o *OnlineCache) getUserOnlinePlatformBatch(ctx context.Context, userIDs []string) (map[string][]int32, error) { + if len(userIDs) == 0 { + return nil, nil + } platformIDsMap, err := o.lruCache.GetBatch(userIDs, func(missingUsers []string) (map[string][]int32, error) { platformIDsMap := make(map[string][]int32) usersStatus, err := o.client.GetUsersOnlinePlatform(ctx, missingUsers) @@ -281,40 +274,6 @@ func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]s return userIDs, offlineUserIDs, nil } -//func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]string, error) { -// onlineUserIDs := make([]string, 0, len(userIDs)) -// for _, userID := range userIDs { -// online, err := o.GetUserOnline(ctx, userID) -// if err != nil { -// return nil, err -// } -// if online { -// onlineUserIDs = append(onlineUserIDs, userID) -// } -// } -// log.ZDebug(ctx, "OnlineCache GetUsersOnline", "userIDs", userIDs, "onlineUserIDs", onlineUserIDs) -// return onlineUserIDs, nil -//} -// -//func (o *OnlineCache) GetGroupOnline(ctx context.Context, groupID string) ([]string, error) { -// userIDs, err := o.group.GetGroupMemberIDs(ctx, groupID) -// if err != nil { -// return nil, err -// } -// var onlineUserIDs []string -// for _, userID := range userIDs { -// online, err := o.GetUserOnline(ctx, userID) -// if err != nil { -// return nil, err -// } -// if online { -// onlineUserIDs = append(onlineUserIDs, userID) -// } -// } -// log.ZDebug(ctx, "OnlineCache GetGroupOnline", "groupID", groupID, "onlineUserIDs", onlineUserIDs, "allUserID", userIDs) -// return onlineUserIDs, nil -//} - func (o *OnlineCache) setUserOnline(userID string, platformIDs []int32) { switch o.fullUserCache { case true: diff --git a/pkg/rpcli/group.go b/pkg/rpcli/group.go index b51283c5dc..623c5cc12a 100644 --- a/pkg/rpcli/group.go +++ b/pkg/rpcli/group.go @@ -46,3 +46,27 @@ func (x *GroupClient) GetGroupMemberUserIDs(ctx context.Context, groupID string) req := &group.GetGroupMemberUserIDsReq{GroupID: groupID} return extractField(ctx, x.GroupClient.GetGroupMemberUserIDs, req, (*group.GetGroupMemberUserIDsResp).GetUserIDs) } + +func (x *GroupClient) GetGroupMembersInfo(ctx context.Context, groupID string, userIDs []string) ([]*sdkws.GroupMemberFullInfo, error) { + if len(userIDs) == 0 { + return nil, nil + } + req := &group.GetGroupMembersInfoReq{GroupID: groupID, UserIDs: userIDs} + return extractField(ctx, x.GroupClient.GetGroupMembersInfo, req, (*group.GetGroupMembersInfoResp).GetMembers) +} + +func (x *GroupClient) GetGroupMemberInfo(ctx context.Context, groupID string, userID string) (*sdkws.GroupMemberFullInfo, error) { + return firstValue(x.GetGroupMembersInfo(ctx, groupID, []string{userID})) +} + +func (x *GroupClient) GetGroupMemberMapInfo(ctx context.Context, groupID string, userIDs []string) (map[string]*sdkws.GroupMemberFullInfo, error) { + members, err := x.GetGroupMembersInfo(ctx, groupID, userIDs) + if err != nil { + return nil, err + } + memberMap := make(map[string]*sdkws.GroupMemberFullInfo) + for _, member := range members { + memberMap[member.UserID] = member + } + return memberMap, nil +} From d091d5f4bfda89a4a4693d07401eef7ced972e53 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Mon, 30 Dec 2024 15:53:07 +0800 Subject: [PATCH 13/27] optimize log output --- go.mod | 2 +- go.sum | 4 ++-- internal/msggateway/hub_server.go | 9 --------- internal/push/offlinepush/dummy/push.go | 7 +++++-- 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 387cb73047..efdac43f83 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/openimsdk/protocol v0.0.72-alpha.69 - github.com/openimsdk/tools v0.0.50-alpha.62 + github.com/openimsdk/tools v0.0.50-alpha.63 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index d207325e92..f87e038d00 100644 --- a/go.sum +++ b/go.sum @@ -349,8 +349,8 @@ github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrk github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= github.com/openimsdk/protocol v0.0.72-alpha.69 h1:b22oY2XTdBR/BePqA73KsrM3GDF3Vk8YcBEXZU4ArJc= github.com/openimsdk/protocol v0.0.72-alpha.69/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= -github.com/openimsdk/tools v0.0.50-alpha.62 h1:e/m1XL7+EXbkOoxr/En/612WcOPKOUHPBj0++gG6MuQ= -github.com/openimsdk/tools v0.0.50-alpha.62/go.mod h1:JowL2jYr8tu4vcQe+5hJh4v3BtSx1T0CIS3pgU/Mw+U= +github.com/openimsdk/tools v0.0.50-alpha.63 h1:dPoVvg4KWqYX/xtK3j96TwX2A/4jwT5S5XIHvSM9hTY= +github.com/openimsdk/tools v0.0.50-alpha.63/go.mod h1:B+oqV0zdewN7OiEHYJm+hW+8/Te7B8tHHgD8rK5ZLZk= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/internal/msggateway/hub_server.go b/internal/msggateway/hub_server.go index 52afe495b6..490711cfeb 100644 --- a/internal/msggateway/hub_server.go +++ b/internal/msggateway/hub_server.go @@ -96,10 +96,6 @@ func NewServer(longConnServer LongConnServer, conf *Config, ready func(srv *Serv return s } -func (s *Server) OnlinePushMsg(context context.Context, req *msggateway.OnlinePushMsgReq) (*msggateway.OnlinePushMsgResp, error) { - panic("implement me") -} - func (s *Server) GetUsersOnlineStatus(ctx context.Context, req *msggateway.GetUsersOnlineStatusReq) (*msggateway.GetUsersOnlineStatusResp, error) { if !authverify.IsAppManagerUid(ctx, s.config.Share.IMAdminUserID) { return nil, errs.ErrNoPermission.WrapMsg("only app manager") @@ -133,11 +129,6 @@ func (s *Server) GetUsersOnlineStatus(ctx context.Context, req *msggateway.GetUs return &resp, nil } -func (s *Server) OnlineBatchPushOneMsg(ctx context.Context, req *msggateway.OnlineBatchPushOneMsgReq) (*msggateway.OnlineBatchPushOneMsgResp, error) { - // todo implement - return nil, nil -} - func (s *Server) pushToUser(ctx context.Context, userID string, msgData *sdkws.MsgData) *msggateway.SingleMsgToUserResults { clients, ok := s.LongConnServer.GetUserAllCons(userID) if !ok { diff --git a/internal/push/offlinepush/dummy/push.go b/internal/push/offlinepush/dummy/push.go index 0bccaf4a40..1babbc7988 100644 --- a/internal/push/offlinepush/dummy/push.go +++ b/internal/push/offlinepush/dummy/push.go @@ -18,6 +18,7 @@ import ( "context" "github.com/openimsdk/open-im-server/v3/internal/push/offlinepush/options" "github.com/openimsdk/tools/log" + "sync/atomic" ) func NewClient() *Dummy { @@ -25,10 +26,12 @@ func NewClient() *Dummy { } type Dummy struct { + v atomic.Bool } func (d *Dummy) Push(ctx context.Context, userIDs []string, title, content string, opts *options.Opts) error { - log.ZDebug(ctx, "dummy push") - log.ZWarn(ctx, "Dummy push", nil, "ps", "The offline push is not configured. To configure it, please go to config/openim-push.yml.") + if d.v.CompareAndSwap(false, true) { + log.ZWarn(ctx, "dummy push", nil, "ps", "the offline push is not configured. to configure it, please go to config/openim-push.yml") + } return nil } From 3059436c6c684ad5bbdee9c12096014ce644bed2 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 31 Dec 2024 18:18:08 +0800 Subject: [PATCH 14/27] feat: support GetLastMessage --- go.mod | 2 +- go.sum | 4 +- internal/msggateway/client.go | 2 + internal/msggateway/constant.go | 1 + internal/msggateway/message_handler.go | 13 ++++++ internal/rpc/msg/sync_msg.go | 8 ++++ pkg/common/storage/controller/msg.go | 25 ++++++++++- pkg/common/storage/database/mgo/msg.go | 62 ++++++++++++++++++++++++++ pkg/common/storage/database/msg.go | 1 + 9 files changed, 114 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index efdac43f83..03a7a4d4dd 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/openimsdk/protocol v0.0.72-alpha.69 + github.com/openimsdk/protocol v0.0.72-alpha.70 github.com/openimsdk/tools v0.0.50-alpha.63 github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 diff --git a/go.sum b/go.sum index f87e038d00..4c297134d5 100644 --- a/go.sum +++ b/go.sum @@ -347,8 +347,8 @@ github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/openimsdk/gomake v0.0.15-alpha.2 h1:5Q8yl8ezy2yx+q8/ucU/t4kJnDfCzNOrkXcDACCqtyM= github.com/openimsdk/gomake v0.0.15-alpha.2/go.mod h1:PndCozNc2IsQIciyn9mvEblYWZwJmAI+06z94EY+csI= -github.com/openimsdk/protocol v0.0.72-alpha.69 h1:b22oY2XTdBR/BePqA73KsrM3GDF3Vk8YcBEXZU4ArJc= -github.com/openimsdk/protocol v0.0.72-alpha.69/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= +github.com/openimsdk/protocol v0.0.72-alpha.70 h1:j7vB81+rTthijRda2b8tlli9oWvPxr4yXHwZ8nPZIBQ= +github.com/openimsdk/protocol v0.0.72-alpha.70/go.mod h1:Iet+piS/jaS+kWWyj6EEr36mk4ISzIRYjoMSVA4dq2M= github.com/openimsdk/tools v0.0.50-alpha.63 h1:dPoVvg4KWqYX/xtK3j96TwX2A/4jwT5S5XIHvSM9hTY= github.com/openimsdk/tools v0.0.50-alpha.63/go.mod h1:B+oqV0zdewN7OiEHYJm+hW+8/Te7B8tHHgD8rK5ZLZk= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 1040f2be23..2890764355 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -242,6 +242,8 @@ func (c *Client) handleMessage(message []byte) error { resp, messageErr = c.setAppBackgroundStatus(ctx, binaryReq) case WsSubUserOnlineStatus: resp, messageErr = c.longConnServer.SubUserOnlineStatus(ctx, c, binaryReq) + case WsPullConvLastMessage: + resp, messageErr = c.longConnServer.GetLastMessage(ctx, binaryReq) default: return fmt.Errorf( "ReqIdentifier failed,sendID:%s,msgIncr:%s,reqIdentifier:%d", diff --git a/internal/msggateway/constant.go b/internal/msggateway/constant.go index a825c05196..b77cc44c30 100644 --- a/internal/msggateway/constant.go +++ b/internal/msggateway/constant.go @@ -52,6 +52,7 @@ const ( WsLogoutMsg = 2003 WsSetBackgroundStatus = 2004 WsSubUserOnlineStatus = 2005 + WsPullConvLastMessage = 2006 WSDataError = 3001 ) diff --git a/internal/msggateway/message_handler.go b/internal/msggateway/message_handler.go index 9b59867d61..ca15e1ef63 100644 --- a/internal/msggateway/message_handler.go +++ b/internal/msggateway/message_handler.go @@ -108,6 +108,7 @@ type MessageHandler interface { GetSeqMessage(ctx context.Context, data *Req) ([]byte, error) UserLogout(ctx context.Context, data *Req) ([]byte, error) SetUserDeviceBackground(ctx context.Context, data *Req) ([]byte, bool, error) + GetLastMessage(ctx context.Context, data *Req) ([]byte, error) } var _ MessageHandler = (*GrpcHandler)(nil) @@ -266,3 +267,15 @@ func (g *GrpcHandler) SetUserDeviceBackground(ctx context.Context, data *Req) ([ } return nil, req.IsBackground, nil } + +func (g *GrpcHandler) GetLastMessage(ctx context.Context, data *Req) ([]byte, error) { + var req msg.GetLastMessageReq + if err := proto.Unmarshal(data.Data, &req); err != nil { + return nil, err + } + resp, err := g.msgClient.GetLastMessage(ctx, &req) + if err != nil { + return nil, err + } + return proto.Marshal(resp) +} diff --git a/internal/rpc/msg/sync_msg.go b/internal/rpc/msg/sync_msg.go index 7d4ffa3e62..6cf1c21d34 100644 --- a/internal/rpc/msg/sync_msg.go +++ b/internal/rpc/msg/sync_msg.go @@ -245,3 +245,11 @@ func (m *msgServer) SearchMessage(ctx context.Context, req *msg.SearchMessageReq func (m *msgServer) GetServerTime(ctx context.Context, _ *msg.GetServerTimeReq) (*msg.GetServerTimeResp, error) { return &msg.GetServerTimeResp{ServerTime: timeutil.GetCurrentTimestampByMill()}, nil } + +func (m *msgServer) GetLastMessage(ctx context.Context, req *msg.GetLastMessageReq) (*msg.GetLastMessageResp, error) { + msgs, err := m.MsgDatabase.GetLastMessage(ctx, req.ConversationIDs, req.UserID) + if err != nil { + return nil, err + } + return &msg.GetLastMessageResp{Msgs: msgs}, nil +} diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index d5ad12584c..a93d581eb8 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -97,6 +97,8 @@ type CommonMsgDatabase interface { DeleteDoc(ctx context.Context, docID string) error GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) + + GetLastMessage(ctx context.Context, conversationIDS []string, userID string) (map[string]*sdkws.MsgData, error) } func NewCommonMsgDatabase(msgDocModel database.Msg, msg cache.MsgCache, seqUser cache.SeqUser, seqConversation cache.SeqConversationCache, kafkaConf *config.Kafka) (CommonMsgDatabase, error) { @@ -811,8 +813,29 @@ func (db *commonMsgDatabase) GetMessageBySeqs(ctx context.Context, conversationI if v, ok := seqMsgs[seq]; ok { res = append(res, convert.MsgDB2Pb(v.Msg)) } else { - res = append(res, &sdkws.MsgData{Seq: seq}) + res = append(res, &sdkws.MsgData{Seq: seq, Status: constant.MsgStatusHasDeleted}) + } + } + return res, nil +} + +func (db *commonMsgDatabase) GetLastMessage(ctx context.Context, conversationIDs []string, userID string) (map[string]*sdkws.MsgData, error) { + res := make(map[string]*sdkws.MsgData) + for _, conversationID := range conversationIDs { + if _, ok := res[conversationID]; ok { + continue + } + msg, err := db.msgDocDatabase.GetLastMessage(ctx, conversationID) + if err != nil { + if errs.Unwrap(err) == mongo.ErrNoDocuments { + continue + } + return nil, err } + tmp := []*model.MsgInfoModel{msg} + db.handlerDeleteAndRevoked(ctx, userID, tmp) + db.handlerQuote(ctx, userID, conversationID, tmp) + res[conversationID] = convert.MsgDB2Pb(msg.Msg) } return res, nil } diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index 03ebff6110..c440d44420 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -997,6 +997,68 @@ func (m *MsgMgo) GetLastMessageSeqByTime(ctx context.Context, conversationID str return seq, nil } +func (m *MsgMgo) GetLastMessage(ctx context.Context, conversationID string) (*model.MsgInfoModel, error) { + pipeline := []bson.M{ + { + "$match": bson.M{ + "doc_id": bson.M{ + "$regex": fmt.Sprintf("^%s", conversationID), + }, + }, + }, + { + "$match": bson.M{ + "msgs.msg.status": bson.M{ + "$lt": constant.MsgStatusHasDeleted, + }, + }, + }, + { + "$sort": bson.M{ + "_id": -1, + }, + }, + { + "$limit": 1, + }, + { + "$project": bson.M{ + "_id": 0, + "doc_id": 0, + }, + }, + { + "$unwind": "$msgs", + }, + { + "$match": bson.M{ + "msgs.msg.status": bson.M{ + "$lt": constant.MsgStatusHasDeleted, + }, + }, + }, + { + "$sort": bson.M{ + "msgs.msg.seq": -1, + }, + }, + { + "$limit": 1, + }, + } + type Result struct { + Msgs *model.MsgInfoModel `bson:"msgs"` + } + res, err := mongoutil.Aggregate[*Result](ctx, m.coll, pipeline) + if err != nil { + return nil, err + } + if len(res) == 0 { + return nil, errs.Wrap(mongo.ErrNoDocuments) + } + return res[0].Msgs, nil +} + func (m *MsgMgo) onlyFindDocIndex(ctx context.Context, docID string, indexes []int64) ([]*model.MsgInfoModel, error) { if len(indexes) == 0 { return nil, nil diff --git a/pkg/common/storage/database/msg.go b/pkg/common/storage/database/msg.go index b44e702964..e3c4e8ece4 100644 --- a/pkg/common/storage/database/msg.go +++ b/pkg/common/storage/database/msg.go @@ -39,5 +39,6 @@ type Msg interface { DeleteDoc(ctx context.Context, docID string) error GetRandBeforeMsg(ctx context.Context, ts int64, limit int) ([]*model.MsgDocModel, error) GetLastMessageSeqByTime(ctx context.Context, conversationID string, time int64) (int64, error) + GetLastMessage(ctx context.Context, conversationID string) (*model.MsgInfoModel, error) FindSeqs(ctx context.Context, conversationID string, seqs []int64) ([]*model.MsgInfoModel, error) } From f3d5634e569bebfa9e493a48f2cb7957a55d60c4 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 31 Dec 2024 18:21:53 +0800 Subject: [PATCH 15/27] feat: support GetLastMessage --- internal/msggateway/client.go | 4 ++-- internal/msggateway/constant.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/msggateway/client.go b/internal/msggateway/client.go index 2890764355..bdb62aece6 100644 --- a/internal/msggateway/client.go +++ b/internal/msggateway/client.go @@ -236,14 +236,14 @@ func (c *Client) handleMessage(message []byte) error { resp, messageErr = c.longConnServer.GetSeqMessage(ctx, binaryReq) case WSGetConvMaxReadSeq: resp, messageErr = c.longConnServer.GetConversationsHasReadAndMaxSeq(ctx, binaryReq) + case WsPullConvLastMessage: + resp, messageErr = c.longConnServer.GetLastMessage(ctx, binaryReq) case WsLogoutMsg: resp, messageErr = c.longConnServer.UserLogout(ctx, binaryReq) case WsSetBackgroundStatus: resp, messageErr = c.setAppBackgroundStatus(ctx, binaryReq) case WsSubUserOnlineStatus: resp, messageErr = c.longConnServer.SubUserOnlineStatus(ctx, c, binaryReq) - case WsPullConvLastMessage: - resp, messageErr = c.longConnServer.GetLastMessage(ctx, binaryReq) default: return fmt.Errorf( "ReqIdentifier failed,sendID:%s,msgIncr:%s,reqIdentifier:%d", diff --git a/internal/msggateway/constant.go b/internal/msggateway/constant.go index b77cc44c30..1e7ab3bb74 100644 --- a/internal/msggateway/constant.go +++ b/internal/msggateway/constant.go @@ -47,12 +47,12 @@ const ( WSSendSignalMsg = 1004 WSPullMsg = 1005 WSGetConvMaxReadSeq = 1006 + WsPullConvLastMessage = 1007 WSPushMsg = 2001 WSKickOnlineMsg = 2002 WsLogoutMsg = 2003 WsSetBackgroundStatus = 2004 WsSubUserOnlineStatus = 2005 - WsPullConvLastMessage = 2006 WSDataError = 3001 ) From 819997adeb03e2461963eb59e4e0dffbbe125bb5 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 2 Jan 2025 16:26:48 +0800 Subject: [PATCH 16/27] feat: s3 switch --- pkg/common/storage/database/mgo/object.go | 12 ++ pkg/common/storage/database/object.go | 4 + tools/s3/internal/conversion.go | 202 ++++++++++++++++++++++ tools/s3/main.go | 23 +++ 4 files changed, 241 insertions(+) create mode 100644 tools/s3/internal/conversion.go create mode 100644 tools/s3/main.go diff --git a/pkg/common/storage/database/mgo/object.go b/pkg/common/storage/database/mgo/object.go index 624eb84a0d..ee48c66936 100644 --- a/pkg/common/storage/database/mgo/object.go +++ b/pkg/common/storage/database/mgo/object.go @@ -112,3 +112,15 @@ func (o *S3Mongo) FindExpirationObject(ctx context.Context, engine string, expir func (o *S3Mongo) GetKeyCount(ctx context.Context, engine string, key string) (int64, error) { return mongoutil.Count(ctx, o.coll, bson.M{"engine": engine, "key": key}) } + +func (o *S3Mongo) GetEngineCount(ctx context.Context, engine string) (int64, error) { + return mongoutil.Count(ctx, o.coll, bson.M{"engine": engine}) +} + +func (o *S3Mongo) GetEngineInfo(ctx context.Context, engine string, limit int, skip int) ([]*model.Object, error) { + return mongoutil.Find[*model.Object](ctx, o.coll, bson.M{"engine": engine}, options.Find().SetLimit(int64(limit)).SetSkip(int64(skip))) +} + +func (o *S3Mongo) UpdateEngine(ctx context.Context, oldEngine, oldName string, newEngine string) error { + return mongoutil.UpdateOne(ctx, o.coll, bson.M{"engine": oldEngine, "name": oldName}, bson.M{"$set": bson.M{"engine": newEngine}}, false) +} diff --git a/pkg/common/storage/database/object.go b/pkg/common/storage/database/object.go index 5541a159b4..a0e4ebe2bf 100644 --- a/pkg/common/storage/database/object.go +++ b/pkg/common/storage/database/object.go @@ -27,4 +27,8 @@ type ObjectInfo interface { Delete(ctx context.Context, engine string, name []string) error FindExpirationObject(ctx context.Context, engine string, expiration time.Time, needDelType []string, count int64) ([]*model.Object, error) GetKeyCount(ctx context.Context, engine string, key string) (int64, error) + + GetEngineCount(ctx context.Context, engine string) (int64, error) + GetEngineInfo(ctx context.Context, engine string, limit int, skip int) ([]*model.Object, error) + UpdateEngine(ctx context.Context, oldEngine, oldName string, newEngine string) error } diff --git a/tools/s3/internal/conversion.go b/tools/s3/internal/conversion.go new file mode 100644 index 0000000000..ba2174535e --- /dev/null +++ b/tools/s3/internal/conversion.go @@ -0,0 +1,202 @@ +package internal + +import ( + "context" + "errors" + "fmt" + "github.com/mitchellh/mapstructure" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" + "github.com/openimsdk/tools/db/mongoutil" + "github.com/openimsdk/tools/db/redisutil" + "github.com/openimsdk/tools/s3" + "github.com/openimsdk/tools/s3/aws" + "github.com/openimsdk/tools/s3/cos" + "github.com/openimsdk/tools/s3/kodo" + "github.com/openimsdk/tools/s3/minio" + "github.com/openimsdk/tools/s3/oss" + "github.com/spf13/viper" + "go.mongodb.org/mongo-driver/mongo" + "log" + "net/http" + "path/filepath" + "time" +) + +const defaultTimeout = time.Second * 10 + +func readConf(path string, val any) error { + v := viper.New() + v.SetConfigFile(path) + if err := v.ReadInConfig(); err != nil { + return err + } + fn := func(config *mapstructure.DecoderConfig) { + config.TagName = "mapstructure" + } + return v.Unmarshal(val, fn) +} + +func getS3(path string, name string, thirdConf *config.Third) (s3.Interface, error) { + switch name { + case "minio": + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) + defer cancel() + var minioConf config.Minio + if err := readConf(filepath.Join(path, minioConf.GetConfigFileName()), &minioConf); err != nil { + return nil, err + } + var redisConf config.Redis + if err := readConf(filepath.Join(path, redisConf.GetConfigFileName()), &redisConf); err != nil { + return nil, err + } + rdb, err := redisutil.NewRedisClient(ctx, redisConf.Build()) + if err != nil { + return nil, err + } + return minio.NewMinio(ctx, redis.NewMinioCache(rdb), *minioConf.Build()) + case "cos": + return cos.NewCos(*thirdConf.Object.Cos.Build()) + case "oss": + return oss.NewOSS(*thirdConf.Object.Oss.Build()) + case "kodo": + return kodo.NewKodo(*thirdConf.Object.Kodo.Build()) + case "aws": + return aws.NewAws(*thirdConf.Object.Aws.Build()) + default: + return nil, fmt.Errorf("invalid object enable: %s", name) + } +} + +func getMongo(path string) (database.ObjectInfo, error) { + var mongoConf config.Mongo + if err := readConf(filepath.Join(path, mongoConf.GetConfigFileName()), &mongoConf); err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) + defer cancel() + mgocli, err := mongoutil.NewMongoDB(ctx, mongoConf.Build()) + if err != nil { + return nil, err + } + return mgo.NewS3Mongo(mgocli.GetDB()) +} + +func Main(path string, engine string) error { + var thirdConf config.Third + if err := readConf(filepath.Join(path, thirdConf.GetConfigFileName()), &thirdConf); err != nil { + return err + } + if thirdConf.Object.Enable == engine { + return errors.New("same s3 storage") + } + s3db, err := getMongo(path) + if err != nil { + return err + } + oldS3, err := getS3(path, engine, &thirdConf) + if err != nil { + return err + } + newS3, err := getS3(path, thirdConf.Object.Enable, &thirdConf) + if err != nil { + return err + } + count, err := getEngineCount(s3db, oldS3.Engine()) + if err != nil { + return err + } + log.Printf("engine %s count: %d", oldS3.Engine(), count) + var skip int + for i := 1; i <= count+1; i++ { + log.Printf("start %d/%d", i, count) + start := time.Now() + res, err := doObject(s3db, newS3, oldS3, skip) + if err != nil { + log.Printf("end [%s] %d/%d error %s", time.Since(start), i, count, err) + return err + } + log.Printf("end [%s] %d/%d result %+v", time.Since(start), i, count, *res) + if res.Skip { + skip++ + } + if res.End { + break + } + } + return nil +} + +func getEngineCount(db database.ObjectInfo, name string) (int, error) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) + defer cancel() + count, err := db.GetEngineCount(ctx, name) + if err != nil { + return 0, err + } + return int(count), nil +} + +func doObject(db database.ObjectInfo, newS3, oldS3 s3.Interface, skip int) (*Result, error) { + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeout) + defer cancel() + infos, err := db.GetEngineInfo(ctx, oldS3.Engine(), 1, skip) + if err != nil { + return nil, err + } + if len(infos) == 0 { + return &Result{End: true}, nil + } + obj := infos[0] + if _, err := db.Take(ctx, newS3.Engine(), obj.Name); err == nil { + return &Result{Skip: true}, nil + } else if !errors.Is(err, mongo.ErrNoDocuments) { + return nil, err + } + downloadURL, err := oldS3.AccessURL(ctx, obj.Key, time.Hour, &s3.AccessURLOption{}) + if err != nil { + return nil, err + } + putURL, err := newS3.PresignedPutObject(ctx, obj.Key, time.Hour) + if err != nil { + return nil, err + } + downloadResp, err := http.Get(downloadURL) + if err != nil { + return nil, err + } + defer downloadResp.Body.Close() + switch downloadResp.StatusCode { + case http.StatusNotFound: + return &Result{Skip: true}, nil + case http.StatusOK: + default: + return nil, fmt.Errorf("download object failed %s", downloadResp.Status) + } + log.Printf("file size %d", obj.Size) + request, err := http.NewRequest(http.MethodPut, putURL, downloadResp.Body) + if err != nil { + return nil, err + } + putResp, err := http.DefaultClient.Do(request) + if err != nil { + return nil, err + } + defer putResp.Body.Close() + if putResp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("put object failed %s", putResp.Status) + } + ctx, cancel = context.WithTimeout(context.Background(), defaultTimeout) + defer cancel() + if err := db.UpdateEngine(ctx, obj.Engine, obj.Name, newS3.Engine()); err != nil { + return nil, err + } + return &Result{}, nil +} + +type Result struct { + Skip bool + End bool +} diff --git a/tools/s3/main.go b/tools/s3/main.go new file mode 100644 index 0000000000..1e661c9a7c --- /dev/null +++ b/tools/s3/main.go @@ -0,0 +1,23 @@ +package main + +import ( + "flag" + "fmt" + "github.com/openimsdk/open-im-server/v3/tools/s3/internal" + "os" +) + +func main() { + var ( + name string + config string + ) + flag.StringVar(&name, "name", "", "old previous storage name") + flag.StringVar(&config, "config", "", "config directory") + flag.Parse() + if err := internal.Main(config, name); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + fmt.Fprintln(os.Stdout, "success") +} From 755520aa371cba62ab5c7cb3652e67d0288b6b35 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 2 Jan 2025 17:09:28 +0800 Subject: [PATCH 17/27] feat: s3 switch --- tools/s3/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 tools/s3/README.md diff --git a/tools/s3/README.md b/tools/s3/README.md new file mode 100644 index 0000000000..ac30347d88 --- /dev/null +++ b/tools/s3/README.md @@ -0,0 +1,12 @@ +# After s3 switches the storage engine, convert the data + +- build +```shell +go build -o s3convert main.go +``` + +- start +```shell +./s3convert -config -name +# ./s3convert -config ./../../config -name minio +``` From 7a8346bdd312b1a2b22f4c1b718ff36182da090c Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Tue, 7 Jan 2025 11:21:31 +0800 Subject: [PATCH 18/27] fix: GetUsersOnline --- pkg/rpccache/online.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/rpccache/online.go b/pkg/rpccache/online.go index 32b2f28894..b5308bbe88 100644 --- a/pkg/rpccache/online.go +++ b/pkg/rpccache/online.go @@ -270,8 +270,8 @@ func (o *OnlineCache) GetUsersOnline(ctx context.Context, userIDs []string) ([]s } } - log.ZInfo(ctx, "get users online", "online users length", len(userIDs), "offline users length", len(offlineUserIDs), "cost", time.Since(t)) - return userIDs, offlineUserIDs, nil + log.ZInfo(ctx, "get users online", "online users length", len(onlineUserIDs), "offline users length", len(offlineUserIDs), "cost", time.Since(t)) + return onlineUserIDs, offlineUserIDs, nil } func (o *OnlineCache) setUserOnline(userID string, platformIDs []int32) { From 67586614b8b28e4220a6fb8602ec29646e58fffe Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 9 Jan 2025 16:54:17 +0800 Subject: [PATCH 19/27] feat: SendBusinessNotification supported configuration parameters --- internal/api/msg.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/internal/api/msg.go b/internal/api/msg.go index fc235354c7..2fdbd7483e 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -248,16 +248,21 @@ func (m *MessageApi) SendMessage(c *gin.Context) { func (m *MessageApi) SendBusinessNotification(c *gin.Context) { req := struct { - Key string `json:"key"` - Data string `json:"data"` - SendUserID string `json:"sendUserID" binding:"required"` - RecvUserID string `json:"recvUserID" binding:"required"` + Key string `json:"key"` + Data string `json:"data"` + SendUserID string `json:"sendUserID" binding:"required"` + RecvUserID string `json:"recvUserID" binding:"required"` + SendMsg bool `json:"sendMsg"` + ReliabilityLevel *int `json:"reliabilityLevel"` + UnreadCount bool `json:"unreadCount"` }{} if err := c.BindJSON(&req); err != nil { apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap()) return } - + if req.ReliabilityLevel == nil { + req.ReliabilityLevel = datautil.ToPtr(1) + } if !authverify.IsAppManagerUid(c, m.imAdminUserID) { apiresp.GinError(c, errs.ErrNoPermission.WrapMsg("only app manager can send message")) return @@ -278,9 +283,9 @@ func (m *MessageApi) SendBusinessNotification(c *gin.Context) { CreateTime: timeutil.GetCurrentTimestampByMill(), ClientMsgID: idutil.GetMsgIDByMD5(mcontext.GetOpUserID(c)), Options: config.GetOptionsByNotification(config.NotificationConfig{ - IsSendMsg: false, - ReliabilityLevel: 1, - UnreadCount: false, + IsSendMsg: req.SendMsg, + ReliabilityLevel: *req.ReliabilityLevel, + UnreadCount: req.UnreadCount, }), }, } From 05528afbbb57221f4cbfc29204b9f32702e9e1cc Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 9 Jan 2025 17:04:47 +0800 Subject: [PATCH 20/27] feat: SendBusinessNotification supported configuration parameters --- internal/api/msg.go | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/internal/api/msg.go b/internal/api/msg.go index 2fdbd7483e..2af524a738 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -251,7 +251,8 @@ func (m *MessageApi) SendBusinessNotification(c *gin.Context) { Key string `json:"key"` Data string `json:"data"` SendUserID string `json:"sendUserID" binding:"required"` - RecvUserID string `json:"recvUserID" binding:"required"` + RecvUserID string `json:"recvUserID"` + RecvGroupID string `json:"recvGroupID"` SendMsg bool `json:"sendMsg"` ReliabilityLevel *int `json:"reliabilityLevel"` UnreadCount bool `json:"unreadCount"` @@ -260,6 +261,20 @@ func (m *MessageApi) SendBusinessNotification(c *gin.Context) { apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap()) return } + if req.RecvUserID == "" && req.RecvGroupID == "" { + apiresp.GinError(c, errs.ErrArgs.WrapMsg("recvUserID and recvGroupID cannot be empty at the same time")) + return + } + if req.RecvUserID != "" && req.RecvGroupID != "" { + apiresp.GinError(c, errs.ErrArgs.WrapMsg("recvUserID and recvGroupID cannot be set at the same time")) + return + } + var sessionType int32 + if req.RecvUserID != "" { + sessionType = constant.SingleChatType + } else { + sessionType = constant.ReadGroupChatType + } if req.ReliabilityLevel == nil { req.ReliabilityLevel = datautil.ToPtr(1) } @@ -269,8 +284,9 @@ func (m *MessageApi) SendBusinessNotification(c *gin.Context) { } sendMsgReq := msg.SendMsgReq{ MsgData: &sdkws.MsgData{ - SendID: req.SendUserID, - RecvID: req.RecvUserID, + SendID: req.SendUserID, + RecvID: req.RecvUserID, + GroupID: req.RecvGroupID, Content: []byte(jsonutil.StructToJsonString(&sdkws.NotificationElem{ Detail: jsonutil.StructToJsonString(&struct { Key string `json:"key"` @@ -279,7 +295,7 @@ func (m *MessageApi) SendBusinessNotification(c *gin.Context) { })), MsgFrom: constant.SysMsgType, ContentType: constant.BusinessNotification, - SessionType: constant.SingleChatType, + SessionType: sessionType, CreateTime: timeutil.GetCurrentTimestampByMill(), ClientMsgID: idutil.GetMsgIDByMD5(mcontext.GetOpUserID(c)), Options: config.GetOptionsByNotification(config.NotificationConfig{ From e94ce825d6611fe59346d7f55e81c794275a5fa4 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 9 Jan 2025 17:13:33 +0800 Subject: [PATCH 21/27] feat: SendBusinessNotification supported configuration parameters --- internal/api/msg.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/api/msg.go b/internal/api/msg.go index 2af524a738..b21e792db2 100644 --- a/internal/api/msg.go +++ b/internal/api/msg.go @@ -255,7 +255,6 @@ func (m *MessageApi) SendBusinessNotification(c *gin.Context) { RecvGroupID string `json:"recvGroupID"` SendMsg bool `json:"sendMsg"` ReliabilityLevel *int `json:"reliabilityLevel"` - UnreadCount bool `json:"unreadCount"` }{} if err := c.BindJSON(&req); err != nil { apiresp.GinError(c, errs.ErrArgs.WithDetail(err.Error()).Wrap()) @@ -301,7 +300,7 @@ func (m *MessageApi) SendBusinessNotification(c *gin.Context) { Options: config.GetOptionsByNotification(config.NotificationConfig{ IsSendMsg: req.SendMsg, ReliabilityLevel: *req.ReliabilityLevel, - UnreadCount: req.UnreadCount, + UnreadCount: false, }), }, } From f31d196ae89601f0690391efd843e10f2a1bcbb7 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 10 Jan 2025 10:59:20 +0800 Subject: [PATCH 22/27] feat: seq conversion failed without exiting --- tools/seq/main.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/seq/main.go b/tools/seq/main.go index 16da9f156e..ef2875fe12 100644 --- a/tools/seq/main.go +++ b/tools/seq/main.go @@ -3,8 +3,10 @@ package main import ( "flag" "fmt" - "github.com/openimsdk/open-im-server/v3/tools/seq/internal" + "os" "time" + + "github.com/openimsdk/open-im-server/v3/tools/seq/internal" ) func main() { @@ -17,6 +19,8 @@ func main() { flag.Parse() if err := internal.Main(config, time.Duration(second)*time.Second); err != nil { fmt.Println("seq task", err) + os.Exit(1) + return } fmt.Println("seq task success!") } From 961f43c077b27efefe867721216dcb72c0020590 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Wed, 22 Jan 2025 14:22:30 +0800 Subject: [PATCH 23/27] fix: DeleteDoc crash --- pkg/common/storage/controller/msg.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/common/storage/controller/msg.go b/pkg/common/storage/controller/msg.go index a93d581eb8..b92f9b510a 100644 --- a/pkg/common/storage/controller/msg.go +++ b/pkg/common/storage/controller/msg.go @@ -18,11 +18,12 @@ import ( "context" "encoding/json" "errors" - "github.com/openimsdk/tools/utils/jsonutil" "strconv" "strings" "time" + "github.com/openimsdk/tools/utils/jsonutil" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" @@ -722,13 +723,13 @@ func (db *commonMsgDatabase) DeleteDoc(ctx context.Context, docID string) error if index <= 0 { return errs.ErrInternalServer.WrapMsg("docID is invalid", "docID", docID) } - index, err := strconv.Atoi(docID[index+1:]) + docIndex, err := strconv.Atoi(docID[index+1:]) if err != nil { return errs.WrapMsg(err, "strconv.Atoi", "docID", docID) } conversationID := docID[:index] seqs := make([]int64, db.msgTable.GetSingleGocMsgNum()) - minSeq := db.msgTable.GetMinSeq(index) + minSeq := db.msgTable.GetMinSeq(docIndex) for i := range seqs { seqs[i] = minSeq + int64(i) } From 623335c1452f54b2d0af4fee5d5fce4dc5643449 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 24 Jan 2025 15:40:58 +0800 Subject: [PATCH 24/27] fix: fill send time --- pkg/common/storage/database/mgo/msg.go | 134 ++++++++++++++++++++++++- pkg/common/storage/model/msg.go | 11 +- 2 files changed, 141 insertions(+), 4 deletions(-) diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index c440d44420..9e220072ee 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -1091,22 +1091,150 @@ func (m *MsgMgo) onlyFindDocIndex(ctx context.Context, docID string, indexes []i return msgDocModel[0].Msg, nil } +//func (m *MsgMgo) FindSeqs(ctx context.Context, conversationID string, seqs []int64) ([]*model.MsgInfoModel, error) { +// if len(seqs) == 0 { +// return nil, nil +// } +// result := make([]*model.MsgInfoModel, 0, len(seqs)) +// for docID, seqs := range m.model.GetDocIDSeqsMap(conversationID, seqs) { +// res, err := m.onlyFindDocIndex(ctx, docID, datautil.Slice(seqs, m.model.GetMsgIndex)) +// if err != nil { +// return nil, err +// } +// for i, re := range res { +// if re == nil || re.Msg == nil { +// continue +// } +// result = append(result, res[i]) +// } +// } +// return result, nil +//} + +func (m *MsgMgo) findBeforeDocSendTime(ctx context.Context, docID string, limit int64) (int64, int64, error) { + if limit == 0 { + return 0, 0, nil + } + pipeline := []bson.M{ + { + "$match": bson.M{ + "doc_id": docID, + }, + }, + { + "$project": bson.M{ + "_id": 0, + "doc_id": 0, + //"msgs.msg.send_time": 1, + //"msgs.msg.seq": 1, + }, + }, + { + "$unwind": "$msgs", + }, + { + "$project": bson.M{ + //"_id": 0, + //"doc_id": 0, + "msgs.msg.send_time": 1, + "msgs.msg.seq": 1, + }, + }, + } + if limit > 0 { + pipeline = append(pipeline, bson.M{"$limit": limit}) + } + type Result struct { + Msgs *model.MsgInfoModel `bson:"msgs"` + } + res, err := mongoutil.Aggregate[Result](ctx, m.coll, pipeline) + if err != nil { + return 0, 0, err + } + for i := len(res) - 1; i > 0; i-- { + v := res[i] + if v.Msgs != nil && v.Msgs.Msg != nil && v.Msgs.Msg.SendTime > 0 { + return v.Msgs.Msg.Seq, v.Msgs.Msg.SendTime, nil + } + } + return 0, 0, nil +} + +func (m *MsgMgo) findBeforeSendTime(ctx context.Context, conversationID string, seq int64) (int64, int64, error) { + first := true + for i := m.model.GetDocIndex(seq); i >= 0; i-- { + limit := int64(-1) + if first { + first = false + limit = m.model.GetMsgIndex(seq) + } + docID := m.model.BuildDocIDByIndex(conversationID, i) + msgSeq, msgSendTime, err := m.findBeforeDocSendTime(ctx, docID, limit) + if err != nil { + return 0, 0, err + } + if msgSendTime > 0 { + return msgSeq, msgSendTime, nil + } + } + return 0, 0, nil +} + func (m *MsgMgo) FindSeqs(ctx context.Context, conversationID string, seqs []int64) ([]*model.MsgInfoModel, error) { if len(seqs) == 0 { return nil, nil } + var abnormalSeq []int64 result := make([]*model.MsgInfoModel, 0, len(seqs)) - for docID, seqs := range m.model.GetDocIDSeqsMap(conversationID, seqs) { - res, err := m.onlyFindDocIndex(ctx, docID, datautil.Slice(seqs, m.model.GetMsgIndex)) + for docID, docSeqs := range m.model.GetDocIDSeqsMap(conversationID, seqs) { + res, err := m.onlyFindDocIndex(ctx, docID, datautil.Slice(docSeqs, m.model.GetMsgIndex)) if err != nil { return nil, err } + if len(res) == 0 { + abnormalSeq = append(abnormalSeq, docSeqs...) + continue + } for i, re := range res { - if re == nil || re.Msg == nil { + if re == nil || re.Msg == nil || re.Msg.SendTime == 0 { + abnormalSeq = append(abnormalSeq, docSeqs[i]) continue } result = append(result, res[i]) } } + if len(abnormalSeq) > 0 { + datautil.Sort(abnormalSeq, false) + sendTime := make(map[int64]int64) + var ( + lastSeq int64 + lastSendTime int64 + ) + for _, seq := range abnormalSeq { + if lastSendTime > 0 && lastSeq <= seq { + sendTime[seq] = lastSendTime + continue + } + msgSeq, msgSendTime, err := m.findBeforeSendTime(ctx, conversationID, seq) + if err != nil { + return nil, err + } + if msgSendTime <= 0 { + break + } + sendTime[seq] = msgSendTime + lastSeq = msgSeq + lastSendTime = msgSendTime + } + for _, seq := range abnormalSeq { + result = append(result, &model.MsgInfoModel{ + Msg: &model.MsgDataModel{ + Seq: seq, + Status: constant.MsgStatusHasDeleted, + SendTime: sendTime[seq], + }, + }) + } + } return result, nil } diff --git a/pkg/common/storage/model/msg.go b/pkg/common/storage/model/msg.go index 69113032da..6cf63bfcda 100644 --- a/pkg/common/storage/model/msg.go +++ b/pkg/common/storage/model/msg.go @@ -15,9 +15,10 @@ package model import ( + "strconv" + "github.com/openimsdk/protocol/sdkws" "github.com/openimsdk/tools/errs" - "strconv" ) const ( @@ -108,6 +109,10 @@ func (m *MsgDocModel) IsFull() bool { return m.Msg[len(m.Msg)-1].Msg != nil } +func (m *MsgDocModel) GetDocIndex(seq int64) int64 { + return (seq - 1) / singleGocMsgNum +} + func (m *MsgDocModel) GetDocID(conversationID string, seq int64) string { seqSuffix := (seq - 1) / singleGocMsgNum return m.indexGen(conversationID, seqSuffix) @@ -135,6 +140,10 @@ func (*MsgDocModel) indexGen(conversationID string, seqSuffix int64) string { return conversationID + ":" + strconv.FormatInt(seqSuffix, 10) } +func (*MsgDocModel) BuildDocIDByIndex(conversationID string, index int64) string { + return conversationID + ":" + strconv.FormatInt(index, 10) +} + func (*MsgDocModel) GenExceptionMessageBySeqs(seqs []int64) (exceptionMsg []*sdkws.MsgData) { for _, v := range seqs { msgModel := new(sdkws.MsgData) From 29c7a409468d7b0185b418477aa1723c60b5a4a7 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 24 Jan 2025 15:50:32 +0800 Subject: [PATCH 25/27] fix: fill send time --- pkg/common/storage/database/mgo/msg.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/common/storage/database/mgo/msg.go b/pkg/common/storage/database/mgo/msg.go index 9e220072ee..83fefbfe61 100644 --- a/pkg/common/storage/database/mgo/msg.go +++ b/pkg/common/storage/database/mgo/msg.go @@ -1125,8 +1125,6 @@ func (m *MsgMgo) findBeforeDocSendTime(ctx context.Context, docID string, limit "$project": bson.M{ "_id": 0, "doc_id": 0, - //"msgs.msg.send_time": 1, - //"msgs.msg.seq": 1, }, }, { From 8b752aff6f0eabe0b0335d49b8e0dcd96a5fb917 Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Thu, 6 Feb 2025 10:02:48 +0800 Subject: [PATCH 26/27] fix: crash caused by withdrawing messages from users who have left the group --- internal/rpc/msg/revoke.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/rpc/msg/revoke.go b/internal/rpc/msg/revoke.go index 97de0f48a7..c2fb5833f2 100644 --- a/internal/rpc/msg/revoke.go +++ b/internal/rpc/msg/revoke.go @@ -17,9 +17,10 @@ package msg import ( "context" "encoding/json" - "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" "time" + "github.com/openimsdk/open-im-server/v3/pkg/common/storage/model" + "github.com/openimsdk/open-im-server/v3/pkg/authverify" "github.com/openimsdk/open-im-server/v3/pkg/common/servererrs" "github.com/openimsdk/protocol/constant" @@ -79,8 +80,10 @@ func (m *msgServer) RevokeMsg(ctx context.Context, req *msg.RevokeMsgReq) (*msg. switch members[req.UserID].RoleLevel { case constant.GroupOwner: case constant.GroupAdmin: - if members[msgs[0].SendID].RoleLevel != constant.GroupOrdinaryUsers { - return nil, errs.ErrNoPermission.WrapMsg("no permission") + if sendMember, ok := members[msgs[0].SendID]; ok { + if sendMember.RoleLevel != constant.GroupOrdinaryUsers { + return nil, errs.ErrNoPermission.WrapMsg("no permission") + } } default: return nil, errs.ErrNoPermission.WrapMsg("no permission") From 23976e4c74d3e6f08532f77296b10418498396bb Mon Sep 17 00:00:00 2001 From: withchao <993506633@qq.com> Date: Fri, 7 Feb 2025 14:33:17 +0800 Subject: [PATCH 27/27] fix: user msg timestamp --- internal/rpc/conversation/conversation.go | 5 +++-- pkg/common/storage/database/mgo/conversation.go | 9 ++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/internal/rpc/conversation/conversation.go b/internal/rpc/conversation/conversation.go index 53364ff866..76dd606aaf 100644 --- a/internal/rpc/conversation/conversation.go +++ b/internal/rpc/conversation/conversation.go @@ -16,10 +16,11 @@ package conversation import ( "context" - "github.com/openimsdk/open-im-server/v3/pkg/rpcli" "sort" "time" + "github.com/openimsdk/open-im-server/v3/pkg/rpcli" + "github.com/openimsdk/open-im-server/v3/pkg/common/config" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/cache/redis" "github.com/openimsdk/open-im-server/v3/pkg/common/storage/database/mgo" @@ -773,7 +774,7 @@ func (c *conversationServer) ClearUserConversationMsg(ctx context.Context, req * if conversation.IsMsgDestruct == false || conversation.MsgDestructTime == 0 { continue } - seq, err := c.msgClient.GetLastMessageSeqByTime(ctx, conversation.ConversationID, req.Timestamp-conversation.MsgDestructTime) + seq, err := c.msgClient.GetLastMessageSeqByTime(ctx, conversation.ConversationID, req.Timestamp-(conversation.MsgDestructTime*1000)) if err != nil { return nil, err } diff --git a/pkg/common/storage/database/mgo/conversation.go b/pkg/common/storage/database/mgo/conversation.go index 851ec99c40..5368274506 100644 --- a/pkg/common/storage/database/mgo/conversation.go +++ b/pkg/common/storage/database/mgo/conversation.go @@ -243,7 +243,14 @@ func (c *ConversationMgo) FindRandConversation(ctx context.Context, ts int64, li "$add": []any{ bson.M{ "$toLong": "$latest_msg_destruct_time", - }, "$msg_destruct_time"}, + }, + bson.M{ + "$multiply": []any{ + "$msg_destruct_time", + 1000, // convert to milliseconds + }, + }, + }, }, }, },