legacymigrate: implement

Signed-off-by: Sumner Evans <[email protected]>

ROADMAP.md

@@ -42,7 +42,7 @@
   * [x] Message edits
   * [x] Message delete
   * [x] Message reactions
-  * [ ] Message history
+  * [x] Message history
   * [x] Real-time messages
   * [ ] ~~Presence~~ (impossible for now, see https://github.com/mautrix/go/issues/295)
   * [x] Typing notifications
@@ -62,10 +62,10 @@
 * Misc
   * [ ] Multi-user support
   * [ ] Shared group chat portals
-  * [ ] Automatic portal creation
-    * [ ] At startup
-    * [ ] When added to chat
+  * [x] Automatic portal creation
+    * [x] At startup
+    * [x] When added to chat
     * [x] When receiving message
   * [ ] Private chat creation by inviting Matrix puppet of LinkedIn user to new room
   * [ ] Option to use own Matrix account for messages sent from other LinkedIn clients (relay mode)
-  * [ ] Split portal support
+  * [x] Split portal support

cmd/mautrix-linkedin/legacymigrate.go

@@ -17,9 +17,24 @@
 package main
 
 import (
+	_ "embed"
+
 	up "go.mau.fi/util/configupgrade"
 )
 
+const legacyMigrateRenameTables = `
+ALTER TABLE cookie RENAME TO cookie_old;
+ALTER TABLE message RENAME TO message_old;
+ALTER TABLE portal RENAME TO portal_old;
+ALTER TABLE puppet RENAME TO puppet_old;
+ALTER TABLE reaction RENAME TO reaction_old;
+ALTER TABLE "user" RENAME TO user_old;
+ALTER TABLE user_portal RENAME TO user_portal_old;
+`
+
+//go:embed legacymigrate.sql
+var legacyMigrateCopyData string
+
 func migrateLegacyConfig(helper up.Helper) {
 	helper.Set(up.Str, "mautrix.bridge.e2ee", "encryption", "pickle_key")
 }

cmd/mautrix-linkedin/legacymigrate.sql

@@ -0,0 +1,166 @@
+INSERT INTO "user" (bridge_id, mxid, management_room)
+SELECT '', mxid, notice_room FROM user_old;
+
+INSERT INTO user_login (bridge_id, user_mxid, id, remote_name, remote_profile, space_room, metadata)
+SELECT
+    '', -- bridge_id
+    mxid, -- user_mxid
+    li_member_urn, -- id
+    li_member_urn, -- remote_name
+    '{}', -- remote_profile
+    space_mxid, -- space_room
+    '{}' -- metadata
+FROM user_old;
+
+INSERT INTO ghost (
+    bridge_id, id, name, avatar_id, avatar_hash, avatar_mxc,
+    name_set, avatar_set, contact_info_set, is_bot, identifiers, metadata
+)
+SELECT
+    '', -- bridge_id
+    li_member_urn, -- id
+    COALESCE(name, ''), -- name
+    COALESCE(photo_id, ''), -- avatar_id
+    '', -- avatar_hash
+    COALESCE(photo_mxc, ''), -- avatar_mxc
+    name_set,
+    avatar_set,
+    contact_info_set,
+    false, -- is_bot
+    '["linkedin:' || li_member_urn || '"]', -- identifiers
+    '{}' -- metadata
+FROM puppet_old;
+
+INSERT INTO portal (
+    bridge_id, id, receiver, mxid, other_user_id, name, topic, avatar_id, avatar_hash, avatar_mxc,
+    name_set, avatar_set, topic_set, name_is_custom, in_space, room_type, metadata
+)
+SELECT
+    '', -- bridge_id
+    'urn:li:msg_conversation:(urn:li:fsd_profile:' || li_receiver_urn || ',' || li_thread_urn || ')', -- id
+    CASE WHEN li_is_group_chat=0 THEN li_receiver_urn ELSE '' END, -- receiver
+    mxid, -- mxid
+    CASE WHEN NOT li_is_group_chat THEN li_other_user_urn END, -- other_user_id
+    COALESCE(name, ''), -- name
+    COALESCE(topic, ''), -- topic
+    COALESCE(photo_id, ''), -- avatar_id
+    '', -- avatar_hash
+    COALESCE(avatar_url, ''), -- avatar_mxc
+    name_set, -- name_set
+    avatar_set, -- avatar_set
+    topic_set, -- topic_set
+    li_is_group_chat, -- name_is_custom
+    false, -- in_space
+    CASE WHEN li_is_group_chat THEN 'dm' ELSE 'group_dm' END, -- room_type
+    '{}' -- metadata
+FROM portal_old;
+
+INSERT INTO user_portal (bridge_id, user_mxid, login_id, portal_id, portal_receiver, in_space, preferred)
+SELECT
+    '', -- bridge_id
+    user_old.mxid, -- user_mxid
+    user_old.li_member_urn, -- login_id
+    'urn:li:msg_conversation:(urn:li:fsd_profile:' || portal_old.li_receiver_urn || ',' || portal_old.li_thread_urn || ')', -- portal_id
+    CASE WHEN NOT li_is_group_chat THEN li_receiver_urn ELSE '' END, -- portal_receiver
+    false, -- in_space
+    false -- preferred
+FROM portal_old
+JOIN user_old ON user_old.li_member_urn = portal_old.li_receiver_urn;
+
+INSERT INTO message (
+    bridge_id, id, part_id, mxid, room_id, room_receiver, sender_id, sender_mxid, timestamp, edit_count, metadata
+)
+SELECT
+    '', -- bridge_id
+    (
+        'urn:li:msg_message:(urn:li:fsd_profile:' ||
+        li_receiver_urn ||
+        SUBSTR(li_message_urn, INSTR(li_message_urn, ',')) ||
+        ')'
+    ), -- id
+    '', -- part_id
+    mxid,
+    'urn:li:msg_conversation:(urn:li:fsd_profile:' || li_receiver_urn || ',' || li_thread_urn || ')', -- room_id
+    (
+        SELECT CASE WHEN li_is_group_chat=0 THEN li_receiver_urn ELSE '' END
+        FROM portal_old
+        WHERE li_thread_urn=message_old.li_thread_urn
+    ), -- room_receiver
+    li_sender_urn, -- sender_id
+    '', -- sender_mxid
+    timestamp * 1000000, -- timestamp
+    0, -- edit_count
+    '{}' -- metadata
+FROM message_old
+WHERE true
+ON CONFLICT DO NOTHING;
+
+INSERT INTO reaction (
+    bridge_id, message_id, message_part_id, sender_id, emoji_id, room_id, room_receiver, mxid, timestamp, emoji, metadata
+)
+SELECT
+    '', -- bridge_id
+    (
+        'urn:li:msg_message:(urn:li:fsd_profile:' ||
+        li_receiver_urn ||
+        SUBSTR(li_message_urn, INSTR(li_message_urn, ',')) ||
+        ')'
+    ), -- message_id
+    '', -- message_part_id
+    li_sender_urn, -- sender_id
+    reaction, -- emoji_id
+    'urn:li:msg_conversation:(urn:li:fsd_profile:' || li_receiver_urn || ',' || SUBSTR(li_message_urn, 0, INSTR(li_message_urn, ',')) || ')', -- room_id
+    (
+        SELECT CASE WHEN li_is_group_chat=0 THEN li_receiver_urn ELSE '' END
+        FROM portal_old
+        WHERE li_thread_urn=SUBSTR(li_message_urn, 0, INSTR(li_message_urn, ','))
+    ), -- room_receiver
+    mxid,
+    (
+        SELECT (timestamp * 1000000) + 1
+        FROM message_old
+        WHERE message_old.li_message_urn=reaction_old.li_message_urn
+            AND "index"=0
+            AND message_old.li_receiver_urn=reaction_old.li_receiver_urn
+    ), -- timestamp
+    reaction, -- emoji
+    '{}' -- metadata
+FROM reaction_old;
+
+CREATE TABLE IF NOT EXISTS database_owner (
+	key   INTEGER PRIMARY KEY DEFAULT 0,
+	owner TEXT NOT NULL
+);
+INSERT INTO database_owner (key, owner) VALUES (0, "megabridge/mautrix-linkedin");
+
+-- Python -> Go mx_ table migration
+ALTER TABLE mx_room_state DROP COLUMN is_encrypted;
+ALTER TABLE mx_room_state RENAME COLUMN has_full_member_list TO members_fetched;
+UPDATE mx_room_state SET members_fetched=false WHERE members_fetched IS NULL;
+
+-- only: postgres until "end only"
+ALTER TABLE mx_room_state ALTER COLUMN power_levels TYPE jsonb USING power_levels::jsonb;
+ALTER TABLE mx_room_state ALTER COLUMN encryption TYPE jsonb USING encryption::jsonb;
+ALTER TABLE mx_room_state ALTER COLUMN members_fetched SET DEFAULT false;
+ALTER TABLE mx_room_state ALTER COLUMN members_fetched SET NOT NULL;
+-- end only postgres
+
+ALTER TABLE mx_user_profile ADD COLUMN name_skeleton bytea;
+CREATE INDEX mx_user_profile_membership_idx ON mx_user_profile (room_id, membership);
+CREATE INDEX mx_user_profile_name_skeleton_idx ON mx_user_profile (room_id, name_skeleton);
+
+UPDATE mx_user_profile SET displayname='' WHERE displayname IS NULL;
+UPDATE mx_user_profile SET avatar_url='' WHERE avatar_url IS NULL;
+
+CREATE TABLE mx_registrations (
+    user_id TEXT PRIMARY KEY
+);
+
+UPDATE mx_version SET version=7;
+
+DROP TABLE message_old;
+DROP TABLE puppet_old;
+DROP TABLE reaction_old;
+DROP TABLE user_portal_old;
+DROP TABLE user_old;
+DROP TABLE portal_old;

cmd/mautrix-linkedin/main.go

@@ -17,12 +17,16 @@
 package main
 
 import (
+	"context"
 	"net/http"
 
 	"maunium.net/go/mautrix/bridgev2/bridgeconfig"
 	"maunium.net/go/mautrix/bridgev2/matrix/mxmain"
 
+	"go.mau.fi/util/dbutil"
+
 	"go.mau.fi/mautrix-linkedin/pkg/connector"
+	"go.mau.fi/mautrix-linkedin/pkg/linkedingo"
 )
 
 // Information to find out exactly which commit the bridge was built from.
@@ -49,7 +53,74 @@ func main() {
 			m.Matrix.Provisioning.Router.HandleFunc("/v1/api/logout", legacyProvLogout).Methods(http.MethodPost)
 		}
 	}
+	m.PostInit = func() {
+		log := m.Log.With().Str("component", "database_migrator").Logger()
+		ctx := log.WithContext(context.TODO())
+
+		// The old bridge does not have a database_owner table, so use that to
+		// detect if the migration has already happened.
+		exists, err := m.DB.TableExists(ctx, "database_owner")
+		if err != nil {
+			log.Err(err).Msg("Failed to check if database_owner table exists")
+			return
+		} else if exists {
+			log.Debug().Msg("Database owner table exists, assuming database is already migrated")
+			return
+		}
 
+		expectedVersion := 10
+		var dbVersion int
+		err = m.DB.QueryRow(ctx, "SELECT version FROM version").Scan(&dbVersion)
+		if err != nil {
+			log.Fatal().Err(err).Msg("Failed to get database version")
+		} else if dbVersion < expectedVersion {
+			log.Fatal().
+				Int("expected_version", expectedVersion).
+				Int("version", dbVersion).
+				Msg("Unsupported database version. Please upgrade to beeper/linkedin v0.5.4 or higher before upgrading to v0.6.0.")
+			return
+		} else if dbVersion > expectedVersion {
+			log.Fatal().
+				Int("expected_version", expectedVersion).
+				Int("version", dbVersion).
+				Msg("Unsupported database version (higher than expected)")
+			return
+		}
+		log.Info().Msg("Detected legacy database, migrating...")
+		err = m.DB.DoTxn(ctx, nil, func(ctx context.Context) error {
+			if err := m.LegacyMigrateSimple(legacyMigrateRenameTables, legacyMigrateCopyData, 16)(ctx); err != nil {
+				return err
+			}
+			rows, err := m.DB.Query(ctx, "SELECT mxid, name, value FROM cookie_old")
+			if err != nil {
+				return err
+			}
+			cookies := map[string]*linkedingo.StringCookieJar{}
+			for rows.Next() {
+				var mxid, name, value string
+				if err := rows.Scan(&mxid, &name, &value); err != nil {
+					return err
+				}
+				if _, ok := cookies[mxid]; !ok {
+					cookies[mxid] = linkedingo.NewEmptyStringCookieJar()
+				}
+				cookies[mxid].AddCookie(&http.Cookie{Name: name, Value: value})
+			}
+			for mxid, jar := range cookies {
+				metadata := connector.UserLoginMetadata{Cookies: jar}
+				if _, err := m.DB.Exec(ctx, "UPDATE user_login SET metadata = $1 WHERE user_mxid = $2", dbutil.JSON{Data: metadata}, mxid); err != nil {
+					return err
+				}
+			}
+			_, err = m.DB.Exec(ctx, "DROP TABLE cookie_old;")
+			return err
+		})
+		if err != nil {
+			m.LogDBUpgradeErrorAndExit("main", err, "Failed to migrate legacy database")
+		} else {
+			log.Info().Msg("Successfully migrated legacy database")
+		}
+	}
 	m.InitVersion(Tag, Commit, BuildTime)
 	m.Run()
 }

pkg/connector/client.go

@@ -206,7 +206,7 @@ func (l *LinkedInClient) onRealtimeMessage(ctx context.Context, msg linkedingo.M
 				Stringer("entity_urn", msg.EntityURN).
 				Stringer("sender", msg.Sender.EntityURN)
 		},
-		PortalKey:    l.makePortalKey(msg.Conversation.EntityURN),
+		PortalKey:    l.makePortalKey(msg.Conversation),
 		CreatePortal: true,
 		Sender:       l.makeSender(msg.Sender),
 		Timestamp:    msg.DeliveredAt.Time,
@@ -255,7 +255,7 @@ func (l *LinkedInClient) onRealtimeTypingIndicator(decoratedEvent *linkedingo.De
 				Stringer("conversation_urn", typingIndicator.Conversation.EntityURN).
 				Stringer("typing_participant_urn", typingIndicator.TypingParticipant.EntityURN)
 		},
-		PortalKey: l.makePortalKey(typingIndicator.Conversation.EntityURN),
+		PortalKey: l.makePortalKey(typingIndicator.Conversation),
 		Sender:    l.makeSender(typingIndicator.TypingParticipant),
 		Timestamp: decoratedEvent.LeftServerAt.Time,
 	}

pkg/connector/dbmeta.go

@@ -33,5 +33,5 @@ func (lc *LinkedInConnector) GetDBMetaTypes() database.MetaTypes {
 }
 
 type UserLoginMetadata struct {
-	Cookies *linkedingo.Jar `json:"cookies,omitempty"`
+	Cookies *linkedingo.StringCookieJar `json:"cookies,omitempty"`
 }

pkg/connector/ids.go

@@ -7,9 +7,9 @@ import (
 	"go.mau.fi/mautrix-linkedin/pkg/linkedingo"
 )
 
-func (l *LinkedInClient) makePortalKey(entityURN linkedingo.URN) (key networkid.PortalKey) {
-	key.ID = networkid.PortalID(entityURN.String())
-	if l.main.Bridge.Config.SplitPortals {
+func (l *LinkedInClient) makePortalKey(conv linkedingo.Conversation) (key networkid.PortalKey) {
+	key.ID = networkid.PortalID(conv.EntityURN.String())
+	if !conv.GroupChat || l.main.Bridge.Config.SplitPortals {
 		key.Receiver = l.userLogin.ID
 	}
 	return key

pkg/connector/sync.go

@@ -49,7 +49,7 @@ func (l *LinkedInClient) syncConversations(ctx context.Context) {
 				updatedBefore = conv.LastActivityAt.Time
 			}
 
-			portalKey := l.makePortalKey(conv.EntityURN)
+			portalKey := l.makePortalKey(conv)
 			portal, err := l.main.Bridge.GetPortalByKey(ctx, portalKey)
 			if err != nil {
 				log.Err(err).Msg("Failed to get portal")

pkg/linkedingo/client.go

@@ -25,7 +25,7 @@ import (
 
 type Client struct {
 	http          *http.Client
-	jar           *Jar
+	jar           *StringCookieJar
 	userEntityURN URN
 
 	realtimeSessionID uuid.UUID
@@ -41,7 +41,7 @@ type Client struct {
 	i18nLocale           string
 }
 
-func NewClient(ctx context.Context, userEntityURN URN, jar *Jar, handlers Handlers) *Client {
+func NewClient(ctx context.Context, userEntityURN URN, jar *StringCookieJar, handlers Handlers) *Client {
 	return &Client{
 		userEntityURN:     userEntityURN,
 		jar:               jar,

pkg/linkedingo/request.go

@@ -46,7 +46,7 @@ func (c *Client) newAuthedRequest(method, urlStr string) *authedRequest {
 	ar.header.Add("sec-ch-prefers-color-scheme", "light")
 	ar.header.Add("sec-ch-ua", `"Chromium";v="131", "Not_A Brand";v="24"`)
 	ar.header.Add("sec-ch-ua-mobile", "?0")
-	ar.header.Add("sec-ch-ua-platform", `"Linux"`)
+	ar.header.Add("sec-ch-ua-platform", `"macOS"`)
 
 	return &ar
 }