feat: halfvec v2

Author: I-Info

packages/service/common/vectorStore/pg/class.ts

@@ -35,10 +35,9 @@ export class PgVectorCtrl {
       await PgClient.query(
         `CREATE INDEX CONCURRENTLY IF NOT EXISTS create_time_index ON ${DatasetVectorTableName} USING btree(createtime);`
       );
-      // TODO: enable halfvector index
-      // await PgClient.query(
-      //   `CREATE INDEX CONCURRENTLY IF NOT EXISTS halfvector_index ON ${DatasetVectorTableName} USING hnsw (halfvector halfvec_ip_ops) WITH (m = 32, ef_construction = 128);`
-      // );
+      await PgClient.query(
+        `CREATE INDEX CONCURRENTLY IF NOT EXISTS halfvector_index ON ${DatasetVectorTableName} USING hnsw (halfvector halfvec_ip_ops) WITH (m = 32, ef_construction = 128);`
+      );
 
       addLog.info('init pg successful');
     } catch (error) {
@@ -49,11 +48,9 @@ export class PgVectorCtrl {
     const { teamId, datasetId, collectionId, vector, retry = 3 } = props;
 
     try {
-      // TODO: remove vector
       const { rowCount, rows } = await PgClient.insert(DatasetVectorTableName, {
         values: [
           [
-            { key: 'vector', value: `[${vector}]` },
             { key: 'halfvector', value: `[${vector}]` },
             { key: 'team_id', value: String(teamId) },
             { key: 'dataset_id', value: String(datasetId) },
@@ -180,33 +177,18 @@ export class PgVectorCtrl {
       // );
       // console.log(explan[2].rows);
 
-      // TODO: use halfvector
-      // const results: any = await PgClient.query(
-      //   `
-      //         BEGIN;
-      //           SET LOCAL hnsw.ef_search = ${global.systemEnv?.pgHNSWEfSearch || 100};
-      //           select id, collection_id, halfvector <#> '[${vector}]' AS score
-      //             from ${DatasetVectorTableName}
-      //             where team_id='${teamId}'
-      //               AND dataset_id IN (${datasetIds.map((id) => `'${String(id)}'`).join(',')})
-      //               ${filterCollectionIdSql}
-      //               ${forbidCollectionSql}
-      //             order by score limit ${limit};
-      //         COMMIT;`
-      // );
-
       const results: any = await PgClient.query(
         `
-        BEGIN;
-          SET LOCAL hnsw.ef_search = ${global.systemEnv?.pgHNSWEfSearch || 100};
-          select id, collection_id, vector <#> '[${vector}]' AS score
-            from ${DatasetVectorTableName}
-            where team_id='${teamId}'
-              AND dataset_id IN (${datasetIds.map((id) => `'${String(id)}'`).join(',')})
-              ${filterCollectionIdSql}
-              ${forbidCollectionSql}
-            order by score limit ${limit};
-        COMMIT;`
+              BEGIN;
+                SET LOCAL hnsw.ef_search = ${global.systemEnv?.pgHNSWEfSearch || 100};
+                select id, collection_id, halfvector <#> '[${vector}]' AS score
+                  from ${DatasetVectorTableName}
+                  where team_id='${teamId}'
+                    AND dataset_id IN (${datasetIds.map((id) => `'${String(id)}'`).join(',')})
+                    ${filterCollectionIdSql}
+                    ${forbidCollectionSql}
+                  order by score limit ${limit};
+              COMMIT;`
       );
 
       const rows = results?.[2]?.rows as PgSearchRawType[];