Fix share page whisper auth (labring#1161)

Co-authored-by: heheer <[email protected]>

Author: c121914yu

docSite/content/docs/development/configuration.md

@@ -20,7 +20,7 @@ llm模型全部合并
 ```json
 {
   "feConfigs": {
-    "lafEnv": "https://laf.dev" // laf环境
+    "lafEnv": "https://laf.dev" // laf环境。 https://laf.run （杭州阿里云） ,或者私有化的laf环境。如果使用 Laf openapi 功能，需要最新版的 laf 。
   },
   "systemEnv": {
     "vectorMaxProcess": 15,

docSite/content/docs/development/custom-models/bge-rerank.md

@@ -100,7 +100,7 @@ docker run -d --name reranker -p 6006:6006 -e ACCESS_TOKEN=mytoken --gpus all re
 version: "3"
 services:
   reranker:
-    image: registry.cn-hangzhou.aliyuncs.com/fastgpt/rerank:v0.2
+    image: registry.cn-hangzhou.aliyuncs.com/fastgpt/bge-rerank-base:v0.1
     container_name: reranker
     # GPU运行环境，如果宿主机未安装，将deploy配置隐藏即可
     deploy:

docSite/content/docs/development/upgrading/471.md

@@ -19,10 +19,16 @@ curl --location --request POST 'https://{{host}}/api/admin/clearInvalidData' \
 
 该请求会执行脏数据清理（清理无效的文件、清理无效的图片、清理无效的知识库集合、清理无效的向量）
 
+
+## 修改配置文件
+
+增加了Laf环境配置：[点击查看最新的配置文件](/docs/development/configuration/)
+
+
 ## V4.7.1 更新说明
 
 1. 新增 - 语音输入完整配置。支持选择是否打开语音输入（包括分享页面），支持语音输入后自动发送，支持语音输入后自动语音播放（流式）。
-2. 新增 - Pptx 和 xlsx 文件读取。但所有文件读取都放服务端，会消耗更多的服务器资源，以及无法在上传时预览更多内容。
+2. 新增 - pptx 和 xlsx 文件读取。但所有文件读取都放服务端，会消耗更多的服务器资源，以及无法在上传时预览更多内容。
 3. 新增 - 集成 Laf 云函数，可以读取 Laf 账号中的云函数作为 HTTP 模块。
 4. 新增 - 定时器，清理垃圾数据。（采用小范围清理，会清理最近n个小时的，所以请保证服务持续运行，长时间不允许，可以继续执行 clearInvalidData 的接口进行全量清理。）
 5. 商业版新增 - 后台配置系统通知。

docSite/content/docs/workflow/examples/dalle3.md

@@ -88,7 +88,7 @@ Response:
 [
   {
     "moduleId": "userGuide",
-    "name": "core.module.template.User guide",
+    "name": "core.module.template.App system setting",
     "flowType": "userGuide",
     "position": {
       "x": 454.98510354678695,

docSite/content/docs/workflow/examples/feishu_webhook.md

@@ -27,7 +27,7 @@ weight: 404
 [
   {
     "moduleId": "userGuide",
-    "name": "core.module.template.User guide",
+    "name": "core.module.template.App system setting",
     "intro": "core.app.tip.userGuideTip",
     "avatar": "/imgs/module/userGuide.png",
     "flowType": "userGuide",

docSite/content/docs/workflow/examples/google_search.md

@@ -84,7 +84,7 @@ export default async function (ctx: FunctionContext) {
 [
   {
     "moduleId": "userGuide",
-    "name": "core.module.template.User guide",
+    "name": "core.module.template.App system setting",
     "intro": "core.app.tip.userGuideTip",
     "avatar": "/imgs/module/userGuide.png",
     "flowType": "userGuide",

docSite/content/docs/workflow/modules/laf.md

@@ -27,8 +27,6 @@ Laf 提供了 PAT(访问凭证) 来实现 Laf 平台外的快捷登录，可以
 
 填入 PAT 验证后，选择需要绑定的应用（应用需要是 Running 状态），即可调用该应用下的云函数。
 
-> 如果需要解绑则取消绑定后，点击“更新”即可
-
 ![](/imgs/laf2.webp)
 
 ## 编写云函数

packages/global/common/string/textSplitter.ts

@@ -1,5 +1,5 @@
 import { getErrText } from '../error/utils';
-import { countPromptTokens } from './tiktoken';
+import { replaceRegChars } from './tools';
 
 /**
  * text split into chunks
@@ -31,7 +31,7 @@ export const splitText2Chunks = (props: {
   // The larger maxLen is, the next sentence is less likely to trigger splitting
   const stepReges: { reg: RegExp; maxLen: number }[] = [
     ...customReg.map((text) => ({
-      reg: new RegExp(`(${text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')})`, 'g'),
+      reg: new RegExp(`(${replaceRegChars(text)})`, 'g'),
       maxLen: chunkLen * 1.4
     })),
     { reg: /^(#\s[^\n]+)\n/gm, maxLen: chunkLen * 1.2 },

packages/global/common/string/tools.ts

@@ -51,3 +51,5 @@ export const replaceSensitiveText = (text: string) => {
 export const getNanoid = (size = 12) => {
   return customAlphabet('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890', size)();
 };
+
+export const replaceRegChars = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');

packages/global/core/module/template/system/userGuide.ts

@@ -1,6 +1,5 @@
 import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from '../../node/constant';
 import { FlowNodeTemplateType } from '../../type.d';
-import { userGuideTip } from '../tip';
 import {
   ModuleIOValueTypeEnum,
   ModuleInputKeyEnum,
@@ -12,8 +11,8 @@ export const UserGuideModule: FlowNodeTemplateType = {
   templateType: FlowNodeTemplateTypeEnum.userGuide,
   flowType: FlowNodeTypeEnum.userGuide,
   avatar: '/imgs/module/userGuide.png',
-  name: '全局配置',
-  intro: userGuideTip,
+  name: '系统配置',
+  intro: '可以配置应用的系统参数。',
   inputs: [
     {
       key: ModuleInputKeyEnum.welcomeText,

packages/global/core/module/template/tip.ts

@@ -1,4 +1,3 @@
 export const chatNodeSystemPromptTip = 'core.app.tip.chatNodeSystemPromptTip';
-export const userGuideTip = 'core.app.tip.userGuideTip';
 export const welcomeTextTip = 'core.app.tip.welcomeTextTip';
 export const variableTip = 'core.app.tip.variableTip';

packages/global/support/user/team/type.d.ts

@@ -81,4 +81,5 @@ export type TeamTagItemType = {
 export type LafAccountType = {
   token: string;
   appid: string;
+  pat: string;
 };

packages/global/support/wallet/bill/type.d.ts

@@ -18,6 +18,7 @@ export type BillSchemaType = {
     month?: number;
     datasetSize?: number;
     extraPoints?: number;
+    invoice: boolean;
   };
   username: string;
 };

packages/service/common/file/image/schema.ts

@@ -35,6 +35,7 @@ try {
   ImageSchema.index({ expiredTime: 1 }, { expireAfterSeconds: 60 });
   ImageSchema.index({ type: 1 });
   ImageSchema.index({ createTime: 1 });
+  // delete related img
   ImageSchema.index({ teamId: 1, 'metadata.relatedId': 1 });
 } catch (error) {
   console.log(error);

packages/service/common/vectorStore/controller.d.ts

@@ -1,10 +1,9 @@
-export type DeleteDatasetVectorProps = {
+export type DeleteDatasetVectorProps = (
+  | { id: string }
+  | { datasetIds: string[]; collectionIds?: string[] }
+  | { idList: string[] }
+) & {
   teamId: string;
-
-  id?: string;
-  datasetIds?: string[];
-  collectionIds?: string[];
-  idList?: string[];
 };
 
 export type InsertVectorProps = {

packages/service/common/vectorStore/pg/controller.ts

@@ -26,13 +26,7 @@ export async function initPg() {
       `CREATE INDEX CONCURRENTLY IF NOT EXISTS vector_index ON ${PgDatasetTableName} USING hnsw (vector vector_ip_ops) WITH (m = 32, ef_construction = 64);`
     );
     await PgClient.query(
-      `CREATE INDEX CONCURRENTLY IF NOT EXISTS team_dataset_index ON ${PgDatasetTableName} USING btree(team_id, dataset_id);`
-    );
-    await PgClient.query(
-      ` CREATE INDEX CONCURRENTLY IF NOT EXISTS team_collection_index ON ${PgDatasetTableName} USING btree(team_id, collection_id);`
-    );
-    await PgClient.query(
-      `CREATE INDEX CONCURRENTLY IF NOT EXISTS team_id_index ON ${PgDatasetTableName} USING btree(team_id, id);`
+      `CREATE INDEX CONCURRENTLY IF NOT EXISTS team_dataset_collection_index ON ${PgDatasetTableName} USING btree(team_id, dataset_id, collection_id);`
     );
     await PgClient.query(
       `CREATE INDEX CONCURRENTLY IF NOT EXISTS create_time_index ON ${PgDatasetTableName} USING btree(createtime);`
@@ -83,31 +77,33 @@ export const deleteDatasetDataVector = async (
     retry?: number;
   }
 ): Promise<any> => {
-  const { teamId, id, datasetIds, collectionIds, idList, retry = 2 } = props;
+  const { teamId, retry = 2 } = props;
 
   const teamIdWhere = `team_id='${String(teamId)}' AND`;
 
   const where = await (() => {
-    if (id) return `${teamIdWhere} id=${id}`;
+    if ('id' in props && props.id) return `${teamIdWhere} id=${props.id}`;
 
-    if (datasetIds) {
-      return `${teamIdWhere} dataset_id IN (${datasetIds
+    if ('datasetIds' in props && props.datasetIds) {
+      const datasetIdWhere = `dataset_id IN (${props.datasetIds
         .map((id) => `'${String(id)}'`)
         .join(',')})`;
-    }
 
-    if (collectionIds) {
-      return `${teamIdWhere} collection_id IN (${collectionIds
-        .map((id) => `'${String(id)}'`)
-        .join(',')})`;
+      if ('collectionIds' in props && props.collectionIds) {
+        return `${teamIdWhere} ${datasetIdWhere} AND collection_id IN (${props.collectionIds
+          .map((id) => `'${String(id)}'`)
+          .join(',')})`;
+      }
+
+      return `${teamIdWhere} ${datasetIdWhere}`;
     }
 
-    if (idList) {
-      return `${teamIdWhere} id IN (${idList.map((id) => `'${String(id)}'`).join(',')})`;
+    if ('idList' in props && props.idList) {
+      return `${teamIdWhere} id IN (${props.idList.map((id) => `'${String(id)}'`).join(',')})`;
     }
     return Promise.reject('deleteDatasetData: no where');
   })();
-
+  console.log(where, '===');
   try {
     await PgClient.delete(PgDatasetTableName, {
       where: [where]

packages/service/core/dataset/collection/controller.ts

@@ -118,6 +118,37 @@ export function createDefaultCollection({
   );
 }
 
+/* delete collection related images/files */
+export const delCollectionRelatedSource = async ({
+  collections,
+  session
+}: {
+  collections: (CollectionWithDatasetType | DatasetCollectionSchemaType)[];
+  session: ClientSession;
+}) => {
+  if (collections.length === 0) return;
+
+  const teamId = collections[0].teamId;
+
+  if (!teamId) return Promise.reject('teamId is not exist');
+
+  const fileIdList = collections.map((item) => item?.fileId || '').filter(Boolean);
+  const relatedImageIds = collections
+    .map((item) => item?.metadata?.relatedImgId || '')
+    .filter(Boolean);
+
+  // delete images
+  await delImgByRelatedId({
+    teamId,
+    relateIds: relatedImageIds,
+    session
+  });
+  // delete files
+  await delFileByFileIdList({
+    bucketName: BucketNameEnum.dataset,
+    fileIdList
+  });
+};
 /**
  * delete collection and it related data
  */
@@ -134,26 +165,32 @@ export async function delCollectionAndRelatedSources({
 
   if (!teamId) return Promise.reject('teamId is not exist');
 
+  const datasetIds = Array.from(
+    new Set(
+      collections.map((item) => {
+        if (typeof item.datasetId === 'string') {
+          return String(item.datasetId);
+        }
+        return String(item.datasetId._id);
+      })
+    )
+  );
   const collectionIds = collections.map((item) => String(item._id));
-  const fileIdList = collections.map((item) => item?.fileId || '').filter(Boolean);
-  const relatedImageIds = collections
-    .map((item) => item?.metadata?.relatedImgId || '')
-    .filter(Boolean);
+
+  await delCollectionRelatedSource({ collections, session });
 
   // delete training data
   await MongoDatasetTraining.deleteMany({
     teamId,
+    datasetIds: { $in: datasetIds },
     collectionId: { $in: collectionIds }
   });
-
   // delete dataset.datas
-  await MongoDatasetData.deleteMany({ teamId, collectionId: { $in: collectionIds } }, { session });
-  // delete imgs
-  await delImgByRelatedId({
-    teamId,
-    relateIds: relatedImageIds,
-    session
-  });
+  await MongoDatasetData.deleteMany(
+    { teamId, datasetIds: { $in: datasetIds }, collectionId: { $in: collectionIds } },
+    { session }
+  );
+
   // delete collections
   await MongoDatasetCollection.deleteMany(
     {
@@ -163,9 +200,5 @@ export async function delCollectionAndRelatedSources({
   );
 
   // no session delete: delete files, vector data
-  await deleteDatasetDataVector({ teamId, collectionIds });
-  await delFileByFileIdList({
-    bucketName: BucketNameEnum.dataset,
-    fileIdList
-  });
+  await deleteDatasetDataVector({ teamId, datasetIds, collectionIds });
 }

packages/service/core/dataset/controller.ts

@@ -1,8 +1,11 @@
 import { CollectionWithDatasetType, DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
 import { MongoDatasetCollection } from './collection/schema';
 import { MongoDataset } from './schema';
-import { delCollectionAndRelatedSources } from './collection/controller';
+import { delCollectionRelatedSource } from './collection/controller';
 import { ClientSession } from '../../common/mongo';
+import { MongoDatasetTraining } from './training/schema';
+import { MongoDatasetData } from './data/schema';
+import { deleteDatasetDataVector } from '../../common/vectorStore/controller';
 
 /* ============= dataset ========== */
 /* find all datasetId by top datasetId */
@@ -82,5 +85,26 @@ export async function delDatasetRelevantData({
     '_id teamId fileId metadata'
   ).lean();
 
-  await delCollectionAndRelatedSources({ collections, session });
+  // image and file
+  await delCollectionRelatedSource({ collections, session });
+
+  // delete training data
+  await MongoDatasetTraining.deleteMany({
+    teamId,
+    datasetId: { $in: datasetIds }
+  });
+  // delete dataset.datas
+  await MongoDatasetData.deleteMany({ teamId, datasetId: { $in: datasetIds } }, { session });
+
+  // delete collections
+  await MongoDatasetCollection.deleteMany(
+    {
+      teamId,
+      datasetId: { $in: datasetIds }
+    },
+    { session }
+  );
+
+  // no session delete: delete files, vector data
+  await deleteDatasetDataVector({ teamId, datasetIds });
 }

packages/service/core/dataset/data/controller.ts

@@ -77,17 +77,18 @@ const DatasetDataSchema = new Schema({
 });
 
 try {
-  // list collection and count data; list data
+  // list collection and count data; list data; delete collection(relate data)
   DatasetDataSchema.index(
     { teamId: 1, datasetId: 1, collectionId: 1, chunkIndex: 1, updateTime: -1 },
     { background: true }
   );
-  // same data check
-  DatasetDataSchema.index({ teamId: 1, collectionId: 1, q: 1, a: 1 }, { background: true });
   // full text index
   DatasetDataSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' }, { background: true });
   // Recall vectors after data matching
-  DatasetDataSchema.index({ teamId: 1, datasetId: 1, 'indexes.dataId': 1 }, { background: true });
+  DatasetDataSchema.index(
+    { teamId: 1, datasetId: 1, collectionId: 1, 'indexes.dataId': 1 },
+    { background: true }
+  );
   DatasetDataSchema.index({ updateTime: 1 }, { background: true });
 } catch (error) {
   console.log(error);

packages/service/core/dataset/data/schema.ts

@@ -93,6 +93,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
       {
         teamId,
         datasetId: { $in: datasetIds },
+        collectionId: { $in: results.map((item) => item.collectionId) },
         'indexes.dataId': { $in: results.map((item) => item.id?.trim()) }
       },
       'datasetId collectionId q a chunkIndex indexes'

packages/service/core/dataset/search/controller.ts

@@ -6,7 +6,6 @@ import type {
 } from '@fastgpt/global/core/dataset/api.d';
 import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
 import { simpleText } from '@fastgpt/global/common/string/tools';
-import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
 import { ClientSession } from '../../../common/mongo';
 import { getLLMModel, getVectorModel } from '../../ai/model';
 import { addLog } from '../../../common/system/log';