4.8.9 test fix (#2291)

* perf: read file icon * perf:icon * fix: i18n * perf: hide pro api * perf: upload expired time * perf: upload file frequency limit * perf: upload file ux * perf: input file tip * perf: qa custom chunk size * feat: dataset openapi * fix: auth dataset list * fix: openapi doc * perf: zero temperature change to 0.01 * perf: read file prompt * perf: read file prompt * perf: free plan tip * feat: cron job usage
labring · Aug 8, 2024 · 3ba9c21 · 3ba9c21
1 parent 7b388b2
commit 3ba9c21
Show file tree

Hide file tree

Showing 42 changed files with 779 additions and 770 deletions.
diff --git a/docSite/content/zh-cn/docs/development/openapi/dataset.md b/docSite/content/zh-cn/docs/development/openapi/dataset.md
@@ -531,6 +531,8 @@ data 为集合的 ID。
 {{< tab tabName="请求示例" >}}
 {{< markdownify >}}
 
+使用代码上传时，请注意中文 filename 需要进行 encode 处理，否则容易乱码。
+
 ```bash
 curl --location --request POST 'http://localhost:3000/api/core/dataset/collection/create/localFile' \
 --header 'Authorization: Bearer {{authorization}}' \
@@ -1173,7 +1175,7 @@ curl --location --request PUT 'http://localhost:3000/api/core/dataset/data/updat
 --header 'Authorization: Bearer {{authorization}}' \
 --header 'Content-Type: application/json' \
 --data-raw '{
-    "id":"65abd4b29d1448617cba61db",
+    "dataId":"65abd4b29d1448617cba61db",
     "q":"测试111",
     "a":"sss",
     "indexes":[
@@ -1196,7 +1198,7 @@ curl --location --request PUT 'http://localhost:3000/api/core/dataset/data/updat
 {{< markdownify >}}
 
 {{% alert icon=" " context="success" %}}
-- id: 数据的id
+- dataId: 数据的id
 - q: 主要数据（选填）
 - a: 辅助数据（选填）
 - indexes: 自定义索引（选填），类型参考`为集合批量添加添加数据`。如果创建时候有自定义索引，

diff --git a/docSite/content/zh-cn/docs/development/upgrading/489.md b/docSite/content/zh-cn/docs/development/upgrading/489.md
@@ -38,12 +38,14 @@ curl --location --request POST 'https://{{host}}/api/admin/init/489' \
 6. 商业版新增 - 知识库搜索节点支持标签过滤和创建时间过滤。
 7. 商业版新增 - 转移 App owner 权限。
 8. 新增 - 删除所有对话引导内容。
-9. 优化 - 对话框信息懒加载，减少网络传输。
-10. 优化 - 清除选文件缓存，支持重复选择同一个文件。
-11. 修复 - 知识库上传文件，网络不稳定或文件较多情况下，进度无法到 100%。
-12. 修复 - 删除应用后回到聊天选择最后一次对话的应用为删除的应用时提示无该应用问题。
-13. 修复 - 插件动态变量配置默认值时，无法正常显示默认值。
-14. 修复 - 工具调用温度和最大回复值未生效。
-15. 修复 - 函数调用模式，assistant role 中，GPT 模型必须传入 content 参数。（不影响大部分模型，目前基本都改用用 ToolChoice 模式，FC 模式已弃用）。
-16. 修复 - 知识库文件上传进度更新可能异常。
-17. 修复 - 知识库 rebuilding 时候，页面总是刷新到第一页。
+9. 新增 - QA 拆分支持自定义 chunk 大小，并优化 gpt4o-mini 拆分时，chunk 太大导致生成内容很少的问题。
+10. 优化 - 对话框信息懒加载，减少网络传输。
+11. 优化 - 清除选文件缓存，支持重复选择同一个文件。
+12. 修复 - 知识库上传文件，网络不稳定或文件较多情况下，进度无法到 100%。
+13. 修复 - 删除应用后回到聊天选择最后一次对话的应用为删除的应用时提示无该应用问题。
+14. 修复 - 插件动态变量配置默认值时，无法正常显示默认值。
+15. 修复 - 工具调用温度和最大回复值未生效。
+16. 修复 - 函数调用模式，assistant role 中，GPT 模型必须传入 content 参数。（不影响大部分模型，目前基本都改用用 ToolChoice 模式，FC 模式已弃用）。
+17. 修复 - 知识库文件上传进度更新可能异常。
+18. 修复 - 知识库 rebuilding 时候，页面总是刷新到第一页。
+19. 修复 - 知识库 list openapi 鉴权问题。
diff --git a/packages/global/common/error/utils.ts b/packages/global/common/error/utils.ts
@@ -1,6 +1,6 @@
 import { replaceSensitiveText } from '../string/tools';
 
-export const getErrText = (err: any, def = '') => {
+export const getErrText = (err: any, def = ''): any => {
   const msg: string =
     typeof err === 'string'
       ? err

diff --git a/packages/global/common/file/constants.ts b/packages/global/common/file/constants.ts
@@ -7,10 +7,12 @@ export enum BucketNameEnum {
 }
 export const bucketNameMap = {
   [BucketNameEnum.dataset]: {
-    label: i18nT('file:bucket_file')
+    label: i18nT('file:bucket_file'),
+    previewExpireMinutes: 30 // 30 minutes
   },
   [BucketNameEnum.chat]: {
-    label: i18nT('file:bucket_chat')
+    label: i18nT('file:bucket_chat'),
+    previewExpireMinutes: 7 * 24 * 60 // 7 days
   }
 };
 

diff --git a/packages/global/common/file/type.d.ts b/packages/global/common/file/type.d.ts
@@ -5,5 +5,4 @@ export type FileTokenQuery = {
   teamId: string;
   tmbId: string;
   fileId: string;
-  expiredTime?: number;
 };
diff --git a/packages/global/common/frequenctLimit/type.d.ts b/packages/global/common/frequenctLimit/type.d.ts
@@ -0,0 +1,5 @@
+export type AuthFrequencyLimitProps = {
+  eventId: string;
+  maxAmount: number;
+  expiredTime: Date;
+};
diff --git a/packages/global/core/ai/prompt/agent.ts b/packages/global/core/ai/prompt/agent.ts
@@ -1,9 +1,9 @@
 export const Prompt_AgentQA = {
   description: `<Context></Context> 标记中是一段文本，学习和分析它，并整理学习成果：
 - 提出问题并给出每个问题的答案。
-- 答案需详细完整，尽可能保留原文描述。
+- 答案需详细完整，尽可能保留原文描述，可以适当扩展答案描述。
 - 答案可以包含普通文字、链接、代码、表格、公示、媒体链接等 Markdown 元素。
-- 最多提出 30 个问题。
+- 最多提出 50 个问题。
 `,
   fixedText: `请按以下格式整理学习成果:
 <Context>

diff --git a/packages/global/core/workflow/template/system/readFiles/index.tsx b/packages/global/core/workflow/template/system/readFiles/index.tsx
@@ -19,7 +19,7 @@ export const ReadFilesNodes: FlowNodeTemplateType = {
   flowNodeType: FlowNodeTypeEnum.readFiles,
   sourceHandle: getHandleConfig(true, true, true, true),
   targetHandle: getHandleConfig(true, true, true, true),
-  avatar: 'core/app/simpleMode/file',
+  avatar: 'core/workflow/template/readFiles',
   name: i18nT('app:workflow.read_files'),
   intro: i18nT('app:workflow.read_files_tip'),
   showStatus: true,

diff --git a/packages/global/support/wallet/usage/constants.ts b/packages/global/support/wallet/usage/constants.ts
@@ -1,8 +1,11 @@
+import { i18nT } from '../../../../web/i18n/utils';
+
 export enum UsageSourceEnum {
   fastgpt = 'fastgpt',
   api = 'api',
   shareLink = 'shareLink',
-  training = 'training'
+  training = 'training',
+  cronJob = 'cronJob'
 }
 
 export const UsageSourceMap = {
@@ -17,5 +20,8 @@ export const UsageSourceMap = {
   },
   [UsageSourceEnum.training]: {
     label: 'dataset.Training Name'
+  },
+  [UsageSourceEnum.cronJob]: {
+    label: i18nT('common:cron_job_run_app')
   }
 };
diff --git a/packages/service/common/file/gridfs/utils.ts b/packages/service/common/file/gridfs/utils.ts
@@ -45,7 +45,7 @@ export const stream2Encoding = async (stream: NodeJS.ReadableStream) => {
   })();
 
   const enc = detectFileEncoding(buffer);
-  console.log('Get encoding time', Date.now() - start, enc);
+
   return {
     encoding: enc,
     stream: copyStream

diff --git a/packages/service/common/file/multer.ts b/packages/service/common/file/multer.ts
@@ -3,7 +3,6 @@ import multer from 'multer';
 import path from 'path';
 import { BucketNameEnum, bucketNameMap } from '@fastgpt/global/common/file/constants';
 import { getNanoid } from '@fastgpt/global/common/string/tools';
-import { tmpFileDirPath } from './constants';
 
 type FileType = {
   fieldname: string;

diff --git a/packages/service/core/workflow/dispatch/agent/extract.ts b/packages/service/core/workflow/dispatch/agent/extract.ts
@@ -228,7 +228,7 @@ const toolChoice = async (props: ActionProps) => {
 
   const response = await ai.chat.completions.create({
     model: extractModel.model,
-    temperature: 0,
+    temperature: 0.01,
     messages: filterMessages,
     tools,
     tool_choice: { type: 'function', function: { name: agentFunName } }
@@ -273,7 +273,7 @@ const functionCall = async (props: ActionProps) => {
 
   const response = await ai.chat.completions.create({
     model: extractModel.model,
-    temperature: 0,
+    temperature: 0.01,
     messages: filterMessages,
     function_call: {
       name: agentFunName

diff --git a/packages/service/core/workflow/dispatch/tools/readFiles.ts b/packages/service/core/workflow/dispatch/tools/readFiles.ts
@@ -49,7 +49,7 @@ export const dispatchReadFiles = async (props: Props): Promise<Response> => {
     chatConfig,
     params: { fileUrlList = [] }
   } = props;
-  const maxFiles = chatConfig?.fileSelectConfig?.maxFiles || 0;
+  const maxFiles = chatConfig?.fileSelectConfig?.maxFiles || 20;
 
   // Get files from histories
   const filesFromHistories = histories
@@ -70,29 +70,35 @@ export const dispatchReadFiles = async (props: Props): Promise<Response> => {
     })
     .flat();
 
-  const parseUrlList = [...fileUrlList, ...filesFromHistories].slice(0, maxFiles);
-
-  const readFilesResult = await Promise.all(
-    parseUrlList
-      .map(async (url) => {
-        // System file
-        if (url.startsWith('/') || (requestOrigin && url.startsWith(requestOrigin))) {
-          // Parse url, get filename query. Keep only documents that can be parsed
-          const parseUrl = new URL(url);
-          const filenameQuery = parseUrl.searchParams.get('filename');
-          if (filenameQuery) {
-            const extensionQuery = filenameQuery.split('.').pop()?.toLowerCase() || '';
-            if (!documentFileType.includes(extensionQuery)) {
-              return;
-            }
+  // Concat fileUrlList and filesFromHistories; remove not supported files
+  const parseUrlList = [...fileUrlList, ...filesFromHistories]
+    .map((url) => {
+      // System file
+      if (url.startsWith('/') || (requestOrigin && url.startsWith(requestOrigin))) {
+        // Parse url, get filename query. Keep only documents that can be parsed
+        const parseUrl = new URL(url);
+        const filenameQuery = parseUrl.searchParams.get('filename');
+        if (filenameQuery) {
+          const extensionQuery = filenameQuery.split('.').pop()?.toLowerCase() || '';
+          if (!documentFileType.includes(extensionQuery)) {
+            return '';
           }
+        }
 
-          //  Remove the origin(Make intranet requests directly)
-          if (requestOrigin && url.startsWith(requestOrigin)) {
-            url = url.replace(requestOrigin, '');
-          }
+        //  Remove the origin(Make intranet requests directly)
+        if (requestOrigin && url.startsWith(requestOrigin)) {
+          url = url.replace(requestOrigin, '');
         }
+      }
 
+      return url;
+    })
+    .filter(Boolean)
+    .slice(0, maxFiles);
+  console.log(parseUrlList);
+  const readFilesResult = await Promise.all(
+    parseUrlList
+      .map(async (url) => {
         // Get from buffer
         const fileBuffer = await MongoRawTextBuffer.findOne({ sourceId: url }, undefined, {
           ...readFromSecondary

diff --git a/packages/service/core/workflow/dispatchV1/agent/classifyQuestion.ts b/packages/service/core/workflow/dispatchV1/agent/classifyQuestion.ts
@@ -188,7 +188,7 @@ const toolChoice = async (props: ActionProps) => {
 
   const response = await ai.chat.completions.create({
     model: cqModel.model,
-    temperature: 0,
+    temperature: 0.01,
     messages: filterMessages,
     tools,
     tool_choice: { type: 'function', function: { name: agentFunName } }
@@ -235,7 +235,7 @@ const functionCall = async (props: ActionProps) => {
 
   const response = await ai.chat.completions.create({
     model: cqModel.model,
-    temperature: 0,
+    temperature: 0.01,
     messages: filterMessages,
     function_call: {
       name: agentFunName

diff --git a/packages/service/support/permission/controller.ts b/packages/service/support/permission/controller.ts
@@ -10,6 +10,8 @@ import { MongoResourcePermission } from './schema';
 import { ClientSession } from 'mongoose';
 import { ParentIdType } from '@fastgpt/global/common/parentFolder/type';
 import { ResourcePermissionType } from '@fastgpt/global/support/permission/type';
+import { bucketNameMap } from '@fastgpt/global/common/file/constants';
+import { addMinutes } from 'date-fns';
 
 export const getResourcePermission = async ({
   resourceType,
@@ -250,15 +252,15 @@ export const clearCookie = (res: NextApiResponse) => {
 };
 
 /* file permission */
-export const createFileToken = ({
-  expiredTime = Math.floor(Date.now() / 1000) + 60 * 30,
-  ...data
-}: FileTokenQuery) => {
+export const createFileToken = (data: FileTokenQuery) => {
   if (!process.env.FILE_TOKEN_KEY) {
     return Promise.reject('System unset FILE_TOKEN_KEY');
   }
 
-  const key = process.env.FILE_TOKEN_KEY as string;
+  const expireMinutes = bucketNameMap[data.bucketName].previewExpireMinutes;
+  const expiredTime = Math.floor(addMinutes(new Date(), expireMinutes).getTime() / 1000);
+
+  const key = (process.env.FILE_TOKEN_KEY as string) ?? 'filetoken';
   const token = jwt.sign(
     {
       ...data,
@@ -274,7 +276,7 @@ export const authFileToken = (token?: string) =>
     if (!token) {
       return reject(ERROR_ENUM.unAuthFile);
     }
-    const key = process.env.FILE_TOKEN_KEY as string;
+    const key = (process.env.FILE_TOKEN_KEY as string) ?? 'filetoken';
 
     jwt.verify(token, key, function (err, decoded: any) {
       if (err || !decoded.bucketName || !decoded?.teamId || !decoded?.tmbId || !decoded?.fileId) {

diff --git a/packages/web/components/common/Avatar/index.tsx b/packages/web/components/common/Avatar/index.tsx
@@ -10,7 +10,7 @@ const Avatar = ({ w = '30px', src, ...props }: ImageProps) => {
   const isIcon = !!iconPaths[src as any];
 
   return isIcon ? (
-    <Box {...props}>
+    <Box display={'inline-flex'} {...props}>
       <MyIcon name={src as any} w={w} borderRadius={props.borderRadius} />
     </Box>
   ) : (

diff --git a/packages/web/components/common/Icon/constants.ts b/packages/web/components/common/Icon/constants.ts
@@ -199,6 +199,7 @@ export const iconPaths = {
     import('./icons/core/workflow/template/queryExtension.svg'),
   'core/workflow/template/questionClassify': () =>
     import('./icons/core/workflow/template/questionClassify.svg'),
+  'core/workflow/template/readFiles': () => import('./icons/core/workflow/template/readFiles.svg'),
   'core/workflow/template/reply': () => import('./icons/core/workflow/template/reply.svg'),
   'core/workflow/template/runApp': () => import('./icons/core/workflow/template/runApp.svg'),
   'core/workflow/template/stopTool': () => import('./icons/core/workflow/template/stopTool.svg'),

diff --git a/packages/web/components/common/Icon/icons/core/app/simpleMode/tts.svg b/packages/web/components/common/Icon/icons/core/app/simpleMode/tts.svg
diff --git a/packages/web/components/common/Icon/icons/core/workflow/template/readFiles.svg b/packages/web/components/common/Icon/icons/core/workflow/template/readFiles.svg
diff --git a/packages/web/i18n/en/app.json b/packages/web/i18n/en/app.json
@@ -103,10 +103,12 @@
     "Simple bot": "Simple bot",
     "Workflow bot": "Workflow"
   },
-  "upload_file_max_amount": "Maximum number of files to be uploaded in a single round",
+  "upload_file_max_amount": "Max files",
+  "upload_file_max_amount_tip": "1. The maximum number of files to be uploaded at a time.\n2. The maximum number of files that can be remembered in the dialog window: Files in the history are automatically retrieved for each round of conversation, and files that are out of range are forgotten.",
   "version": {
     "Revert success": "Revert success"
   },
+  "vision_model_title": "Enable image recognition",
   "week": {
     "Friday": "Friday",
     "Monday": "Monday",
@@ -122,7 +124,7 @@
     "read_files": "Documents parse",
     "read_files_result": "Document parsing results",
     "read_files_result_desc": "The original text of the document consists of the file name and the document content. Multiple files are separated by horizontal lines.",
-    "read_files_tip": "Parse the document link passed in the conversation and return the corresponding document content",
+    "read_files_tip": "Parse all uploaded documents in the conversation and return the corresponding document content",
     "template": {
       "communication": "Communication"
     },