4.8.11 perf (#2768)

* perf: watch local * perf: dataset list ui * perf: Check workflow invalid edges in saved * remove log * perf: Forbid touch scale * perf: rename dataset process * feat: support child app unstream mode * feat: Dispatch child app will record detail * feat: Save childApp run log * fix: share page init error * perf: chatId reset
labring · Sep 23, 2024 · 3ab9347 · 3ab9347
1 parent 4245ea4
commit 3ab9347
Show file tree

Hide file tree

Showing 38 changed files with 252 additions and 143 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -21,7 +21,7 @@
     "i18n-ally.namespace": true,
     "i18n-ally.pathMatcher": "{locale}/{namespaces}.json",
     "i18n-ally.extract.targetPickingStrategy": "most-similar-by-key",
-    "i18n-ally.translate.engines": ["deepl", "google"],
+    "i18n-ally.translate.engines": ["google"],
     "[typescript]": {
         "editor.defaultFormatter": "esbenp.prettier-vscode"
     }

diff --git a/docSite/content/zh-cn/docs/development/configuration.md b/docSite/content/zh-cn/docs/development/configuration.md
@@ -60,7 +60,7 @@ weight: 708
       "charsPointsPrice": 0,
       "censor": false,
       "vision": true,
-      "datasetProcess": false,
+      "datasetProcess": true,
       "usedInClassify": true,
       "usedInExtractFields": true,
       "usedInToolCall": true,
@@ -83,7 +83,7 @@ weight: 708
       "charsPointsPrice": 0,
       "censor": false,
       "vision": false,
-      "datasetProcess": false,
+      "datasetProcess": true,
       "usedInClassify": true,
       "usedInExtractFields": true,
       "usedInToolCall": true,
@@ -110,7 +110,7 @@ weight: 708
       "charsPointsPrice": 0,
       "censor": false,
       "vision": false,
-      "datasetProcess": false,
+      "datasetProcess": true,
       "usedInClassify": true,
       "usedInExtractFields": true,
       "usedInToolCall": true,

diff --git a/docSite/content/zh-cn/docs/development/one-api.md b/docSite/content/zh-cn/docs/development/one-api.md
@@ -135,7 +135,7 @@ CHAT_API_KEY=sk-xxxxxx
       "charsPointsPrice": 0, 
       "censor": false,
       "vision": false, // 是否支持图片输入
-      "datasetProcess": false, // 是否设置为知识库处理模型
+      "datasetProcess": true, // 是否设置为知识库处理模型
       "usedInClassify": true, // 是否用于问题分类
       "usedInExtractFields": true, // 是否用于字段提取
       "usedInToolCall": true, // 是否用于工具调用

diff --git a/docSite/content/zh-cn/docs/development/upgrading/4811.md b/docSite/content/zh-cn/docs/development/upgrading/4811.md
@@ -27,7 +27,7 @@ weight: 813
     "charsPointsPrice": 0,
     "censor": false,
     "vision": false,
-    "datasetProcess": false,
+    "datasetProcess": true,
     "usedInClassify": true,
     "usedInExtractFields": true,
     "usedInToolCall": true,
@@ -56,7 +56,7 @@ weight: 813
     "charsPointsPrice": 0,
     "censor": false,
     "vision": false,
-    "datasetProcess": false,
+    "datasetProcess": true,
     "usedInClassify": true,
     "usedInExtractFields": true,
     "usedInToolCall": true,
@@ -92,11 +92,14 @@ weight: 813
 6. 新增 - 支持 Openai o1 模型，需增加模型的 `defaultConfig` 配置，覆盖 `temperature`、`max_tokens` 和 `stream`配置，o1 不支持 stream 模式, 详细可重新拉取 `config.json` 配置文件查看。
 7. 新增 - AI 对话节点知识库引用，支持配置 role=system 和 role=user，已配置的过自定义提示词的节点将会保持 user 模式，其余用户将转成 system 模式。
 8. 新增 - 插件支持上传系统文件。
-9. 优化 - 工作流嵌套层级限制 20 层，避免因编排不合理导致的无限死循环。
-10. 优化 - 工作流 handler 性能优化。
-11. 优化 - 工作流快捷键，避免调试测试时也会触发。
-12. 优化 - 流输出，切换 tab 时仍可以继续输出。
-13. 优化 - 完善外部文件知识库相关 API
-14. 修复 - 知识库选择权限问题。
-15. 修复 - 空 chatId 发起对话，首轮携带用户选择时会异常。
-16. 修复 - createDataset 接口，intro 为赋值。
+9. 新增 - 支持工作流嵌套子应用时，可以设置`非流模式`。
+10. 新增 - 调试模式下，子应用调用，支持返回详细运行数据。
+11. 新增 - 保留所有模式下子应用嵌套调用的日志。
+12. 优化 - 工作流嵌套层级限制 20 层，避免因编排不合理导致的无限死循环。
+13. 优化 - 工作流 handler 性能优化。
+14. 优化 - 工作流快捷键，避免调试测试时也会触发。
+15. 优化 - 流输出，切换 tab 时仍可以继续输出。
+16. 优化 - 完善外部文件知识库相关 API
+17. 修复 - 知识库选择权限问题。
+18. 修复 - 空 chatId 发起对话，首轮携带用户选择时会异常。
+19. 修复 - createDataset 接口，intro 为赋值。
diff --git a/files/helm/fastgpt/templates/configmap-config.yaml b/files/helm/fastgpt/templates/configmap-config.yaml
@@ -19,7 +19,7 @@ data:
           "charsPointsPrice": 0,
           "censor": false,
           "vision": false,
-          "datasetProcess": false,
+          "datasetProcess": true,
           "usedInClassify": true,
           "usedInExtractFields": true,
           "usedInToolCall": true,
@@ -63,7 +63,7 @@ data:
           "charsPointsPrice": 0,
           "censor": false,
           "vision": false,
-          "datasetProcess": false,
+          "datasetProcess": true,
           "usedInClassify": true,
           "usedInExtractFields": true,
           "usedInToolCall": true,
@@ -85,7 +85,7 @@ data:
           "charsPointsPrice": 0,
           "censor": false,
           "vision": true,
-          "datasetProcess": false,
+          "datasetProcess": true,
           "usedInClassify": false,
           "usedInExtractFields": false,
           "usedInToolCall": false,

diff --git a/packages/global/core/workflow/constants.ts b/packages/global/core/workflow/constants.ts
@@ -70,6 +70,7 @@ export enum NodeInputKeyEnum {
   anyInput = 'system_anyInput',
   textareaInput = 'system_textareaInput',
   addInputParam = 'system_addInputParam',
+  forbidStream = 'system_forbid_stream',
 
   // history
   historyMaxAmount = 'maxContext',

diff --git a/packages/global/core/workflow/template/input.ts b/packages/global/core/workflow/template/input.ts
@@ -105,3 +105,12 @@ export const Input_Template_Node_Height: FlowNodeInputItemType = {
   label: '',
   value: 900
 };
+
+export const Input_Template_Stream_MODE: FlowNodeInputItemType = {
+  key: NodeInputKeyEnum.forbidStream,
+  renderTypeList: [FlowNodeInputTypeEnum.switch],
+  valueType: WorkflowIOValueTypeEnum.boolean,
+  label: i18nT('workflow:template.forbid_stream'),
+  description: i18nT('workflow:template.forbid_stream_desc'),
+  value: false
+};
diff --git a/packages/global/core/workflow/utils.ts b/packages/global/core/workflow/utils.ts
@@ -31,7 +31,11 @@ import {
 import { IfElseResultEnum } from './template/system/ifElse/constant';
 import { RuntimeNodeItemType } from './runtime/type';
 import { getReferenceVariableValue } from './runtime/utils';
-import { Input_Template_History, Input_Template_UserChatInput } from './template/input';
+import {
+  Input_Template_History,
+  Input_Template_Stream_MODE,
+  Input_Template_UserChatInput
+} from './template/input';
 import { i18nT } from '../../../web/i18n/utils';
 import { RuntimeUserPromptType, UserChatItemType } from '../../core/chat/type';
 import { getNanoid } from '../../common/string/tools';
@@ -179,17 +183,21 @@ export const pluginData2FlowNodeIO = ({
   const pluginOutput = nodes.find((node) => node.flowNodeType === FlowNodeTypeEnum.pluginOutput);
 
   return {
-    inputs:
-      pluginInput?.inputs.map((item) => ({
-        ...item,
-        ...getModuleInputUiField(item),
-        value: getOrInitModuleInputValue(item),
-        canEdit: false,
-        renderTypeList:
-          item.renderTypeList[0] === FlowNodeInputTypeEnum.customVariable
-            ? [FlowNodeInputTypeEnum.reference, FlowNodeInputTypeEnum.input]
-            : item.renderTypeList
-      })) || [],
+    inputs: pluginInput
+      ? [
+          Input_Template_Stream_MODE,
+          ...pluginInput?.inputs.map((item) => ({
+            ...item,
+            ...getModuleInputUiField(item),
+            value: getOrInitModuleInputValue(item),
+            canEdit: false,
+            renderTypeList:
+              item.renderTypeList[0] === FlowNodeInputTypeEnum.customVariable
+                ? [FlowNodeInputTypeEnum.reference, FlowNodeInputTypeEnum.input]
+                : item.renderTypeList
+          }))
+        ]
+      : [],
     outputs: pluginOutput
       ? [
           ...pluginOutput.inputs.map((item) => ({
@@ -250,6 +258,7 @@ export const appData2FlowNodeIO = ({
 
   return {
     inputs: [
+      Input_Template_Stream_MODE,
       Input_Template_History,
       Input_Template_UserChatInput,
       // ...(showFileLink ? [Input_Template_File_Link] : []),

diff --git a/packages/service/core/ai/functions/createQuestionGuide.ts b/packages/service/core/ai/functions/createQuestionGuide.ts
@@ -4,7 +4,11 @@ import { countGptMessagesTokens } from '../../../common/string/tiktoken/index';
 import { loadRequestMessages } from '../../chat/utils';
 import { llmCompletionsBodyFormat } from '../utils';
 
-export const Prompt_QuestionGuide = `你是一个AI智能助手，可以回答和解决我的问题。请结合前面的对话记录，帮我生成 3 个问题，引导我继续提问，生成问题的语言要与原问题相同。问题的长度应小于20个字符，按 JSON 格式返回: ["问题1", "问题2", "问题3"]`;
+export const Prompt_QuestionGuide = `你是一个AI智能助手，你的任务是结合对话记录，推测我下一步的问题。
+你需要生成 3 个可能的问题，引导我继续提问，生成的问题要求：
+1. 生成问题的语言，与最后一个用户提问语言一致。
+2. 问题的长度应小于20个字符。
+3. 按 JSON 格式返回: ["question1", "question2", "question3"]。`;
 
 export async function createQuestionGuide({
   messages,

diff --git a/packages/service/core/app/plugin/controller.ts b/packages/service/core/app/plugin/controller.ts
@@ -34,7 +34,7 @@ export async function splitCombinePluginId(id: string) {
   return { source, pluginId: id };
 }
 
-const getPluginTemplateById = async (
+const getChildAppTemplateById = async (
   id: string
 ): Promise<SystemPluginTemplateItemType & { teamId?: string }> => {
   const { source, pluginId } = await splitCombinePluginId(id);
@@ -69,44 +69,48 @@ const getPluginTemplateById = async (
 };
 
 /* format plugin modules to plugin preview module */
-export async function getPluginPreviewNode({ id }: { id: string }): Promise<FlowNodeTemplateType> {
-  const plugin = await getPluginTemplateById(id);
-  const isPlugin = !!plugin.workflow.nodes.find(
+export async function getChildAppPreviewNode({
+  id
+}: {
+  id: string;
+}): Promise<FlowNodeTemplateType> {
+  const app = await getChildAppTemplateById(id);
+  const isPlugin = !!app.workflow.nodes.find(
     (node) => node.flowNodeType === FlowNodeTypeEnum.pluginInput
   );
 
   return {
     id: getNanoid(),
-    pluginId: plugin.id,
-    templateType: plugin.templateType,
+    pluginId: app.id,
+    templateType: app.templateType,
     flowNodeType: isPlugin ? FlowNodeTypeEnum.pluginModule : FlowNodeTypeEnum.appModule,
-    avatar: plugin.avatar,
-    name: plugin.name,
-    intro: plugin.intro,
-    inputExplanationUrl: plugin.inputExplanationUrl,
-    showStatus: plugin.showStatus,
+    avatar: app.avatar,
+    name: app.name,
+    intro: app.intro,
+    inputExplanationUrl: app.inputExplanationUrl,
+    showStatus: app.showStatus,
     isTool: isPlugin,
-    version: plugin.version,
+    version: app.version,
     sourceHandle: getHandleConfig(true, true, true, true),
     targetHandle: getHandleConfig(true, true, true, true),
     ...(isPlugin
-      ? pluginData2FlowNodeIO({ nodes: plugin.workflow.nodes })
-      : appData2FlowNodeIO({ chatConfig: plugin.workflow.chatConfig }))
+      ? pluginData2FlowNodeIO({ nodes: app.workflow.nodes })
+      : appData2FlowNodeIO({ chatConfig: app.workflow.chatConfig }))
   };
 }
 
 /* run plugin time */
-export async function getPluginRuntimeById(id: string): Promise<PluginRuntimeType> {
-  const plugin = await getPluginTemplateById(id);
+export async function getChildAppRuntimeById(id: string): Promise<PluginRuntimeType> {
+  const app = await getChildAppTemplateById(id);
 
   return {
-    id: plugin.id,
-    teamId: plugin.teamId,
-    name: plugin.name,
-    avatar: plugin.avatar,
-    showStatus: plugin.showStatus,
-    currentCost: plugin.currentCost,
-    nodes: plugin.workflow.nodes,
-    edges: plugin.workflow.edges
+    id: app.id,
+    teamId: app.teamId,
+    name: app.name,
+    avatar: app.avatar,
+    showStatus: app.showStatus,
+    currentCost: app.currentCost,
+    nodes: app.workflow.nodes,
+    edges: app.workflow.edges
   };
 }
diff --git a/packages/service/core/app/plugin/utils.ts b/packages/service/core/app/plugin/utils.ts
@@ -13,6 +13,7 @@ export const computedPluginUsage = async (
 ) => {
   const { source } = await splitCombinePluginId(plugin.id);
 
+  // Commercial plugin: n points per times
   if (source === PluginSourceEnum.commercial) {
     return plugin.currentCost ?? 0;
   }

diff --git a/packages/service/core/workflow/dispatch/plugin/run.ts b/packages/service/core/workflow/dispatch/plugin/run.ts
@@ -2,7 +2,7 @@ import type { ModuleDispatchProps } from '@fastgpt/global/core/workflow/runtime/
 import { dispatchWorkFlow } from '../index';
 import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
 import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
-import { getPluginRuntimeById } from '../../../app/plugin/controller';
+import { getChildAppRuntimeById } from '../../../app/plugin/controller';
 import {
   getWorkflowEntryNodeIds,
   initWorkflowEdgeStatus,
@@ -16,8 +16,10 @@ import { filterSystemVariables } from '../utils';
 import { chatValue2RuntimePrompt } from '@fastgpt/global/core/chat/adapt';
 import { getPluginRunUserQuery } from '@fastgpt/global/core/workflow/utils';
 import { getPluginInputsFromStoreNodes } from '@fastgpt/global/core/app/plugin/utils';
+import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
 
 type RunPluginProps = ModuleDispatchProps<{
+  [NodeInputKeyEnum.forbidStream]?: boolean;
   [key: string]: any;
 }>;
 type RunPluginResponse = DispatchNodeResultType<{}>;
@@ -26,9 +28,8 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPlugi
   const {
     node: { pluginId },
     runningAppInfo,
-    mode,
     query,
-    params: data // Plugin input
+    params: { system_forbid_stream = false, ...data } // Plugin input
   } = props;
 
   if (!pluginId) {
@@ -44,7 +45,7 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPlugi
     per: ReadPermissionVal
   });
 
-  const plugin = await getPluginRuntimeById(pluginId);
+  const plugin = await getChildAppRuntimeById(pluginId);
 
   const runtimeNodes = storeNodes2RuntimeNodes(
     plugin.nodes,
@@ -73,6 +74,13 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPlugi
 
   const { flowResponses, flowUsages, assistantResponses, runTimes } = await dispatchWorkFlow({
     ...props,
+    // Rewrite stream mode
+    ...(system_forbid_stream
+      ? {
+          stream: false,
+          workflowStreamResponse: undefined
+        }
+      : {}),
     runningAppInfo: {
       id: String(plugin.id),
       teamId: plugin.teamId || '',
@@ -95,19 +103,20 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPlugi
     output.moduleLogo = plugin.avatar;
   }
 
-  const isError = !!output?.pluginOutput?.error;
-  const usagePoints = isError ? 0 : await computedPluginUsage(plugin, flowUsages);
+  const usagePoints = await computedPluginUsage(plugin, flowUsages);
+  const childStreamResponse = system_forbid_stream ? false : props.stream;
 
   return {
-    assistantResponses,
+    // 嵌套运行时，如果 childApp stream=false，实际上不会有任何内容输出给用户，所以不需要存储
+    assistantResponses: childStreamResponse ? assistantResponses : [],
     // responseData, // debug
     [DispatchNodeResponseKeyEnum.runTimes]: runTimes,
     [DispatchNodeResponseKeyEnum.nodeResponse]: {
       moduleLogo: plugin.avatar,
       totalPoints: usagePoints,
       pluginOutput: output?.pluginOutput,
       pluginDetail:
-        mode === 'test' && plugin.teamId === runningAppInfo.teamId
+        pluginData && pluginData.permission.hasWritePer // Not system plugin
           ? flowResponses.filter((item) => {
               const filterArr = [FlowNodeTypeEnum.pluginOutput];
               return !filterArr.includes(item.moduleType as any);