diff --git a/.github/ISSUE_TEMPLATE/bugs.md b/.github/ISSUE_TEMPLATE/bugs.md index 4c0f0bf1b4f..5988823b5d1 100644 --- a/.github/ISSUE_TEMPLATE/bugs.md +++ b/.github/ISSUE_TEMPLATE/bugs.md @@ -21,7 +21,7 @@ assignees: '' - [ ] 公有云版本 - [ ] 私有部署版本, 具体版本号: -**问题描述** +**问题描述, 日志截图** **复现步骤** diff --git a/packages/global/core/workflow/template/system/tools.ts b/packages/global/core/workflow/template/system/tools.ts index bb649e4c857..43085546648 100644 --- a/packages/global/core/workflow/template/system/tools.ts +++ b/packages/global/core/workflow/template/system/tools.ts @@ -64,5 +64,14 @@ export const ToolModule: FlowNodeTemplateType = { Input_Template_History, Input_Template_UserChatInput ], - outputs: [] + outputs: [ + { + id: NodeOutputKeyEnum.answerText, + key: NodeOutputKeyEnum.answerText, + label: 'core.module.output.label.Ai response content', + description: 'core.module.output.description.Ai response content', + valueType: WorkflowIOValueTypeEnum.string, + type: FlowNodeOutputTypeEnum.static + } + ] }; diff --git a/packages/service/common/file/gridfs/controller.ts b/packages/service/common/file/gridfs/controller.ts index ca3d0751306..34e9658d4b9 100644 --- a/packages/service/common/file/gridfs/controller.ts +++ b/packages/service/common/file/gridfs/controller.ts @@ -6,7 +6,6 @@ import { DatasetFileSchema } from '@fastgpt/global/core/dataset/type'; import { MongoFileSchema } from './schema'; import { detectFileEncoding } from '@fastgpt/global/common/file/tools'; import { CommonErrEnum } from '@fastgpt/global/common/error/code/common'; -import { ReadFileByBufferParams } from '../read/type'; import { MongoRwaTextBuffer } from '../../buffer/rawText/schema'; import { readFileRawContent } from '../read/utils'; import { PassThrough } from 'stream'; @@ -197,19 +196,15 @@ export const readFileContentFromMongo = async ({ }); })(); - const params: ReadFileByBufferParams = { + const { rawText } = await readFileRawContent({ + extension, + csvFormat, teamId, buffer: fileBuffers, encoding, metadata: { relatedId: fileId } - }; - - const { rawText } = await readFileRawContent({ - extension, - csvFormat, - params }); if (rawText.trim()) { diff --git a/packages/service/common/file/read/html.ts b/packages/service/common/file/read/html.ts deleted file mode 100644 index b3f9476de2f..00000000000 --- a/packages/service/common/file/read/html.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { ReadFileByBufferParams, ReadFileResponse } from './type.d'; -import { initMarkdownText } from './utils'; -import { htmlToMarkdown } from '../../string/markdown'; -import { readFileRawText } from './rawText'; - -export const readHtmlRawText = async ( - params: ReadFileByBufferParams -): Promise => { - const { teamId, metadata } = params; - const { rawText: html } = readFileRawText(params); - - const md = await htmlToMarkdown(html); - - const rawText = await initMarkdownText({ - teamId, - md, - metadata - }); - - return { - rawText - }; -}; diff --git a/packages/service/common/file/read/markdown.ts b/packages/service/common/file/read/markdown.ts deleted file mode 100644 index 982e75240d4..00000000000 --- a/packages/service/common/file/read/markdown.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { ReadFileByBufferParams, ReadFileResponse } from './type.d'; -import { initMarkdownText } from './utils'; -import { readFileRawText } from './rawText'; - -export const readMarkdown = async (params: ReadFileByBufferParams): Promise => { - const { teamId, metadata } = params; - const { rawText: md } = readFileRawText(params); - - const rawText = await initMarkdownText({ - teamId, - md, - metadata - }); - - return { - rawText - }; -}; diff --git a/packages/service/common/file/read/type.d.ts b/packages/service/common/file/read/type.d.ts deleted file mode 100644 index fbbbda5e9ff..00000000000 --- a/packages/service/common/file/read/type.d.ts +++ /dev/null @@ -1,12 +0,0 @@ -export type ReadFileByBufferParams = { - teamId: string; - buffer: Buffer; - encoding: string; - metadata?: Record; -}; - -export type ReadFileResponse = { - rawText: string; - formatText?: string; - metadata?: Record; -}; diff --git a/packages/service/common/file/read/utils.ts b/packages/service/common/file/read/utils.ts index 676d3d022df..c35b671fc34 100644 --- a/packages/service/common/file/read/utils.ts +++ b/packages/service/common/file/read/utils.ts @@ -1,16 +1,10 @@ -import { markdownProcess } from '@fastgpt/global/common/string/markdown'; +import { markdownProcess, simpleMarkdownText } from '@fastgpt/global/common/string/markdown'; import { uploadMongoImg } from '../image/controller'; import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants'; import { addHours } from 'date-fns'; -import { ReadFileByBufferParams } from './type'; -import { readFileRawText } from '../read/rawText'; -import { readMarkdown } from '../read/markdown'; -import { readHtmlRawText } from '../read/html'; -import { readPdfFile } from '../read/pdf'; -import { readWordFile } from '../read/word'; -import { readCsvRawText } from '../read/csv'; -import { readPptxRawText } from '../read/pptx'; -import { readXlsxRawText } from '../read/xlsx'; + +import { WorkerNameEnum, runWorker } from '../../../worker/utils'; +import { ReadFileResponse } from '../../../worker/file/type'; export const initMarkdownText = ({ teamId, @@ -36,46 +30,39 @@ export const initMarkdownText = ({ export const readFileRawContent = async ({ extension, csvFormat, - params + teamId, + buffer, + encoding, + metadata }: { csvFormat?: boolean; extension: string; - params: ReadFileByBufferParams; + teamId: string; + buffer: Buffer; + encoding: string; + metadata?: Record; }) => { - switch (extension) { - case 'txt': - return readFileRawText(params); - case 'md': - return readMarkdown(params); - case 'html': - return readHtmlRawText(params); - case 'pdf': - return readPdfFile(params); - case 'docx': - return readWordFile(params); - case 'pptx': - return readPptxRawText(params); - case 'xlsx': - const xlsxResult = await readXlsxRawText(params); - if (csvFormat) { - return { - rawText: xlsxResult.formatText || '' - }; - } - return { - rawText: xlsxResult.rawText - }; - case 'csv': - const csvResult = await readCsvRawText(params); - if (csvFormat) { - return { - rawText: csvResult.formatText || '' - }; - } - return { - rawText: csvResult.rawText - }; - default: - return Promise.reject('Only support .txt, .md, .html, .pdf, .docx, pptx, .csv, .xlsx'); + const result = await runWorker(WorkerNameEnum.readFile, { + extension, + csvFormat, + encoding, + buffer + }); + + // markdown data format + if (['md', 'html', 'docx'].includes(extension)) { + result.rawText = await initMarkdownText({ + teamId: teamId, + md: result.rawText, + metadata: metadata + }); } + + return result; +}; + +export const htmlToMarkdown = async (html?: string | null) => { + const md = await runWorker(WorkerNameEnum.htmlStr2Md, { html: html || '' }); + + return simpleMarkdownText(md); }; diff --git a/packages/service/common/file/read/word.ts b/packages/service/common/file/read/word.ts deleted file mode 100644 index 0ef5baa81e0..00000000000 --- a/packages/service/common/file/read/word.ts +++ /dev/null @@ -1,35 +0,0 @@ -import mammoth from 'mammoth'; -import { htmlToMarkdown } from '../../string/markdown'; -import { ReadFileByBufferParams, ReadFileResponse } from './type'; -import { initMarkdownText } from './utils'; - -/** - * read docx to markdown - */ -export const readWordFile = async ({ - teamId, - buffer, - metadata = {} -}: ReadFileByBufferParams): Promise => { - try { - const { value: html } = await mammoth.convertToHtml({ - buffer - }); - - const md = await htmlToMarkdown(html); - - const rawText = await initMarkdownText({ - teamId, - md, - metadata - }); - - return { - rawText, - metadata: {} - }; - } catch (error) { - console.log('error doc read:', error); - return Promise.reject('Can not read doc file, please convert to PDF'); - } -}; diff --git a/packages/service/common/string/cheerio.ts b/packages/service/common/string/cheerio.ts index 5bb56495baa..d7056fdb496 100644 --- a/packages/service/common/string/cheerio.ts +++ b/packages/service/common/string/cheerio.ts @@ -1,7 +1,7 @@ import { UrlFetchParams, UrlFetchResponse } from '@fastgpt/global/common/file/api'; import * as cheerio from 'cheerio'; import axios from 'axios'; -import { htmlToMarkdown } from './markdown'; +import { htmlToMarkdown } from '../file/read/utils'; export const cheerioToHtml = ({ fetchUrl, @@ -77,7 +77,9 @@ export const urlsFetch = async ({ $, selector }); + console.log('html====', html); const md = await htmlToMarkdown(html); + console.log('html====', md); return { url, diff --git a/packages/service/common/string/markdown.ts b/packages/service/common/string/markdown.ts deleted file mode 100644 index 158b95d85ce..00000000000 --- a/packages/service/common/string/markdown.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { simpleMarkdownText } from '@fastgpt/global/common/string/markdown'; -import { WorkerNameEnum, runWorker } from '../../worker/utils'; - -/* html string to markdown */ -export const htmlToMarkdown = async (html?: string | null) => { - const md = await runWorker(WorkerNameEnum.htmlStr2Md, { html: html || '' }); - - return simpleMarkdownText(md); -}; diff --git a/packages/service/common/vectorStore/pg/controller.ts b/packages/service/common/vectorStore/pg/controller.ts index 2cf0da090ec..679aa807d6e 100644 --- a/packages/service/common/vectorStore/pg/controller.ts +++ b/packages/service/common/vectorStore/pg/controller.ts @@ -23,7 +23,7 @@ export async function initPg() { `); await PgClient.query( - `CREATE INDEX CONCURRENTLY IF NOT EXISTS vector_index ON ${PgDatasetTableName} USING hnsw (vector vector_ip_ops) WITH (m = 32, ef_construction = 64);` + `CREATE INDEX CONCURRENTLY IF NOT EXISTS vector_index ON ${PgDatasetTableName} USING hnsw (vector vector_ip_ops) WITH (m = 32, ef_construction = 100);` ); await PgClient.query( `CREATE INDEX CONCURRENTLY IF NOT EXISTS team_dataset_collection_index ON ${PgDatasetTableName} USING btree(team_id, dataset_id, collection_id);` diff --git a/packages/service/core/workflow/dispatch/agent/classifyQuestion.ts b/packages/service/core/workflow/dispatch/agent/classifyQuestion.ts index d35971fc274..f09ac158ac8 100644 --- a/packages/service/core/workflow/dispatch/agent/classifyQuestion.ts +++ b/packages/service/core/workflow/dispatch/agent/classifyQuestion.ts @@ -131,7 +131,9 @@ const completions = async ({ console.log(answer, '----'); const id = - agents.find((item) => answer.includes(item.key) || answer.includes(item.value))?.key || ''; + agents.find((item) => answer.includes(item.key))?.key || + agents.find((item) => answer.includes(item.value))?.key || + ''; return { tokens: await countMessagesTokens(messages), diff --git a/packages/service/core/workflow/dispatch/agent/runTool/index.ts b/packages/service/core/workflow/dispatch/agent/runTool/index.ts index 69a39bc04fc..c72620a1895 100644 --- a/packages/service/core/workflow/dispatch/agent/runTool/index.ts +++ b/packages/service/core/workflow/dispatch/agent/runTool/index.ts @@ -23,7 +23,9 @@ import { runToolWithPromptCall } from './promptCall'; import { replaceVariable } from '@fastgpt/global/common/string/tools'; import { Prompt_Tool_Call } from './constants'; -type Response = DispatchNodeResultType<{}>; +type Response = DispatchNodeResultType<{ + [NodeOutputKeyEnum.answerText]: string; +}>; export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise => { const { @@ -129,6 +131,10 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise< const flatUsages = dispatchFlowResponse.map((item) => item.flowUsages).flat(); return { + [NodeOutputKeyEnum.answerText]: assistantResponses + .filter((item) => item.text?.content) + .map((item) => item.text?.content || '') + .join(''), [DispatchNodeResponseKeyEnum.assistantResponses]: assistantResponses, [DispatchNodeResponseKeyEnum.nodeResponse]: { totalPoints: totalPointsUsage, diff --git a/packages/service/core/workflow/dispatch/index.ts b/packages/service/core/workflow/dispatch/index.ts index e8bd3e1c5ee..1b70320237d 100644 --- a/packages/service/core/workflow/dispatch/index.ts +++ b/packages/service/core/workflow/dispatch/index.ts @@ -142,10 +142,8 @@ export async function dispatchWorkFlow({ } if (assistantResponses) { chatAssistantResponse = chatAssistantResponse.concat(assistantResponses); - } - - // save assistant text response - if (answerText) { + } else if (answerText) { + // save assistant text response const isResponseAnswerText = inputs.find((item) => item.key === NodeInputKeyEnum.aiChatIsResponseText)?.value ?? true; if (isResponseAnswerText) { diff --git a/packages/service/core/workflow/dispatch/tools/answer.ts b/packages/service/core/workflow/dispatch/tools/answer.ts index 46e46c8d87d..0269903ce32 100644 --- a/packages/service/core/workflow/dispatch/tools/answer.ts +++ b/packages/service/core/workflow/dispatch/tools/answer.ts @@ -19,24 +19,24 @@ export const dispatchAnswer = (props: Record): AnswerResponse => { res, detail, stream, - node: { name }, params: { text = '' } } = props as AnswerProps; const formatText = typeof text === 'string' ? text : JSON.stringify(text, null, 2); + const responseText = `\n${formatText}`; if (res && stream) { responseWrite({ res, event: detail ? SseResponseEventEnum.fastAnswer : undefined, data: textAdaptGptResponse({ - text: `\n${formatText}` + text: responseText }) }); } return { - [NodeOutputKeyEnum.answerText]: formatText, + [NodeOutputKeyEnum.answerText]: responseText, [DispatchNodeResponseKeyEnum.nodeResponse]: { textOutput: formatText } diff --git a/packages/service/common/file/read/csv.ts b/packages/service/worker/file/extension/csv.ts similarity index 71% rename from packages/service/common/file/read/csv.ts rename to packages/service/worker/file/extension/csv.ts index 81cc6fb499f..85d3a3c5cf4 100644 --- a/packages/service/common/file/read/csv.ts +++ b/packages/service/worker/file/extension/csv.ts @@ -1,9 +1,9 @@ import Papa from 'papaparse'; -import { ReadFileByBufferParams, ReadFileResponse } from './type.d'; +import { ReadRawTextByBuffer, ReadFileResponse } from '../type'; import { readFileRawText } from './rawText'; // 加载源文件内容 -export const readCsvRawText = async (params: ReadFileByBufferParams): Promise => { +export const readCsvRawText = async (params: ReadRawTextByBuffer): Promise => { const { rawText } = readFileRawText(params); const csvArr = Papa.parse(rawText).data as string[][]; diff --git a/packages/service/worker/file/extension/docx.ts b/packages/service/worker/file/extension/docx.ts new file mode 100644 index 00000000000..7936a66729e --- /dev/null +++ b/packages/service/worker/file/extension/docx.ts @@ -0,0 +1,23 @@ +import mammoth from 'mammoth'; +import { ReadRawTextByBuffer, ReadFileResponse } from '../type'; +import { html2md } from '../../htmlStr2Md/utils'; + +/** + * read docx to markdown + */ +export const readDocsFile = async ({ buffer }: ReadRawTextByBuffer): Promise => { + try { + const { value: html } = await mammoth.convertToHtml({ + buffer + }); + + const rawText = html2md(html); + + return { + rawText + }; + } catch (error) { + console.log('error doc read:', error); + return Promise.reject('Can not read doc file, please convert to PDF'); + } +}; diff --git a/packages/service/worker/file/extension/html.ts b/packages/service/worker/file/extension/html.ts new file mode 100644 index 00000000000..18ac0dc1605 --- /dev/null +++ b/packages/service/worker/file/extension/html.ts @@ -0,0 +1,13 @@ +import { ReadRawTextByBuffer, ReadFileResponse } from '../type'; +import { readFileRawText } from './rawText'; +import { html2md } from '../../htmlStr2Md/utils'; + +export const readHtmlRawText = async (params: ReadRawTextByBuffer): Promise => { + const { rawText: html } = readFileRawText(params); + + const rawText = html2md(html); + + return { + rawText + }; +}; diff --git a/packages/service/common/file/read/pdf.ts b/packages/service/worker/file/extension/pdf.ts similarity index 88% rename from packages/service/common/file/read/pdf.ts rename to packages/service/worker/file/extension/pdf.ts index 270e3b4592b..b6e43baf2c9 100644 --- a/packages/service/common/file/read/pdf.ts +++ b/packages/service/worker/file/extension/pdf.ts @@ -1,7 +1,7 @@ import * as pdfjs from 'pdfjs-dist/legacy/build/pdf.mjs'; // @ts-ignore import('pdfjs-dist/legacy/build/pdf.worker.min.mjs'); -import { ReadFileByBufferParams, ReadFileResponse } from './type'; +import { ReadRawTextByBuffer, ReadFileResponse } from '../type'; type TokenType = { str: string; @@ -13,9 +13,7 @@ type TokenType = { hasEOL: boolean; }; -export const readPdfFile = async ({ - buffer -}: ReadFileByBufferParams): Promise => { +export const readPdfFile = async ({ buffer }: ReadRawTextByBuffer): Promise => { const readPDFPage = async (doc: any, pageNo: number) => { const page = await doc.getPage(pageNo); const tokenizedText = await page.getTextContent(); @@ -65,7 +63,6 @@ export const readPdfFile = async ({ loadingTask.destroy(); return { - rawText: pageTexts.join(''), - metadata: {} + rawText: pageTexts.join('') }; }; diff --git a/packages/service/common/file/read/pptx.ts b/packages/service/worker/file/extension/pptx.ts similarity index 61% rename from packages/service/common/file/read/pptx.ts rename to packages/service/worker/file/extension/pptx.ts index 439e394bf42..8d337412804 100644 --- a/packages/service/common/file/read/pptx.ts +++ b/packages/service/worker/file/extension/pptx.ts @@ -1,11 +1,11 @@ -import { ReadFileByBufferParams, ReadFileResponse } from './type.d'; +import { ReadRawTextByBuffer, ReadFileResponse } from '../type'; // import { parseOfficeAsync } from 'officeparser'; -import { parseOffice } from './parseOffice'; +import { parseOffice } from '../parseOffice'; export const readPptxRawText = async ({ buffer, encoding -}: ReadFileByBufferParams): Promise => { +}: ReadRawTextByBuffer): Promise => { const result = await parseOffice({ buffer, encoding: encoding as BufferEncoding, diff --git a/packages/service/common/file/read/rawText.ts b/packages/service/worker/file/extension/rawText.ts similarity index 70% rename from packages/service/common/file/read/rawText.ts rename to packages/service/worker/file/extension/rawText.ts index bdb907cd9c8..d009f5e3217 100644 --- a/packages/service/common/file/read/rawText.ts +++ b/packages/service/worker/file/extension/rawText.ts @@ -1,5 +1,5 @@ -import { ReadFileByBufferParams, ReadFileResponse } from './type.d'; import iconv from 'iconv-lite'; +import { ReadRawTextByBuffer, ReadFileResponse } from '../type'; const rawEncodingList = [ 'ascii', @@ -17,7 +17,7 @@ const rawEncodingList = [ ]; // 加载源文件内容 -export const readFileRawText = ({ buffer, encoding }: ReadFileByBufferParams): ReadFileResponse => { +export const readFileRawText = ({ buffer, encoding }: ReadRawTextByBuffer): ReadFileResponse => { const content = rawEncodingList.includes(encoding) ? buffer.toString(encoding as BufferEncoding) : iconv.decode(buffer, 'gbk'); diff --git a/packages/service/common/file/read/xlsx.ts b/packages/service/worker/file/extension/xlsx.ts similarity index 88% rename from packages/service/common/file/read/xlsx.ts rename to packages/service/worker/file/extension/xlsx.ts index 774e8e7d26e..a6105ed7b27 100644 --- a/packages/service/common/file/read/xlsx.ts +++ b/packages/service/worker/file/extension/xlsx.ts @@ -1,10 +1,10 @@ -import { ReadFileByBufferParams, ReadFileResponse } from './type.d'; +import { ReadRawTextByBuffer, ReadFileResponse } from '../type'; import xlsx from 'node-xlsx'; import Papa from 'papaparse'; export const readXlsxRawText = async ({ buffer -}: ReadFileByBufferParams): Promise => { +}: ReadRawTextByBuffer): Promise => { const result = xlsx.parse(buffer, { skipHidden: false, defval: '' diff --git a/packages/service/common/file/read/parseOffice.ts b/packages/service/worker/file/parseOffice.ts similarity index 97% rename from packages/service/common/file/read/parseOffice.ts rename to packages/service/worker/file/parseOffice.ts index 327b120e949..4777483cf72 100644 --- a/packages/service/common/file/read/parseOffice.ts +++ b/packages/service/worker/file/parseOffice.ts @@ -2,8 +2,8 @@ import { getNanoid } from '@fastgpt/global/common/string/tools'; import fs from 'fs'; import decompress from 'decompress'; import { DOMParser } from '@xmldom/xmldom'; -import { clearDirFiles } from '../utils'; -import { addLog } from '../../system/log'; +import { clearDirFiles } from '../../common/file/utils'; +import { addLog } from '../../common/system/log'; const DEFAULTDECOMPRESSSUBLOCATION = '/tmp'; diff --git a/packages/service/worker/file/read.ts b/packages/service/worker/file/read.ts new file mode 100644 index 00000000000..02f30faa6c6 --- /dev/null +++ b/packages/service/worker/file/read.ts @@ -0,0 +1,71 @@ +import { parentPort } from 'worker_threads'; +import { readFileRawText } from './extension/rawText'; +import { ReadRawTextByBuffer, ReadRawTextProps } from './type'; +import { readHtmlRawText } from './extension/html'; +import { readPdfFile } from './extension/pdf'; +import { readDocsFile } from './extension/docx'; +import { readPptxRawText } from './extension/pptx'; +import { readXlsxRawText } from './extension/xlsx'; +import { readCsvRawText } from './extension/csv'; + +parentPort?.on('message', async (props: ReadRawTextProps) => { + const readFileRawContent = async (params: ReadRawTextByBuffer) => { + switch (params.extension) { + case 'txt': + case 'md': + return readFileRawText(params); + case 'html': + return readHtmlRawText(params); + case 'pdf': + return readPdfFile(params); + case 'docx': + return readDocsFile(params); + case 'pptx': + return readPptxRawText(params); + case 'xlsx': + const xlsxResult = await readXlsxRawText(params); + if (params.csvFormat) { + return { + rawText: xlsxResult.formatText || '' + }; + } + return { + rawText: xlsxResult.rawText + }; + case 'csv': + const csvResult = await readCsvRawText(params); + if (params.csvFormat) { + return { + rawText: csvResult.formatText || '' + }; + } + return { + rawText: csvResult.rawText + }; + default: + return Promise.reject('Only support .txt, .md, .html, .pdf, .docx, pptx, .csv, .xlsx'); + } + }; + + // params.buffer: Uint8Array -> buffer + const buffer = Buffer.from(props.buffer); + const newProps: ReadRawTextByBuffer = { + ...props, + buffer + }; + + try { + parentPort?.postMessage({ + type: 'success', + data: await readFileRawContent(newProps) + }); + } catch (error) { + console.log(error); + parentPort?.postMessage({ + type: 'error', + data: error + }); + } + + global?.close?.(); +}); diff --git a/packages/service/worker/file/type.d.ts b/packages/service/worker/file/type.d.ts new file mode 100644 index 00000000000..0d136861d74 --- /dev/null +++ b/packages/service/worker/file/type.d.ts @@ -0,0 +1,15 @@ +import { ReadFileByBufferParams } from '../../common/file/read/type'; + +export type ReadRawTextProps = { + csvFormat?: boolean; + extension: string; + buffer: T; + encoding: string; +}; + +export type ReadRawTextByBuffer = ReadRawTextProps; + +export type ReadFileResponse = { + rawText: string; + formatText?: string; +}; diff --git a/packages/service/worker/htmlStr2Md.ts b/packages/service/worker/htmlStr2Md.ts deleted file mode 100644 index cd4f0e2d28a..00000000000 --- a/packages/service/worker/htmlStr2Md.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { parentPort } from 'worker_threads'; -import TurndownService from 'turndown'; -//@ts-ignore -import domino from 'domino'; -//@ts-ignore -import * as turndownPluginGfm from 'joplin-turndown-plugin-gfm'; - -const turndownService = new TurndownService({ - headingStyle: 'atx', - bulletListMarker: '-', - codeBlockStyle: 'fenced', - fence: '```', - emDelimiter: '_', - strongDelimiter: '**', - linkStyle: 'inlined', - linkReferenceStyle: 'full' -}); -parentPort?.on('message', (params: { html: string }) => { - const html2md = (html: string): string => { - try { - const window = domino.createWindow(html); - const document = window.document; - - turndownService.remove(['i', 'script', 'iframe']); - turndownService.addRule('codeBlock', { - filter: 'pre', - replacement(_, node) { - const content = node.textContent?.trim() || ''; - // @ts-ignore - const codeName = node?._attrsByQName?.class?.data?.trim() || ''; - - return `\n\`\`\`${codeName}\n${content}\n\`\`\`\n`; - } - }); - - turndownService.use(turndownPluginGfm.gfm); - - // @ts-ignore - return turndownService.turndown(document); - } catch (error) { - return ''; - } - }; - - try { - const md = html2md(params?.html || ''); - - parentPort?.postMessage({ - type: 'success', - data: md - }); - } catch (error) { - parentPort?.postMessage({ - type: 'error', - data: error - }); - } - - global?.close?.(); -}); diff --git a/packages/service/worker/htmlStr2Md/index.ts b/packages/service/worker/htmlStr2Md/index.ts new file mode 100644 index 00000000000..95944ee1b58 --- /dev/null +++ b/packages/service/worker/htmlStr2Md/index.ts @@ -0,0 +1,20 @@ +import { parentPort } from 'worker_threads'; +import { html2md } from './utils'; + +parentPort?.on('message', (params: { html: string }) => { + try { + const md = html2md(params?.html || ''); + + parentPort?.postMessage({ + type: 'success', + data: md + }); + } catch (error) { + parentPort?.postMessage({ + type: 'error', + data: error + }); + } + + global?.close?.(); +}); diff --git a/packages/service/worker/htmlStr2Md/utils.ts b/packages/service/worker/htmlStr2Md/utils.ts new file mode 100644 index 00000000000..8ea2e38be70 --- /dev/null +++ b/packages/service/worker/htmlStr2Md/utils.ts @@ -0,0 +1,40 @@ +import TurndownService from 'turndown'; +const domino = require('domino-ext'); +const turndownPluginGfm = require('joplin-turndown-plugin-gfm'); + +export const html2md = (html: string): string => { + const turndownService = new TurndownService({ + headingStyle: 'atx', + bulletListMarker: '-', + codeBlockStyle: 'fenced', + fence: '```', + emDelimiter: '_', + strongDelimiter: '**', + linkStyle: 'inlined', + linkReferenceStyle: 'full' + }); + + try { + const window = domino.createWindow(html); + const document = window.document; + + turndownService.remove(['i', 'script', 'iframe']); + turndownService.addRule('codeBlock', { + filter: 'pre', + replacement(_, node) { + const content = node.textContent?.trim() || ''; + // @ts-ignore + const codeName = node?._attrsByQName?.class?.data?.trim() || ''; + + return `\n\`\`\`${codeName}\n${content}\n\`\`\`\n`; + } + }); + + turndownService.use(turndownPluginGfm.gfm); + + return turndownService.turndown(document); + } catch (error) { + console.log('html 2 markdown error', error); + return ''; + } +}; diff --git a/packages/service/worker/utils.ts b/packages/service/worker/utils.ts index 70508e7ea5c..f9ab4be72f9 100644 --- a/packages/service/worker/utils.ts +++ b/packages/service/worker/utils.ts @@ -2,6 +2,7 @@ import { Worker } from 'worker_threads'; import path from 'path'; export enum WorkerNameEnum { + readFile = 'readFile', htmlStr2Md = 'htmlStr2Md', countGptMessagesTokens = 'countGptMessagesTokens' } diff --git a/packages/web/hooks/useScrollPagination.tsx b/packages/web/hooks/useScrollPagination.tsx index bbafb66b9ed..e2222a065fb 100644 --- a/packages/web/hooks/useScrollPagination.tsx +++ b/packages/web/hooks/useScrollPagination.tsx @@ -37,7 +37,7 @@ export function useScrollPagination< const [data, setData] = useState([]); const [isLoading, { setTrue, setFalse }] = useBoolean(false); - const [list] = useVirtualList(data, { + const [list] = useVirtualList(data, { containerTarget: containerRef, wrapperTarget: wrapperRef, itemHeight, diff --git a/projects/app/next.config.js b/projects/app/next.config.js index 3deb3ac5592..b8a87085a8b 100644 --- a/projects/app/next.config.js +++ b/projects/app/next.config.js @@ -51,11 +51,15 @@ const nextConfig = { ...entries, 'worker/htmlStr2Md': path.resolve( process.cwd(), - '../../packages/service/worker/htmlStr2Md.ts' + '../../packages/service/worker/htmlStr2Md/index.ts' ), 'worker/countGptMessagesTokens': path.resolve( process.cwd(), '../../packages/service/worker/tiktoken/countGptMessagesTokens.ts' + ), + 'worker/readFile': path.resolve( + process.cwd(), + '../../packages/service/worker/file/read.ts' ) }; } @@ -82,7 +86,12 @@ const nextConfig = { serverComponentsExternalPackages: ['mongoose', 'pg'], // 指定导出包优化,按需引入包模块 optimizePackageImports: ['mongoose', 'pg'], - outputFileTracingRoot: path.join(__dirname, '../../') + outputFileTracingRoot: path.join(__dirname, '../../'), + outputFileTracingIncludes: { + '/api/common/file/previewContent.ts': [ + path.resolve(process.cwd(), '../../packages/service/worker/**/*') + ] + } } }; diff --git a/projects/app/package.json b/projects/app/package.json index c977f1f29b5..6975d9fa1e4 100644 --- a/projects/app/package.json +++ b/projects/app/package.json @@ -1,6 +1,6 @@ { "name": "app", - "version": "4.7.1", + "version": "4.8", "private": false, "scripts": { "dev": "next dev", diff --git a/projects/app/src/components/core/workflow/Flow/nodes/NodeHttp/index.tsx b/projects/app/src/components/core/workflow/Flow/nodes/NodeHttp/index.tsx index 5c670047f33..9f165978c39 100644 --- a/projects/app/src/components/core/workflow/Flow/nodes/NodeHttp/index.tsx +++ b/projects/app/src/components/core/workflow/Flow/nodes/NodeHttp/index.tsx @@ -38,6 +38,7 @@ import IOTitle from '../../components/IOTitle'; import { useContextSelector } from 'use-context-selector'; import { WorkflowContext } from '../../../context'; import { getWorkflowGlobalVariables } from '@/web/core/workflow/utils'; +import { useMemoizedFn } from 'ahooks'; const CurlImportModal = dynamic(() => import('./CurlImportModal')); export const HttpHeaders = [ @@ -108,159 +109,136 @@ const RenderHttpMethodAndUrl = React.memo(function RenderHttpMethodAndUrl({ const requestMethods = inputs.find((item) => item.key === NodeInputKeyEnum.httpMethod); const requestUrl = inputs.find((item) => item.key === NodeInputKeyEnum.httpReqUrl); - const onChangeUrl = useCallback( - (e: React.ChangeEvent) => { + const onChangeUrl = (e: React.ChangeEvent) => { + onChangeNode({ + nodeId, + type: 'updateInput', + key: NodeInputKeyEnum.httpReqUrl, + value: { + ...requestUrl, + value: e.target.value + } + }); + }; + const onBlurUrl = (e: React.ChangeEvent) => { + const val = e.target.value; + // 拆分params和url + const url = val.split('?')[0]; + const params = val.split('?')[1]; + if (params) { + const paramsArr = params.split('&'); + const paramsObj = paramsArr.reduce((acc, cur) => { + const [key, value] = cur.split('='); + return { + ...acc, + [key]: value + }; + }, {}); + const inputParams = inputs.find((item) => item.key === NodeInputKeyEnum.httpParams); + + if (!inputParams || Object.keys(paramsObj).length === 0) return; + + const concatParams: PropsArrType[] = inputParams?.value || []; + Object.entries(paramsObj).forEach(([key, value]) => { + if (!concatParams.find((item) => item.key === key)) { + concatParams.push({ key, value: value as string, type: 'string' }); + } + }); + + onChangeNode({ + nodeId, + type: 'updateInput', + key: NodeInputKeyEnum.httpParams, + value: { + ...inputParams, + value: concatParams + } + }); + onChangeNode({ nodeId, type: 'updateInput', key: NodeInputKeyEnum.httpReqUrl, value: { ...requestUrl, - value: e.target.value + value: url } }); - }, - [nodeId, onChangeNode, requestUrl] - ); - const onBlurUrl = useCallback( - (e: React.ChangeEvent) => { - const val = e.target.value; - // 拆分params和url - const url = val.split('?')[0]; - const params = val.split('?')[1]; - if (params) { - const paramsArr = params.split('&'); - const paramsObj = paramsArr.reduce((acc, cur) => { - const [key, value] = cur.split('='); - return { - ...acc, - [key]: value - }; - }, {}); - const inputParams = inputs.find((item) => item.key === NodeInputKeyEnum.httpParams); - - if (!inputParams || Object.keys(paramsObj).length === 0) return; - - const concatParams: PropsArrType[] = inputParams?.value || []; - Object.entries(paramsObj).forEach(([key, value]) => { - if (!concatParams.find((item) => item.key === key)) { - concatParams.push({ key, value: value as string, type: 'string' }); - } - }); - - onChangeNode({ - nodeId, - type: 'updateInput', - key: NodeInputKeyEnum.httpParams, - value: { - ...inputParams, - value: concatParams - } - }); - - onChangeNode({ - nodeId, - type: 'updateInput', - key: NodeInputKeyEnum.httpReqUrl, - value: { - ...requestUrl, - value: url - } - }); - - toast({ - status: 'success', - title: t('core.module.http.Url and params have been split') - }); - } - }, - [inputs, nodeId, onChangeNode, requestUrl, t, toast] - ); + toast({ + status: 'success', + title: t('core.module.http.Url and params have been split') + }); + } + }; - const Render = useMemo(() => { - return ( - - - - {t('core.module.Http request settings')} - - + return ( + + + + {t('core.module.Http request settings')} - - { - onChangeNode({ - nodeId, - type: 'updateInput', - key: NodeInputKeyEnum.httpMethod, - value: { - ...requestMethods, - value: e - } - }); - }} - /> - - - - {isOpenCurl && } + - ); - }, [ - inputs, - isOpenCurl, - nodeId, - onBlurUrl, - onChangeNode, - onChangeUrl, - onCloseCurl, - onOpenCurl, - requestMethods, - requestUrl?.value, - t - ]); + + { + onChangeNode({ + nodeId, + type: 'updateInput', + key: NodeInputKeyEnum.httpMethod, + value: { + ...requestMethods, + value: e + } + }); + }} + /> + + - return Render; + {isOpenCurl && } + + ); }); export function RenderHttpProps({ @@ -644,15 +622,17 @@ const NodeHttp = ({ data, selected }: NodeProps) => { const splitToolInputs = useContextSelector(WorkflowContext, (v) => v.splitToolInputs); const { toolInputs, commonInputs, isTool } = splitToolInputs(inputs, nodeId); - const CustomComponents = useMemo( - () => ({ - [NodeInputKeyEnum.httpMethod]: () => ( - - ), - [NodeInputKeyEnum.httpHeaders]: () => - }), - [inputs, nodeId] - ); + const HttpMethodAndUrl = useMemoizedFn(() => ( + + )); + const Headers = useMemoizedFn(() => ); + + const CustomComponents = useMemo(() => { + return { + [NodeInputKeyEnum.httpMethod]: HttpMethodAndUrl, + [NodeInputKeyEnum.httpHeaders]: Headers + }; + }, [Headers, HttpMethodAndUrl]); return ( diff --git a/projects/app/src/components/core/workflow/Flow/nodes/NodeTools.tsx b/projects/app/src/components/core/workflow/Flow/nodes/NodeTools.tsx index 4d771390c1e..50b49321367 100644 --- a/projects/app/src/components/core/workflow/Flow/nodes/NodeTools.tsx +++ b/projects/app/src/components/core/workflow/Flow/nodes/NodeTools.tsx @@ -10,10 +10,11 @@ import { ToolSourceHandle } from './render/Handle/ToolHandle'; import { Box } from '@chakra-ui/react'; import IOTitle from '../components/IOTitle'; import MyIcon from '@fastgpt/web/components/common/Icon'; +import RenderOutput from './render/RenderOutput'; const NodeTools = ({ data, selected }: NodeProps) => { const { t } = useTranslation(); - const { nodeId, inputs } = data; + const { nodeId, inputs, outputs } = data; return ( @@ -21,7 +22,10 @@ const NodeTools = ({ data, selected }: NodeProps) => { - + + + + { ); const RenderLabel = useMemo(() => { - const renderType = renderTypeList[selectedTypeIndex || 0]; + const renderType = renderTypeList?.[selectedTypeIndex || 0]; return (