From 610ebded3bf1ef6555c3990bf6b9e0b4d924ec93 Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Mon, 13 May 2024 17:07:29 +0800 Subject: [PATCH 01/27] Dir tree doc and move some code (#1466) * tree doc and move some code * fix: ts --- docSite/content/docs/development/intro.md | 54 +++++++++++++ packages/global/common/string/time.ts | 36 +++++++++ project.inlang/project_id | 1 - project.inlang/settings.json | 23 ------ .../components/ChatBox/hooks/useChatBox.tsx | 2 +- .../app/src/components/core/app/TTSSelect.tsx | 2 +- .../workflow/Flow/nodes/NodeSystemConfig.tsx | 2 +- projects/app/src/constants/dataset.ts | 56 ------------- projects/app/src/pages/_error.tsx | 3 +- .../pages/account/components/InformTable.tsx | 2 +- .../components/Publish/FeiShu/index.tsx | 4 +- .../detail/components/Publish/Link/index.tsx | 4 +- .../detail/components/SimpleEdit/EditForm.tsx | 2 +- .../detail/components/InputDataModal.tsx | 2 +- .../pages/dataset/detail/components/Test.tsx | 2 +- .../login/components/ForgetPasswordForm.tsx | 2 +- .../login/components/LoginForm/LoginForm.tsx | 2 +- .../login/components/LoginForm/WechatForm.tsx | 2 +- .../LoginForm/components/FormLayout.tsx | 2 +- .../pages/login/components/RegisterForm.tsx | 2 +- projects/app/src/pages/login/index.tsx | 2 +- projects/app/src/types/index.d.ts | 4 +- projects/app/src/utils/tools.ts | 81 ------------------- .../app/src/web/common/system/constants.ts | 5 ++ projects/app/src/web/common/utils/voice.ts | 2 +- projects/app/src/web/context/useInitApp.ts | 3 +- .../app.ts => web/core/app/constants.ts} | 0 .../app/src/web/core/app/store/useAppStore.ts | 2 +- .../common.ts => web/core/chat/constants.ts} | 6 -- .../app/src/web/core/dataset/constants.ts | 57 +++++++++++++ .../app/src/web/core/dataset/store/dataset.ts | 2 +- .../support/user/login/constants.ts} | 5 -- 32 files changed, 178 insertions(+), 196 deletions(-) delete mode 100644 project.inlang/project_id delete mode 100644 project.inlang/settings.json delete mode 100644 projects/app/src/constants/dataset.ts delete mode 100644 projects/app/src/utils/tools.ts create mode 100644 projects/app/src/web/common/system/constants.ts rename projects/app/src/{constants/app.ts => web/core/app/constants.ts} (100%) rename projects/app/src/{constants/common.ts => web/core/chat/constants.ts} (99%) rename projects/app/src/{constants/user.ts => web/support/user/login/constants.ts} (68%) diff --git a/docSite/content/docs/development/intro.md b/docSite/content/docs/development/intro.md index 8859d23c3b8c..b0ff877b1de2 100644 --- a/docSite/content/docs/development/intro.md +++ b/docSite/content/docs/development/intro.md @@ -133,3 +133,57 @@ FastGPT 在`pnpm i`后会执行`postinstall`脚本，用于自动生成`ChakraUI 遇到困难了吗？有任何问题吗? 加入微信群与开发者和用户保持沟通。 + +## 代码结构说明 + +### nextjs + +FastGPT 使用了 nextjs 的 page route 作为框架。为了区分好前后端代码，在目录分配上会分成 global, service, web 3个自目录，分别对应着 `前后端共用`、`后端专用`、`前端专用`的代码。 + +### monorepo +FastGPT 才用 pnpm workspace 方式构建 monorepo 项目，主要分为两个部分： + +- projects/app - FastGPT 主项目 +- packages/ - 子模块 + - global - 共用代码，通常是放一些前后端都能执行的函数、类型声明、常量。 + - service - 服务端代码 + - web - 前端代码 + - plugin - 工作流自定义插件的代码 + +### 领域驱动模式（DDD） + +FastGPT 在代码模块划分时，按DDD的思想进行划分，主要分为以下几个领域： + +core - 核心功能（知识库，工作流，应用，对话） +support - 支撑功能（用户体系，计费，鉴权等） +common - 基础功能（日志管理，文件读写等） + +{{% details title="代码结构说明" closed="true" %}} +``` +. +├── .github // github 相关配置 +├── .husky // 格式化配置 +├── docSite // 文档 +├── files // 一些外部文件，例如 docker-compose, helm +├── packages // 子包 +│ ├── global // 前后端通用子包 +│ ├── plugins // 工作流插件（需要自定义包时候使用到） +│ ├── service // 后端子包 +│ └── web // 前端子包 +├── projects +│ └── app // FastGPT 主项目 +├── python // 存放一些模型代码，和 FastGPT 本身无关 +└── scripts // 一些自动化脚本 + ├── icon // icon预览脚本，可以在顶层 pnpm initIcon(把svg写入到代码中), pnpm previewIcon（预览icon） + └── postinstall.sh // chakraUI自定义theme初始化 ts 类型 +├── package.json // 顶层monorepo +├── pnpm-lock.yaml +├── pnpm-workspace.yaml // monorepo 声明 +├── Dockerfile +├── LICENSE +├── README.md +├── README_en.md +├── README_ja.md +├── dev.md +``` +{{% /details %}} diff --git a/packages/global/common/string/time.ts b/packages/global/common/string/time.ts index 4b3211102f46..e61aac7e3244 100644 --- a/packages/global/common/string/time.ts +++ b/packages/global/common/string/time.ts @@ -6,6 +6,42 @@ export const formatTime2YMDHM = (time?: Date) => export const formatTime2YMD = (time?: Date) => (time ? dayjs(time).format('YYYY-MM-DD') : ''); export const formatTime2HM = (time: Date = new Date()) => dayjs(time).format('HH:mm'); +/** + * 格式化时间成聊天格式 + */ +export const formatTimeToChatTime = (time: Date) => { + const now = dayjs(); + const target = dayjs(time); + + // 如果传入时间小于60秒，返回刚刚 + if (now.diff(target, 'second') < 60) { + return '刚刚'; + } + + // 如果时间是今天，展示几时:几分 + if (now.isSame(target, 'day')) { + return target.format('HH:mm'); + } + + // 如果是昨天，展示昨天 + if (now.subtract(1, 'day').isSame(target, 'day')) { + return '昨天'; + } + + // 如果是前天，展示前天 + if (now.subtract(2, 'day').isSame(target, 'day')) { + return '前天'; + } + + // 如果是今年，展示某月某日 + if (now.isSame(target, 'year')) { + return target.format('MM/DD'); + } + + // 如果是更久之前，展示某年某月某日 + return target.format('YYYY/M/D'); +}; + /* cron time parse */ export const cronParser2Fields = (cronString: string) => { try { diff --git a/project.inlang/project_id b/project.inlang/project_id deleted file mode 100644 index e7a4d2b389f9..000000000000 --- a/project.inlang/project_id +++ /dev/null @@ -1 +0,0 @@ -82dc3f099ca89165c3415d2ba96d195d3a1805b98373e8eb3d814b6994e62489 \ No newline at end of file diff --git a/project.inlang/settings.json b/project.inlang/settings.json deleted file mode 100644 index aff02ae447fe..000000000000 --- a/project.inlang/settings.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "$schema": "https://inlang.com/schema/project-settings", - "sourceLanguageTag": "en", - "languageTags": [ - "en", - "zh" - ], - "modules": [ - "https://cdn.jsdelivr.net/npm/@inlang/plugin-i18next@4/dist/index.js", - "https://cdn.jsdelivr.net/npm/@inlang/message-lint-rule-empty-pattern@1/dist/index.js", - "https://cdn.jsdelivr.net/npm/@inlang/message-lint-rule-identical-pattern@1/dist/index.js", - "https://cdn.jsdelivr.net/npm/@inlang/message-lint-rule-without-source@1/dist/index.js", - "https://cdn.jsdelivr.net/npm/@inlang/message-lint-rule-missing-translation@1/dist/index.js" - ], - "plugin.inlang.i18next": { - "pathPattern": { - "common": "./projects/app/i18n/{languageTag}/common.json" - }, - "variableReferencePattern": [ - "{{", "}}" - ] - } -} \ No newline at end of file diff --git a/projects/app/src/components/ChatBox/hooks/useChatBox.tsx b/projects/app/src/components/ChatBox/hooks/useChatBox.tsx index 31adb6a60cc3..10da0510534b 100644 --- a/projects/app/src/components/ChatBox/hooks/useChatBox.tsx +++ b/projects/app/src/components/ChatBox/hooks/useChatBox.tsx @@ -1,7 +1,7 @@ import { ExportChatType } from '@/types/chat'; import { ChatItemType } from '@fastgpt/global/core/chat/type'; import { useCallback } from 'react'; -import { htmlTemplate } from '@/constants/common'; +import { htmlTemplate } from '@/web/core/chat/constants'; import { fileDownload } from '@/web/common/file/utils'; export const useChatBox = () => { diff --git a/projects/app/src/components/core/app/TTSSelect.tsx b/projects/app/src/components/core/app/TTSSelect.tsx index 80525388c453..d5bea9714bc7 100644 --- a/projects/app/src/components/core/app/TTSSelect.tsx +++ b/projects/app/src/components/core/app/TTSSelect.tsx @@ -4,7 +4,7 @@ import { QuestionOutlineIcon } from '@chakra-ui/icons'; import { Box, Button, Flex, ModalBody, useDisclosure, Image } from '@chakra-ui/react'; import React, { useCallback, useMemo } from 'react'; import { useTranslation } from 'next-i18next'; -import { TTSTypeEnum } from '@/constants/app'; +import { TTSTypeEnum } from '@/web/core/app/constants'; import type { AppTTSConfigType } from '@fastgpt/global/core/app/type.d'; import { useAudioPlay } from '@/web/common/utils/voice'; import { useSystemStore } from '@/web/common/system/useSystemStore'; diff --git a/projects/app/src/components/core/workflow/Flow/nodes/NodeSystemConfig.tsx b/projects/app/src/components/core/workflow/Flow/nodes/NodeSystemConfig.tsx index bc31cde4b94e..a5b5aecc9f8a 100644 --- a/projects/app/src/components/core/workflow/Flow/nodes/NodeSystemConfig.tsx +++ b/projects/app/src/components/core/workflow/Flow/nodes/NodeSystemConfig.tsx @@ -11,7 +11,7 @@ import TTSSelect from '@/components/core/app/TTSSelect'; import WhisperConfig from '@/components/core/app/WhisperConfig'; import { splitGuideModule } from '@fastgpt/global/core/workflow/utils'; import { useTranslation } from 'next-i18next'; -import { TTSTypeEnum } from '@/constants/app'; +import { TTSTypeEnum } from '@/web/core/app/constants'; import MyIcon from '@fastgpt/web/components/common/Icon'; import MyTooltip from '@/components/MyTooltip'; import NodeCard from './render/NodeCard'; diff --git a/projects/app/src/constants/dataset.ts b/projects/app/src/constants/dataset.ts deleted file mode 100644 index 46ba6bf98cd7..000000000000 --- a/projects/app/src/constants/dataset.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { defaultQAModels, defaultVectorModels } from '@fastgpt/global/core/ai/model'; -import type { - DatasetCollectionItemType, - DatasetItemType -} from '@fastgpt/global/core/dataset/type.d'; - -export const defaultDatasetDetail: DatasetItemType = { - _id: '', - parentId: '', - userId: '', - teamId: '', - tmbId: '', - updateTime: new Date(), - type: 'dataset', - avatar: '/icon/logo.svg', - name: '', - intro: '', - status: 'active', - permission: 'private', - isOwner: false, - canWrite: false, - vectorModel: defaultVectorModels[0], - agentModel: defaultQAModels[0] -}; - -export const defaultCollectionDetail: DatasetCollectionItemType = { - _id: '', - teamId: '', - tmbId: '', - datasetId: { - _id: '', - parentId: '', - userId: '', - teamId: '', - tmbId: '', - updateTime: new Date(), - type: 'dataset', - avatar: '/icon/logo.svg', - name: '', - intro: '', - status: 'active', - permission: 'private', - vectorModel: defaultVectorModels[0].model, - agentModel: defaultQAModels[0].model - }, - parentId: '', - name: '', - type: 'file', - updateTime: new Date(), - canWrite: false, - sourceName: '', - sourceId: '', - createTime: new Date(), - trainingType: 'chunk', - chunkSize: 0 -}; diff --git a/projects/app/src/pages/_error.tsx b/projects/app/src/pages/_error.tsx index 1a1ed8fcd2bf..bbaae955a39c 100644 --- a/projects/app/src/pages/_error.tsx +++ b/projects/app/src/pages/_error.tsx @@ -3,6 +3,7 @@ import { useRouter } from 'next/router'; import { serviceSideProps } from '@/web/common/utils/i18n'; import { useSystemStore } from '@/web/common/system/useSystemStore'; import { Box } from '@chakra-ui/react'; +import { TrackEventName } from '@/web/common/system/constants'; function Error() { const router = useRouter(); @@ -10,7 +11,7 @@ function Error() { useEffect(() => { setTimeout(() => { - window.umami?.track('pageError', { + window.umami?.track(TrackEventName.pageError, { userAgent: navigator.userAgent, platform: navigator.platform, appName: navigator.appName, diff --git a/projects/app/src/pages/account/components/InformTable.tsx b/projects/app/src/pages/account/components/InformTable.tsx index 6810c32b2beb..50d5ef66e2fa 100644 --- a/projects/app/src/pages/account/components/InformTable.tsx +++ b/projects/app/src/pages/account/components/InformTable.tsx @@ -2,7 +2,7 @@ import React from 'react'; import { Box, Button, Flex, useTheme } from '@chakra-ui/react'; import { getInforms, readInform } from '@/web/support/user/inform/api'; import type { UserInformSchema } from '@fastgpt/global/support/user/inform/type'; -import { formatTimeToChatTime } from '@/utils/tools'; +import { formatTimeToChatTime } from '@fastgpt/global/common/string/time'; import { useSystemStore } from '@/web/common/system/useSystemStore'; import MyIcon from '@fastgpt/web/components/common/Icon'; import { usePagination } from '@fastgpt/web/hooks/usePagination'; diff --git a/projects/app/src/pages/app/detail/components/Publish/FeiShu/index.tsx b/projects/app/src/pages/app/detail/components/Publish/FeiShu/index.tsx index 9d6621ef3afd..41e8eb4d8d8e 100644 --- a/projects/app/src/pages/app/detail/components/Publish/FeiShu/index.tsx +++ b/projects/app/src/pages/app/detail/components/Publish/FeiShu/index.tsx @@ -15,9 +15,9 @@ import MyIcon from '@fastgpt/web/components/common/Icon'; import { useLoading } from '@fastgpt/web/hooks/useLoading'; import { useQuery } from '@tanstack/react-query'; import { getShareChatList, delShareChatById } from '@/web/support/outLink/api'; -import { formatTimeToChatTime } from '@/utils/tools'; +import { formatTimeToChatTime } from '@fastgpt/global/common/string/time'; import { useCopyData } from '@/web/common/hooks/useCopyData'; -import { defaultFeishuOutLinkForm } from '@/constants/app'; +import { defaultFeishuOutLinkForm } from '@/web/core/app/constants'; import type { FeishuType, OutLinkEditType } from '@fastgpt/global/support/outLink/type.d'; import { PublishChannelEnum } from '@fastgpt/global/support/outLink/constant'; import { useTranslation } from 'next-i18next'; diff --git a/projects/app/src/pages/app/detail/components/Publish/Link/index.tsx b/projects/app/src/pages/app/detail/components/Publish/Link/index.tsx index 92f3882ee850..7beb5491d2db 100644 --- a/projects/app/src/pages/app/detail/components/Publish/Link/index.tsx +++ b/projects/app/src/pages/app/detail/components/Publish/Link/index.tsx @@ -27,10 +27,10 @@ import { createShareChat, putShareChat } from '@/web/support/outLink/api'; -import { formatTimeToChatTime } from '@/utils/tools'; +import { formatTimeToChatTime } from '@fastgpt/global/common/string/time'; import { useCopyData } from '@/web/common/hooks/useCopyData'; import { useForm } from 'react-hook-form'; -import { defaultOutLinkForm } from '@/constants/app'; +import { defaultOutLinkForm } from '@/web/core/app/constants'; import type { OutLinkEditType, OutLinkSchema } from '@fastgpt/global/support/outLink/type.d'; import { useRequest } from '@/web/common/hooks/useRequest'; import { PublishChannelEnum } from '@fastgpt/global/support/outLink/constant'; diff --git a/projects/app/src/pages/app/detail/components/SimpleEdit/EditForm.tsx b/projects/app/src/pages/app/detail/components/SimpleEdit/EditForm.tsx index d2f86b0399bb..d81681ba514a 100644 --- a/projects/app/src/pages/app/detail/components/SimpleEdit/EditForm.tsx +++ b/projects/app/src/pages/app/detail/components/SimpleEdit/EditForm.tsx @@ -26,7 +26,7 @@ import SearchParamsTip from '@/components/core/dataset/SearchParamsTip'; import SettingLLMModel from '@/components/core/ai/SettingLLMModel'; import type { SettingAIDataType } from '@fastgpt/global/core/app/type.d'; import DeleteIcon, { hoverDeleteStyles } from '@fastgpt/web/components/common/Icon/delete'; -import { TTSTypeEnum } from '@/constants/app'; +import { TTSTypeEnum } from '@/web/core/app/constants'; import { getSystemVariables } from '@/web/core/app/utils'; import { useUpdate } from 'ahooks'; import { useI18n } from '@/web/context/I18n'; diff --git a/projects/app/src/pages/dataset/detail/components/InputDataModal.tsx b/projects/app/src/pages/dataset/detail/components/InputDataModal.tsx index 4114ecbd7d17..cb3f05388f33 100644 --- a/projects/app/src/pages/dataset/detail/components/InputDataModal.tsx +++ b/projects/app/src/pages/dataset/detail/components/InputDataModal.tsx @@ -20,7 +20,7 @@ import { getDefaultIndex } from '@fastgpt/global/core/dataset/utils'; import { DatasetDataIndexItemType } from '@fastgpt/global/core/dataset/type'; import SideTabs from '@/components/SideTabs'; import DeleteIcon from '@fastgpt/web/components/common/Icon/delete'; -import { defaultCollectionDetail } from '@/constants/dataset'; +import { defaultCollectionDetail } from '@/web/core/dataset/constants'; import { getDocPath } from '@/web/common/system/doc'; import RawSourceBox from '@/components/core/dataset/RawSourceBox'; import MyBox from '@fastgpt/web/components/common/MyBox'; diff --git a/projects/app/src/pages/dataset/detail/components/Test.tsx b/projects/app/src/pages/dataset/detail/components/Test.tsx index 105b29185a43..c0245417a440 100644 --- a/projects/app/src/pages/dataset/detail/components/Test.tsx +++ b/projects/app/src/pages/dataset/detail/components/Test.tsx @@ -5,7 +5,7 @@ import { useSearchTestStore, SearchTestStoreItemType } from '@/web/core/dataset/ import { postSearchText } from '@/web/core/dataset/api'; import MyIcon from '@fastgpt/web/components/common/Icon'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; -import { formatTimeToChatTime } from '@/utils/tools'; +import { formatTimeToChatTime } from '@fastgpt/global/common/string/time'; import { getErrText } from '@fastgpt/global/common/error/utils'; import { useToast } from '@fastgpt/web/hooks/useToast'; import { customAlphabet } from 'nanoid'; diff --git a/projects/app/src/pages/login/components/ForgetPasswordForm.tsx b/projects/app/src/pages/login/components/ForgetPasswordForm.tsx index b0f9c3774db2..3f73c9cf0248 100644 --- a/projects/app/src/pages/login/components/ForgetPasswordForm.tsx +++ b/projects/app/src/pages/login/components/ForgetPasswordForm.tsx @@ -1,7 +1,7 @@ import React, { useState, Dispatch, useCallback } from 'react'; import { FormControl, Box, Input, Button } from '@chakra-ui/react'; import { useForm } from 'react-hook-form'; -import { LoginPageTypeEnum } from '@/constants/user'; +import { LoginPageTypeEnum } from '@/web/support/user/login/constants'; import { postFindPassword } from '@/web/support/user/api'; import { useSendCode } from '@/web/support/user/hooks/useSendCode'; import type { ResLogin } from '@/global/support/api/userRes.d'; diff --git a/projects/app/src/pages/login/components/LoginForm/LoginForm.tsx b/projects/app/src/pages/login/components/LoginForm/LoginForm.tsx index d7a0d4900dae..5e8f49f6b5da 100644 --- a/projects/app/src/pages/login/components/LoginForm/LoginForm.tsx +++ b/projects/app/src/pages/login/components/LoginForm/LoginForm.tsx @@ -1,7 +1,7 @@ import React, { useState, Dispatch, useCallback } from 'react'; import { FormControl, Flex, Input, Button, Box, Link } from '@chakra-ui/react'; import { useForm } from 'react-hook-form'; -import { LoginPageTypeEnum } from '@/constants/user'; +import { LoginPageTypeEnum } from '@/web/support/user/login/constants'; import { postLogin } from '@/web/support/user/api'; import type { ResLogin } from '@/global/support/api/userRes'; import { useToast } from '@fastgpt/web/hooks/useToast'; diff --git a/projects/app/src/pages/login/components/LoginForm/WechatForm.tsx b/projects/app/src/pages/login/components/LoginForm/WechatForm.tsx index f1f90d4765a6..de534ea145bc 100644 --- a/projects/app/src/pages/login/components/LoginForm/WechatForm.tsx +++ b/projects/app/src/pages/login/components/LoginForm/WechatForm.tsx @@ -1,5 +1,5 @@ import React, { Dispatch } from 'react'; -import { LoginPageTypeEnum } from '@/constants/user'; +import { LoginPageTypeEnum } from '@/web/support/user/login/constants'; import type { ResLogin } from '@/global/support/api/userRes'; import { Box, Center, Image } from '@chakra-ui/react'; import { useQuery } from '@tanstack/react-query'; diff --git a/projects/app/src/pages/login/components/LoginForm/components/FormLayout.tsx b/projects/app/src/pages/login/components/LoginForm/components/FormLayout.tsx index a6acb621f956..3fc8445bfdc0 100644 --- a/projects/app/src/pages/login/components/LoginForm/components/FormLayout.tsx +++ b/projects/app/src/pages/login/components/LoginForm/components/FormLayout.tsx @@ -1,4 +1,4 @@ -import { LoginPageTypeEnum } from '@/constants/user'; +import { LoginPageTypeEnum } from '@/web/support/user/login/constants'; import { useSystemStore } from '@/web/common/system/useSystemStore'; import { AbsoluteCenter, Box, Button, Flex, Image } from '@chakra-ui/react'; import { LOGO_ICON } from '@fastgpt/global/common/system/constants'; diff --git a/projects/app/src/pages/login/components/RegisterForm.tsx b/projects/app/src/pages/login/components/RegisterForm.tsx index e7ba46a5fa24..b98caa0e0b83 100644 --- a/projects/app/src/pages/login/components/RegisterForm.tsx +++ b/projects/app/src/pages/login/components/RegisterForm.tsx @@ -1,7 +1,7 @@ import React, { useState, Dispatch, useCallback } from 'react'; import { FormControl, Box, Input, Button } from '@chakra-ui/react'; import { useForm } from 'react-hook-form'; -import { LoginPageTypeEnum } from '@/constants/user'; +import { LoginPageTypeEnum } from '@/web/support/user/login/constants'; import { postRegister } from '@/web/support/user/api'; import { useSendCode } from '@/web/support/user/hooks/useSendCode'; import type { ResLogin } from '@/global/support/api/userRes'; diff --git a/projects/app/src/pages/login/index.tsx b/projects/app/src/pages/login/index.tsx index bc76c1f0a020..a0464b5b03c0 100644 --- a/projects/app/src/pages/login/index.tsx +++ b/projects/app/src/pages/login/index.tsx @@ -1,6 +1,6 @@ import React, { useState, useCallback, useEffect } from 'react'; import { Box, Center, Flex, useDisclosure } from '@chakra-ui/react'; -import { LoginPageTypeEnum } from '@/constants/user'; +import { LoginPageTypeEnum } from '@/web/support/user/login/constants'; import { useSystemStore } from '@/web/common/system/useSystemStore'; import type { ResLogin } from '@/global/support/api/userRes.d'; import { useRouter } from 'next/router'; diff --git a/projects/app/src/types/index.d.ts b/projects/app/src/types/index.d.ts index b8a1f85e4ee7..5b0e10384e36 100644 --- a/projects/app/src/types/index.d.ts +++ b/projects/app/src/types/index.d.ts @@ -7,7 +7,7 @@ import { VectorModelItemType, WhisperModelType } from '@fastgpt/global/core/ai/model.d'; -import { TrackEventName } from '@/constants/common'; +import { TrackEventName } from '@/web/common/system/constants'; import { SubPlanType } from '@fastgpt/global/support/wallet/sub/type'; export type PagingData = { @@ -31,7 +31,7 @@ declare global { grecaptcha: any; QRCode: any; umami?: { - track: (event: `${TrackEventName}`, data: any) => void; + track: (event: TrackEventName, data: any) => void; }; } } diff --git a/projects/app/src/utils/tools.ts b/projects/app/src/utils/tools.ts deleted file mode 100644 index 76e7fcfe8f8f..000000000000 --- a/projects/app/src/utils/tools.ts +++ /dev/null @@ -1,81 +0,0 @@ -import dayjs from 'dayjs'; - -/** - * 对象转成 query 字符串 - */ -export const Obj2Query = (obj: Record) => { - const queryParams = new URLSearchParams(); - for (const key in obj) { - queryParams.append(key, `${obj[key]}`); - } - return queryParams.toString(); -}; - -/** - * parse string to query object - */ -export const parseQueryString = (str: string) => { - const queryObject: Record = {}; - - const splitStr = str.split('?'); - - str = splitStr[1] || splitStr[0]; - - // 将字符串按照 '&' 分割成键值对数组 - const keyValuePairs = str.split('&'); - - // 遍历键值对数组，将每个键值对解析为对象的属性和值 - keyValuePairs.forEach(function (keyValuePair) { - const pair = keyValuePair.split('='); - const key = decodeURIComponent(pair[0]); - const value = decodeURIComponent(pair[1] || ''); - - // 如果对象中已经存在该属性，则将值转换为数组 - if (queryObject.hasOwnProperty(key)) { - if (!Array.isArray(queryObject[key])) { - queryObject[key] = [queryObject[key]]; - } - queryObject[key].push(value); - } else { - queryObject[key] = value; - } - }); - - return queryObject; -}; - -/** - * 格式化时间成聊天格式 - */ -export const formatTimeToChatTime = (time: Date) => { - const now = dayjs(); - const target = dayjs(time); - - // 如果传入时间小于60秒，返回刚刚 - if (now.diff(target, 'second') < 60) { - return '刚刚'; - } - - // 如果时间是今天，展示几时:几分 - if (now.isSame(target, 'day')) { - return target.format('HH:mm'); - } - - // 如果是昨天，展示昨天 - if (now.subtract(1, 'day').isSame(target, 'day')) { - return '昨天'; - } - - // 如果是前天，展示前天 - if (now.subtract(2, 'day').isSame(target, 'day')) { - return '前天'; - } - - // 如果是今年，展示某月某日 - if (now.isSame(target, 'year')) { - return target.format('MM/DD'); - } - - // 如果是更久之前，展示某年某月某日 - return target.format('YYYY/M/D'); -}; diff --git a/projects/app/src/web/common/system/constants.ts b/projects/app/src/web/common/system/constants.ts new file mode 100644 index 000000000000..c381db9d90b2 --- /dev/null +++ b/projects/app/src/web/common/system/constants.ts @@ -0,0 +1,5 @@ +export enum TrackEventName { + windowError = 'windowError', + pageError = 'pageError', + wordReadError = 'wordReadError' +} diff --git a/projects/app/src/web/common/utils/voice.ts b/projects/app/src/web/common/utils/voice.ts index 6c1589b645ce..7df6a0244e4b 100644 --- a/projects/app/src/web/common/utils/voice.ts +++ b/projects/app/src/web/common/utils/voice.ts @@ -2,7 +2,7 @@ import { useState, useCallback, useEffect, useMemo, useRef } from 'react'; import { useToast } from '@fastgpt/web/hooks/useToast'; import { getErrText } from '@fastgpt/global/common/error/utils'; import type { AppTTSConfigType } from '@fastgpt/global/core/app/type.d'; -import { TTSTypeEnum } from '@/constants/app'; +import { TTSTypeEnum } from '@/web/core/app/constants'; import { useTranslation } from 'next-i18next'; import type { OutLinkChatAuthProps } from '@fastgpt/global/support/permission/chat.d'; import { getToken } from '@/web/support/user/auth'; diff --git a/projects/app/src/web/context/useInitApp.ts b/projects/app/src/web/context/useInitApp.ts index bfab4fee028d..309ec24ff035 100644 --- a/projects/app/src/web/context/useInitApp.ts +++ b/projects/app/src/web/context/useInitApp.ts @@ -6,6 +6,7 @@ import { useSystemStore } from '@/web/common/system/useSystemStore'; import type { FastGPTFeConfigsType } from '@fastgpt/global/common/system/types/index.d'; import { change2DefaultLng, setLngStore } from '@/web/common/utils/i18n'; import { useMemoizedFn, useMount } from 'ahooks'; +import { TrackEventName } from '../common/system/constants'; export const useInitApp = () => { const router = useRouter(); @@ -52,7 +53,7 @@ export const useInitApp = () => { initUserLanguage(); const errorTrack = (event: ErrorEvent) => { - window.umami?.track('windowError', { + window.umami?.track(TrackEventName.windowError, { device: { userAgent: navigator.userAgent, platform: navigator.platform, diff --git a/projects/app/src/constants/app.ts b/projects/app/src/web/core/app/constants.ts similarity index 100% rename from projects/app/src/constants/app.ts rename to projects/app/src/web/core/app/constants.ts diff --git a/projects/app/src/web/core/app/store/useAppStore.ts b/projects/app/src/web/core/app/store/useAppStore.ts index 6e5a8acdac65..84e943a2292c 100644 --- a/projects/app/src/web/core/app/store/useAppStore.ts +++ b/projects/app/src/web/core/app/store/useAppStore.ts @@ -2,7 +2,7 @@ import { create } from 'zustand'; import { devtools, persist } from 'zustand/middleware'; import { immer } from 'zustand/middleware/immer'; import { getMyApps, getModelById, putAppById } from '@/web/core/app/api'; -import { defaultApp } from '@/constants/app'; +import { defaultApp } from '../constants'; import type { AppUpdateParams } from '@/global/core/app/api.d'; import { AppDetailType, AppListItemType } from '@fastgpt/global/core/app/type.d'; import { PostPublishAppProps } from '@/global/core/app/api'; diff --git a/projects/app/src/constants/common.ts b/projects/app/src/web/core/chat/constants.ts similarity index 99% rename from projects/app/src/constants/common.ts rename to projects/app/src/web/core/chat/constants.ts index 8ab1dba66d07..be9326aaae55 100644 --- a/projects/app/src/constants/common.ts +++ b/projects/app/src/web/core/chat/constants.ts @@ -1,9 +1,3 @@ -export enum TrackEventName { - windowError = 'windowError', - pageError = 'pageError', - wordReadError = 'wordReadError' -} - export const htmlTemplate = ` diff --git a/projects/app/src/web/core/dataset/constants.ts b/projects/app/src/web/core/dataset/constants.ts index 666d3b8b14e2..2658f24ed1fc 100644 --- a/projects/app/src/web/core/dataset/constants.ts +++ b/projects/app/src/web/core/dataset/constants.ts @@ -1,3 +1,60 @@ +import { defaultQAModels, defaultVectorModels } from '@fastgpt/global/core/ai/model'; +import type { + DatasetCollectionItemType, + DatasetItemType +} from '@fastgpt/global/core/dataset/type.d'; + +export const defaultDatasetDetail: DatasetItemType = { + _id: '', + parentId: '', + userId: '', + teamId: '', + tmbId: '', + updateTime: new Date(), + type: 'dataset', + avatar: '/icon/logo.svg', + name: '', + intro: '', + status: 'active', + permission: 'private', + isOwner: false, + canWrite: false, + vectorModel: defaultVectorModels[0], + agentModel: defaultQAModels[0] +}; + +export const defaultCollectionDetail: DatasetCollectionItemType = { + _id: '', + teamId: '', + tmbId: '', + datasetId: { + _id: '', + parentId: '', + userId: '', + teamId: '', + tmbId: '', + updateTime: new Date(), + type: 'dataset', + avatar: '/icon/logo.svg', + name: '', + intro: '', + status: 'active', + permission: 'private', + vectorModel: defaultVectorModels[0].model, + agentModel: defaultQAModels[0].model + }, + parentId: '', + name: '', + type: 'file', + updateTime: new Date(), + canWrite: false, + sourceName: '', + sourceId: '', + createTime: new Date(), + trainingType: 'chunk', + chunkSize: 0 +}; + export enum ImportProcessWayEnum { auto = 'auto', custom = 'custom' diff --git a/projects/app/src/web/core/dataset/store/dataset.ts b/projects/app/src/web/core/dataset/store/dataset.ts index 4f458fe9aa61..d849ad230bcf 100644 --- a/projects/app/src/web/core/dataset/store/dataset.ts +++ b/projects/app/src/web/core/dataset/store/dataset.ts @@ -9,7 +9,7 @@ import { putDatasetById, postWebsiteSync } from '@/web/core/dataset/api'; -import { defaultDatasetDetail } from '@/constants/dataset'; +import { defaultDatasetDetail } from '../constants'; import type { DatasetUpdateBody } from '@fastgpt/global/core/dataset/api.d'; import { DatasetStatusEnum } from '@fastgpt/global/core/dataset/constants'; import { postCreateTrainingUsage } from '@/web/support/wallet/usage/api'; diff --git a/projects/app/src/constants/user.ts b/projects/app/src/web/support/user/login/constants.ts similarity index 68% rename from projects/app/src/constants/user.ts rename to projects/app/src/web/support/user/login/constants.ts index a10c1d1ad8ea..ca24fc4df6f3 100644 --- a/projects/app/src/constants/user.ts +++ b/projects/app/src/web/support/user/login/constants.ts @@ -4,8 +4,3 @@ export enum LoginPageTypeEnum { forgetPassword = 'forgetPassword', wechat = 'wechat' } - -export enum PromotionEnum { - register = 'register', - pay = 'pay' -} From 8d2230f24f8dc49446412dc6703d80b16edc185f Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Mon, 13 May 2024 17:08:37 +0800 Subject: [PATCH 02/27] Update intro.md --- docSite/content/docs/development/intro.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docSite/content/docs/development/intro.md b/docSite/content/docs/development/intro.md index b0ff877b1de2..e44c8aa58f04 100644 --- a/docSite/content/docs/development/intro.md +++ b/docSite/content/docs/development/intro.md @@ -141,7 +141,7 @@ FastGPT 在`pnpm i`后会执行`postinstall`脚本，用于自动生成`ChakraUI FastGPT 使用了 nextjs 的 page route 作为框架。为了区分好前后端代码，在目录分配上会分成 global, service, web 3个自目录，分别对应着 `前后端共用`、`后端专用`、`前端专用`的代码。 ### monorepo -FastGPT 才用 pnpm workspace 方式构建 monorepo 项目，主要分为两个部分： +FastGPT 采用 pnpm workspace 方式构建 monorepo 项目，主要分为两个部分： - projects/app - FastGPT 主项目 - packages/ - 子模块 From 240f60c0ca55e6609152b8b7e6ac1cde3145f04d Mon Sep 17 00:00:00 2001 From: Fengrui Liu Date: Tue, 14 May 2024 00:17:44 +0800 Subject: [PATCH 03/27] Fixes: fix edge handler with onDelEdge (#1471) * fixes: Fix edge handler * fixes: fix edge handler with onDelEdge * fixes: fix edge handler with onDelEdge --- projects/app/src/components/core/workflow/context.tsx | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/projects/app/src/components/core/workflow/context.tsx b/projects/app/src/components/core/workflow/context.tsx index 7e9f0d6efc4c..a2ffd6d81bf5 100644 --- a/projects/app/src/components/core/workflow/context.tsx +++ b/projects/app/src/components/core/workflow/context.tsx @@ -36,6 +36,7 @@ import { createContext } from 'use-context-selector'; import { defaultRunningStatus } from './constants'; import { checkNodeRunStatus } from '@fastgpt/global/core/workflow/runtime/utils'; import { EventNameEnum, eventBus } from '@/web/common/utils/eventbus'; +import { getHandleId } from '@fastgpt/global/core/workflow/utils'; type OnChange = (changes: ChangesType[]) => void; @@ -321,7 +322,7 @@ const WorkflowContextProvider = ({ item.key === props.key ? props.value : item ); } else if (type === 'replaceInput') { - onDelEdge({ nodeId, targetHandle: props.key }); + onDelEdge({ nodeId, targetHandle: getHandleId(nodeId, 'target', props.key) }); const oldInputIndex = node.data.inputs.findIndex((item) => item.key === props.key); updateObj.inputs = node.data.inputs.filter((item) => item.key !== props.key); setTimeout(() => { @@ -350,14 +351,14 @@ const WorkflowContextProvider = ({ } } } else if (type === 'delInput') { - onDelEdge({ nodeId, targetHandle: props.key }); + onDelEdge({ nodeId, targetHandle: getHandleId(nodeId, 'target', props.key) }); updateObj.inputs = node.data.inputs.filter((item) => item.key !== props.key); } else if (type === 'updateOutput') { updateObj.outputs = node.data.outputs.map((item) => item.key === props.key ? props.value : item ); } else if (type === 'replaceOutput') { - onDelEdge({ nodeId, sourceHandle: props.key }); + onDelEdge({ nodeId, sourceHandle: getHandleId(nodeId, 'source', props.key) }); const oldOutputIndex = node.data.outputs.findIndex((item) => item.key === props.key); updateObj.outputs = node.data.outputs.filter((item) => item.key !== props.key); console.log(props.value); @@ -387,7 +388,7 @@ const WorkflowContextProvider = ({ } } } else if (type === 'delOutput') { - onDelEdge({ nodeId, sourceHandle: props.key }); + onDelEdge({ nodeId, sourceHandle: getHandleId(nodeId, 'source', props.key) }); updateObj.outputs = node.data.outputs.filter((item) => item.key !== props.key); } From b779e2806dcac89bd2ac968e646d804b40366e70 Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Tue, 14 May 2024 12:58:04 +0800 Subject: [PATCH 04/27] fix doc (#1475) --- docSite/content/docs/development/migration/ docker_mongo.md | 3 +-- docSite/content/docs/development/upgrading/481.md | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docSite/content/docs/development/migration/ docker_mongo.md b/docSite/content/docs/development/migration/ docker_mongo.md index 2354b6315304..50cc6d24d899 100644 --- a/docSite/content/docs/development/migration/ docker_mongo.md +++ b/docSite/content/docs/development/migration/ docker_mongo.md @@ -1,10 +1,9 @@ --- -weight: 762 title: "Docker Mongo迁移(dump模式)" description: "FastGPT Docker Mongo迁移" icon: database draft: false -images: [] +weight: 762 --- ## 作者 diff --git a/docSite/content/docs/development/upgrading/481.md b/docSite/content/docs/development/upgrading/481.md index 26dfea4d6b70..1fa86b622772 100644 --- a/docSite/content/docs/development/upgrading/481.md +++ b/docSite/content/docs/development/upgrading/481.md @@ -4,7 +4,7 @@ description: 'FastGPT V4.8.1 更新说明' icon: 'upgrade' draft: false toc: true -weight: 825 +weight: 823 --- ## 初始化脚本 From fb04889a31a9cb1732281bd3380a2199b319d2b8 Mon Sep 17 00:00:00 2001 From: heheer <71265218+newfish-cmyk@users.noreply.github.com> Date: Tue, 14 May 2024 23:26:03 +0800 Subject: [PATCH 05/27] feat: node version (#1484) * feat: node version tip * fix * i18n * init version * fix ts * fix ts * fix ts --- packages/global/core/app/utils.ts | 1 + .../global/core/plugin/httpPlugin/utils.ts | 3 + .../core/workflow/template/system/aiChat.ts | 1 + .../template/system/assignedAnswer.ts | 1 + .../template/system/classifyQuestion.ts | 1 + .../template/system/contextExtract.ts | 1 + .../workflow/template/system/datasetConcat.ts | 1 + .../workflow/template/system/datasetSearch.ts | 1 + .../workflow/template/system/emptyNode.ts | 1 + .../template/system/globalVariable.ts | 1 + .../core/workflow/template/system/http468.ts | 1 + .../workflow/template/system/ifElse/index.ts | 1 + .../core/workflow/template/system/laf.ts | 1 + .../workflow/template/system/pluginInput.ts | 1 + .../workflow/template/system/pluginOutput.ts | 1 + .../template/system/queryExtension.ts | 1 + .../core/workflow/template/system/runApp.ts | 1 + .../workflow/template/system/runPlugin.ts | 1 + .../core/workflow/template/system/stopTool.ts | 1 + .../workflow/template/system/systemConfig.ts | 1 + .../core/workflow/template/system/tools.ts | 1 + .../template/system/variableUpdate/index.tsx | 1 + .../workflow/template/system/workflowStart.ts | 1 + packages/global/core/workflow/type/index.d.ts | 1 + packages/service/core/plugin/controller.ts | 1 + packages/web/styles/theme.ts | 13 +++ projects/app/i18n/en/app.json | 6 ++ projects/app/i18n/zh/app.json | 6 ++ .../workflow/Flow/nodes/render/NodeCard.tsx | 82 +++++++++++++++---- .../app/src/components/core/workflow/utils.ts | 1 + .../getSystemPluginTemplates.ts | 1 + .../pluginTemplate/getTeamPluginTemplates.ts | 1 + .../pages/plugin/list/component/EditModal.tsx | 2 + projects/app/src/web/core/app/templates.ts | 16 ++++ projects/app/src/web/core/app/utils.ts | 8 ++ projects/app/src/web/core/workflow/adapt.ts | 2 + projects/app/src/web/core/workflow/utils.ts | 40 ++++++++- 37 files changed, 185 insertions(+), 20 deletions(-) diff --git a/packages/global/core/app/utils.ts b/packages/global/core/app/utils.ts index 679282528e07..44a00a4bc82d 100644 --- a/packages/global/core/app/utils.ts +++ b/packages/global/core/app/utils.ts @@ -131,6 +131,7 @@ export const appWorkflow2Form = ({ nodes }: { nodes: StoreNodeItemType[] }) => { intro: node.intro || '', flowNodeType: node.flowNodeType, showStatus: node.showStatus, + version: '481', inputs: node.inputs, outputs: node.outputs, templateType: FlowNodeTemplateTypeEnum.other diff --git a/packages/global/core/plugin/httpPlugin/utils.ts b/packages/global/core/plugin/httpPlugin/utils.ts index 5eb9ea05a3f0..c2121ea3e4b2 100644 --- a/packages/global/core/plugin/httpPlugin/utils.ts +++ b/packages/global/core/plugin/httpPlugin/utils.ts @@ -282,6 +282,7 @@ export const httpApiSchema2Plugins = async ({ x: 616.4226348688949, y: -165.05298493910115 }, + version: PluginInputModule.version, inputs: pluginInputs, outputs: pluginOutputs }, @@ -296,6 +297,7 @@ export const httpApiSchema2Plugins = async ({ x: 1607.7142331269126, y: -151.8669210746189 }, + version: PluginOutputModule.version, inputs: [ { key: pluginOutputKey, @@ -334,6 +336,7 @@ export const httpApiSchema2Plugins = async ({ x: 1042.549746602742, y: -447.77496332641647 }, + version: HttpModule468.version, inputs: [ { key: NodeInputKeyEnum.addInputParam, diff --git a/packages/global/core/workflow/template/system/aiChat.ts b/packages/global/core/workflow/template/system/aiChat.ts index f00c97b90564..8f3feeebac53 100644 --- a/packages/global/core/workflow/template/system/aiChat.ts +++ b/packages/global/core/workflow/template/system/aiChat.ts @@ -31,6 +31,7 @@ export const AiChatModule: FlowNodeTemplateType = { intro: 'AI 大模型对话', showStatus: true, isTool: true, + version: '481', inputs: [ Input_Template_SettingAiModel, // --- settings modal diff --git a/packages/global/core/workflow/template/system/assignedAnswer.ts b/packages/global/core/workflow/template/system/assignedAnswer.ts index 5ccb38ab3ec5..6f133b5571ac 100644 --- a/packages/global/core/workflow/template/system/assignedAnswer.ts +++ b/packages/global/core/workflow/template/system/assignedAnswer.ts @@ -17,6 +17,7 @@ export const AssignedAnswerModule: FlowNodeTemplateType = { name: '指定回复', intro: '该模块可以直接回复一段指定的内容。常用于引导、提示。非字符串内容传入时，会转成字符串进行输出。', + version: '481', inputs: [ { key: NodeInputKeyEnum.answerText, diff --git a/packages/global/core/workflow/template/system/classifyQuestion.ts b/packages/global/core/workflow/template/system/classifyQuestion.ts index c2afe4a4653b..f8325d7337fe 100644 --- a/packages/global/core/workflow/template/system/classifyQuestion.ts +++ b/packages/global/core/workflow/template/system/classifyQuestion.ts @@ -29,6 +29,7 @@ export const ClassifyQuestionModule: FlowNodeTemplateType = { name: '问题分类', intro: `根据用户的历史记录和当前问题判断该次提问的类型。可以添加多组问题类型，下面是一个模板例子：\n类型1: 打招呼\n类型2: 关于商品“使用”问题\n类型3: 关于商品“购买”问题\n类型4: 其他问题`, showStatus: true, + version: '481', inputs: [ { ...Input_Template_SelectAIModel, diff --git a/packages/global/core/workflow/template/system/contextExtract.ts b/packages/global/core/workflow/template/system/contextExtract.ts index c34697ba7131..d34608b1857e 100644 --- a/packages/global/core/workflow/template/system/contextExtract.ts +++ b/packages/global/core/workflow/template/system/contextExtract.ts @@ -25,6 +25,7 @@ export const ContextExtractModule: FlowNodeTemplateType = { intro: '可从文本中提取指定的数据，例如：sql语句、搜索关键词、代码等', showStatus: true, isTool: true, + version: '481', inputs: [ { ...Input_Template_SelectAIModel, diff --git a/packages/global/core/workflow/template/system/datasetConcat.ts b/packages/global/core/workflow/template/system/datasetConcat.ts index 60b4d1779968..ee2576e6f1ac 100644 --- a/packages/global/core/workflow/template/system/datasetConcat.ts +++ b/packages/global/core/workflow/template/system/datasetConcat.ts @@ -42,6 +42,7 @@ export const DatasetConcatModule: FlowNodeTemplateType = { name: '知识库搜索引用合并', intro: '可以将多个知识库搜索结果进行合并输出。使用 RRF 的合并方式进行最终排序输出。', showStatus: false, + version: '481', inputs: [ { key: NodeInputKeyEnum.datasetMaxTokens, diff --git a/packages/global/core/workflow/template/system/datasetSearch.ts b/packages/global/core/workflow/template/system/datasetSearch.ts index 2808ca6f9319..b8cf918e2ba8 100644 --- a/packages/global/core/workflow/template/system/datasetSearch.ts +++ b/packages/global/core/workflow/template/system/datasetSearch.ts @@ -28,6 +28,7 @@ export const DatasetSearchModule: FlowNodeTemplateType = { intro: Dataset_SEARCH_DESC, showStatus: true, isTool: true, + version: '481', inputs: [ { key: NodeInputKeyEnum.datasetSelectList, diff --git a/packages/global/core/workflow/template/system/emptyNode.ts b/packages/global/core/workflow/template/system/emptyNode.ts index c26d1b1d2a24..0b58eb305a0e 100644 --- a/packages/global/core/workflow/template/system/emptyNode.ts +++ b/packages/global/core/workflow/template/system/emptyNode.ts @@ -12,6 +12,7 @@ export const EmptyNode: FlowNodeTemplateType = { avatar: '', name: '', intro: '', + version: '481', inputs: [], outputs: [] }; diff --git a/packages/global/core/workflow/template/system/globalVariable.ts b/packages/global/core/workflow/template/system/globalVariable.ts index 27f7de779cc0..59053487710e 100644 --- a/packages/global/core/workflow/template/system/globalVariable.ts +++ b/packages/global/core/workflow/template/system/globalVariable.ts @@ -20,6 +20,7 @@ export const getGlobalVariableNode = ({ avatar: '/imgs/workflow/variable.png', name: '全局变量', intro: '', + version: '481', inputs: [], outputs: variables.map((item) => ({ id: item.key, diff --git a/packages/global/core/workflow/template/system/http468.ts b/packages/global/core/workflow/template/system/http468.ts index 515c15da2b39..26d49701b137 100644 --- a/packages/global/core/workflow/template/system/http468.ts +++ b/packages/global/core/workflow/template/system/http468.ts @@ -25,6 +25,7 @@ export const HttpModule468: FlowNodeTemplateType = { intro: '可以发出一个 HTTP 请求，实现更为复杂的操作（联网搜索、数据库查询等）', showStatus: true, isTool: true, + version: '481', inputs: [ { ...Input_Template_DynamicInput, diff --git a/packages/global/core/workflow/template/system/ifElse/index.ts b/packages/global/core/workflow/template/system/ifElse/index.ts index 0a244d6622ec..314e925b4166 100644 --- a/packages/global/core/workflow/template/system/ifElse/index.ts +++ b/packages/global/core/workflow/template/system/ifElse/index.ts @@ -22,6 +22,7 @@ export const IfElseNode: FlowNodeTemplateType = { name: '判断器', intro: '根据一定的条件，执行不同的分支。', showStatus: true, + version: '481', inputs: [ { key: NodeInputKeyEnum.ifElseList, diff --git a/packages/global/core/workflow/template/system/laf.ts b/packages/global/core/workflow/template/system/laf.ts index 2140432bb7fb..154533ef7b0f 100644 --- a/packages/global/core/workflow/template/system/laf.ts +++ b/packages/global/core/workflow/template/system/laf.ts @@ -25,6 +25,7 @@ export const LafModule: FlowNodeTemplateType = { intro: '可以调用Laf账号下的云函数。', showStatus: true, isTool: true, + version: '481', inputs: [ { ...Input_Template_DynamicInput, diff --git a/packages/global/core/workflow/template/system/pluginInput.ts b/packages/global/core/workflow/template/system/pluginInput.ts index 8e0d83f6b9d1..ab3b42ad0e20 100644 --- a/packages/global/core/workflow/template/system/pluginInput.ts +++ b/packages/global/core/workflow/template/system/pluginInput.ts @@ -15,6 +15,7 @@ export const PluginInputModule: FlowNodeTemplateType = { name: '自定义插件输入', intro: '自定义配置外部输入，使用插件时，仅暴露自定义配置的输入', showStatus: false, + version: '481', inputs: [], outputs: [] }; diff --git a/packages/global/core/workflow/template/system/pluginOutput.ts b/packages/global/core/workflow/template/system/pluginOutput.ts index 9b709a3b0156..2521e582d91d 100644 --- a/packages/global/core/workflow/template/system/pluginOutput.ts +++ b/packages/global/core/workflow/template/system/pluginOutput.ts @@ -15,6 +15,7 @@ export const PluginOutputModule: FlowNodeTemplateType = { name: '自定义插件输出', intro: '自定义配置外部输出，使用插件时，仅暴露自定义配置的输出', showStatus: false, + version: '481', inputs: [], outputs: [] }; diff --git a/packages/global/core/workflow/template/system/queryExtension.ts b/packages/global/core/workflow/template/system/queryExtension.ts index 6ee6689da2d5..fa23d5a2282f 100644 --- a/packages/global/core/workflow/template/system/queryExtension.ts +++ b/packages/global/core/workflow/template/system/queryExtension.ts @@ -29,6 +29,7 @@ export const AiQueryExtension: FlowNodeTemplateType = { intro: '使用问题优化功能，可以提高知识库连续对话时搜索的精度。使用该功能后，会先利用 AI 根据上下文构建一个或多个新的检索词，这些检索词更利于进行知识库搜索。该模块已内置在知识库搜索模块中，如果您仅进行一次知识库搜索，可直接使用知识库内置的补全功能。', showStatus: true, + version: '481', inputs: [ { ...Input_Template_SelectAIModel, diff --git a/packages/global/core/workflow/template/system/runApp.ts b/packages/global/core/workflow/template/system/runApp.ts index 504ca3f0e274..0d39310a3af0 100644 --- a/packages/global/core/workflow/template/system/runApp.ts +++ b/packages/global/core/workflow/template/system/runApp.ts @@ -23,6 +23,7 @@ export const RunAppModule: FlowNodeTemplateType = { name: '应用调用', intro: '可以选择一个其他应用进行调用', showStatus: true, + version: '481', inputs: [ { key: NodeInputKeyEnum.runAppSelectApp, diff --git a/packages/global/core/workflow/template/system/runPlugin.ts b/packages/global/core/workflow/template/system/runPlugin.ts index 419d6e06164b..43b9045719d7 100644 --- a/packages/global/core/workflow/template/system/runPlugin.ts +++ b/packages/global/core/workflow/template/system/runPlugin.ts @@ -13,6 +13,7 @@ export const RunPluginModule: FlowNodeTemplateType = { name: '', showStatus: false, isTool: true, + version: '481', inputs: [], // [{key:'pluginId'},...] outputs: [] }; diff --git a/packages/global/core/workflow/template/system/stopTool.ts b/packages/global/core/workflow/template/system/stopTool.ts index 713c653fc4cd..6881bd53b5e6 100644 --- a/packages/global/core/workflow/template/system/stopTool.ts +++ b/packages/global/core/workflow/template/system/stopTool.ts @@ -13,6 +13,7 @@ export const StopToolNode: FlowNodeTemplateType = { name: '工具调用终止', intro: '该模块需配置工具调用使用。当该模块被执行时，本次工具调用将会强制结束，并且不再调用AI针对工具调用结果回答问题。', + version: '481', inputs: [], outputs: [] }; diff --git a/packages/global/core/workflow/template/system/systemConfig.ts b/packages/global/core/workflow/template/system/systemConfig.ts index 69a8044bfee9..cb7aa5d11a54 100644 --- a/packages/global/core/workflow/template/system/systemConfig.ts +++ b/packages/global/core/workflow/template/system/systemConfig.ts @@ -18,6 +18,7 @@ export const SystemConfigNode: FlowNodeTemplateType = { intro: '可以配置应用的系统参数。', unique: true, forbidDelete: true, + version: '481', inputs: [ { key: NodeInputKeyEnum.welcomeText, diff --git a/packages/global/core/workflow/template/system/tools.ts b/packages/global/core/workflow/template/system/tools.ts index 8c0050df3304..4c9e15b50688 100644 --- a/packages/global/core/workflow/template/system/tools.ts +++ b/packages/global/core/workflow/template/system/tools.ts @@ -30,6 +30,7 @@ export const ToolModule: FlowNodeTemplateType = { name: '工具调用(实验)', intro: '通过AI模型自动选择一个或多个功能块进行调用，也可以对插件进行调用。', showStatus: true, + version: '481', inputs: [ { ...Input_Template_SettingAiModel, diff --git a/packages/global/core/workflow/template/system/variableUpdate/index.tsx b/packages/global/core/workflow/template/system/variableUpdate/index.tsx index 6867d98bba3b..56771cd09861 100644 --- a/packages/global/core/workflow/template/system/variableUpdate/index.tsx +++ b/packages/global/core/workflow/template/system/variableUpdate/index.tsx @@ -18,6 +18,7 @@ export const VariableUpdateNode: FlowNodeTemplateType = { intro: '可以更新指定节点的输出值或更新全局变量', showStatus: true, isTool: false, + version: '481', inputs: [ { key: NodeInputKeyEnum.updateList, diff --git a/packages/global/core/workflow/template/system/workflowStart.ts b/packages/global/core/workflow/template/system/workflowStart.ts index f43b7839c4cb..ceb654b4907a 100644 --- a/packages/global/core/workflow/template/system/workflowStart.ts +++ b/packages/global/core/workflow/template/system/workflowStart.ts @@ -19,6 +19,7 @@ export const WorkflowStart: FlowNodeTemplateType = { intro: '', forbidDelete: true, unique: true, + version: '481', inputs: [{ ...Input_Template_UserChatInput, toolDescription: '用户问题' }], outputs: [ { diff --git a/packages/global/core/workflow/type/index.d.ts b/packages/global/core/workflow/type/index.d.ts index 5491454651f8..f24fd07ebd81 100644 --- a/packages/global/core/workflow/type/index.d.ts +++ b/packages/global/core/workflow/type/index.d.ts @@ -28,6 +28,7 @@ export type FlowNodeCommonType = { name: string; intro?: string; // template list intro showStatus?: boolean; // chatting response step status + version: string; // data inputs: FlowNodeInputItemType[]; diff --git a/packages/service/core/plugin/controller.ts b/packages/service/core/plugin/controller.ts index 535d8eaef478..a2718cd35554 100644 --- a/packages/service/core/plugin/controller.ts +++ b/packages/service/core/plugin/controller.ts @@ -72,6 +72,7 @@ export async function getPluginPreviewNode({ id }: { id: string }): Promise { const setHoverNodeId = useContextSelector(WorkflowContext, (v) => v.setHoverNodeId); const onUpdateNodeError = useContextSelector(WorkflowContext, (v) => v.onUpdateNodeError); const onChangeNode = useContextSelector(WorkflowContext, (v) => v.onChangeNode); + const onResetNode = useContextSelector(WorkflowContext, (v) => v.onResetNode); // custom title edit const { onOpenModal: onOpenCustomTitleModal, EditModal: EditTitleModal } = useEditTitle({ @@ -77,12 +81,21 @@ const NodeCard = (props: Props) => { [isTool, nodeList] ); + const node = nodeList.find((node) => node.nodeId === nodeId); + const template = moduleTemplatesFlat.find((item) => item.flowNodeType === node?.flowNodeType); + const hasNewVersion = useMemo(() => { + return ( + template?.flowNodeType !== FlowNodeTypeEnum.pluginModule && + node?.version !== template?.version + ); + }, [node?.version, template?.flowNodeType, template?.version]); + /* Node header */ const Header = useMemo(() => { return ( {/* debug */} - + {/* tool target handle */} {showToolHandle && } @@ -123,6 +136,33 @@ const NodeCard = (props: Props) => { }} /> )} + + {hasNewVersion && ( + + + + )} { t, name, menuForbid, + hasNewVersion, pluginId, flowNodeType, intro, onOpenCustomTitleModal, onChangeNode, toast, - appT + appT, + node, + template, + onResetNode ]); return ( @@ -236,7 +280,8 @@ const MenuRender = React.memo(function MenuRender({ inputs: node.data.inputs, outputs: node.data.outputs, showStatus: node.data.showStatus, - pluginId: node.data.pluginId + pluginId: node.data.pluginId, + version: node.data.version }; return state.concat( storeNode2FlowNode({ @@ -250,7 +295,8 @@ const MenuRender = React.memo(function MenuRender({ showStatus: template.showStatus, pluginId: template.pluginId, inputs: template.inputs, - outputs: template.outputs + outputs: template.outputs, + version: template.version } }) ); @@ -265,7 +311,7 @@ const MenuRender = React.memo(function MenuRender({ }, [setEdges, setNodes] ); - const onclickSyncVersion = useCallback(async () => { + const onClickSyncVersion = useCallback(async () => { if (!pluginId) return; try { setLoading(true); @@ -310,7 +356,7 @@ const MenuRender = React.memo(function MenuRender({ icon: 'common/refreshLight', label: t('plugin.Synchronous version'), variant: 'whiteBase', - onClick: onOpenConfirmSync(onclickSyncVersion) + onClick: onOpenConfirmSync(onClickSyncVersion) } ] : []), @@ -362,21 +408,21 @@ const MenuRender = React.memo(function MenuRender({ ); }, [ - ConfirmDeleteModal, - ConfirmSyncModal, - DebugInputModal, - flowNodeType, - menuForbid?.copy, menuForbid?.debug, + menuForbid?.copy, menuForbid?.delete, - nodeId, - onCopyNode, - onDelNode, - onOpenConfirmDeleteNode, + t, + flowNodeType, onOpenConfirmSync, - onclickSyncVersion, + onClickSyncVersion, + onOpenConfirmDeleteNode, + ConfirmSyncModal, + ConfirmDeleteModal, + DebugInputModal, openDebugNode, - t + nodeId, + onCopyNode, + onDelNode ]); return Render; diff --git a/projects/app/src/components/core/workflow/utils.ts b/projects/app/src/components/core/workflow/utils.ts index 298b41871763..232ec5beb53a 100644 --- a/projects/app/src/components/core/workflow/utils.ts +++ b/projects/app/src/components/core/workflow/utils.ts @@ -19,6 +19,7 @@ export const flowNode2StoreNodes = ({ flowNodeType: item.data.flowNodeType, showStatus: item.data.showStatus, position: item.position, + version: item.data.version, inputs: item.data.inputs, outputs: item.data.outputs, pluginId: item.data.pluginId diff --git a/projects/app/src/pages/api/core/plugin/pluginTemplate/getSystemPluginTemplates.ts b/projects/app/src/pages/api/core/plugin/pluginTemplate/getSystemPluginTemplates.ts index ef3a64c69864..9f56f59fa1b2 100644 --- a/projects/app/src/pages/api/core/plugin/pluginTemplate/getSystemPluginTemplates.ts +++ b/projects/app/src/pages/api/core/plugin/pluginTemplate/getSystemPluginTemplates.ts @@ -22,6 +22,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< intro: plugin.intro, showStatus: true, isTool: plugin.isTool, + version: '481', inputs: [], outputs: [] })) || []; diff --git a/projects/app/src/pages/api/core/plugin/pluginTemplate/getTeamPluginTemplates.ts b/projects/app/src/pages/api/core/plugin/pluginTemplate/getTeamPluginTemplates.ts index c153e1dd11bd..319d5c936763 100644 --- a/projects/app/src/pages/api/core/plugin/pluginTemplate/getTeamPluginTemplates.ts +++ b/projects/app/src/pages/api/core/plugin/pluginTemplate/getTeamPluginTemplates.ts @@ -47,6 +47,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< name: plugin.name, intro: plugin.intro, showStatus: false, + version: '481', inputs: [], outputs: [] })); diff --git a/projects/app/src/pages/plugin/list/component/EditModal.tsx b/projects/app/src/pages/plugin/list/component/EditModal.tsx index 01eb6dd9c61e..ccfa5ec8259e 100644 --- a/projects/app/src/pages/plugin/list/component/EditModal.tsx +++ b/projects/app/src/pages/plugin/list/component/EditModal.tsx @@ -39,6 +39,7 @@ export const defaultForm: EditFormType = { x: 616.4226348688949, y: -165.05298493910115 }, + version: '481', inputs: [], outputs: [] }, @@ -52,6 +53,7 @@ export const defaultForm: EditFormType = { x: 1607.7142331269126, y: -151.8669210746189 }, + version: '481', inputs: [], outputs: [] } diff --git a/projects/app/src/web/core/app/templates.ts b/projects/app/src/web/core/app/templates.ts index 87365fd03618..2b1749759469 100644 --- a/projects/app/src/web/core/app/templates.ts +++ b/projects/app/src/web/core/app/templates.ts @@ -26,6 +26,7 @@ export const appTemplates: (AppItemType & { x: 531.2422736065552, y: -486.7611729549753 }, + version: '481', inputs: [ { key: 'welcomeText', @@ -88,6 +89,7 @@ export const appTemplates: (AppItemType & { x: 558.4082376415505, y: 123.72387429194112 }, + version: '481', inputs: [ { key: 'userChatInput', @@ -119,6 +121,7 @@ export const appTemplates: (AppItemType & { x: 1097.7317280958762, y: -244.16014496351386 }, + version: '481', inputs: [ { key: 'model', @@ -253,6 +256,7 @@ export const appTemplates: (AppItemType & { x: 496.57560693988853, y: -490.7611729549753 }, + version: '481', inputs: [ { key: 'welcomeText', @@ -332,6 +336,7 @@ export const appTemplates: (AppItemType & { x: 558.4082376415505, y: 123.72387429194112 }, + version: '481', inputs: [ { key: 'userChatInput', @@ -363,6 +368,7 @@ export const appTemplates: (AppItemType & { x: 1097.7317280958762, y: -244.16014496351386 }, + version: '481', inputs: [ { key: 'model', @@ -497,6 +503,7 @@ export const appTemplates: (AppItemType & { x: 531.2422736065552, y: -486.7611729549753 }, + version: '481', inputs: [ { key: 'welcomeText', @@ -559,6 +566,7 @@ export const appTemplates: (AppItemType & { x: 558.4082376415505, y: 123.72387429194112 }, + version: '481', inputs: [ { key: 'userChatInput', @@ -590,6 +598,7 @@ export const appTemplates: (AppItemType & { x: 1638.509551404687, y: -341.0428450861567 }, + version: '481', inputs: [ { key: 'model', @@ -709,6 +718,7 @@ export const appTemplates: (AppItemType & { x: 918.5901682164496, y: -227.11542247619582 }, + version: '481', inputs: [ { key: 'datasets', @@ -821,6 +831,7 @@ export const appTemplates: (AppItemType & { x: 531.2422736065552, y: -486.7611729549753 }, + version: '481', inputs: [ { key: 'welcomeText', @@ -883,6 +894,7 @@ export const appTemplates: (AppItemType & { x: 558.4082376415505, y: 123.72387429194112 }, + version: '481', inputs: [ { key: 'userChatInput', @@ -914,6 +926,7 @@ export const appTemplates: (AppItemType & { x: 2701.1267277679685, y: -767.8956312653042 }, + version: '481', inputs: [ { key: 'model', @@ -1034,6 +1047,7 @@ export const appTemplates: (AppItemType & { x: 1020.9667229609946, y: -385.0060974413916 }, + version: '481', inputs: [ { key: 'model', @@ -1114,6 +1128,7 @@ export const appTemplates: (AppItemType & { x: 1874.9167551056487, y: 434.98431875888207 }, + version: '481', inputs: [ { key: 'text', @@ -1139,6 +1154,7 @@ export const appTemplates: (AppItemType & { x: 1851.010152279949, y: -613.3555232387284 }, + version: '481', inputs: [ { key: 'datasets', diff --git a/projects/app/src/web/core/app/utils.ts b/projects/app/src/web/core/app/utils.ts index 494662a04e5d..f2e1fa1a4f32 100644 --- a/projects/app/src/web/core/app/utils.ts +++ b/projects/app/src/web/core/app/utils.ts @@ -27,6 +27,7 @@ export function form2AppWorkflow(data: AppSimpleEditFormType): WorkflowType { x: 531.2422736065552, y: -486.7611729549753 }, + version: '481', inputs: [ { key: NodeInputKeyEnum.welcomeText, @@ -79,6 +80,7 @@ export function form2AppWorkflow(data: AppSimpleEditFormType): WorkflowType { x: 558.4082376415505, y: 123.72387429194112 }, + version: '481', inputs: [ { key: 'userChatInput', @@ -115,6 +117,7 @@ export function form2AppWorkflow(data: AppSimpleEditFormType): WorkflowType { x: 1106.3238387960757, y: -350.6030674683474 }, + version: '481', inputs: [ { key: 'model', @@ -247,6 +250,7 @@ export function form2AppWorkflow(data: AppSimpleEditFormType): WorkflowType { x: 1638.509551404687, y: -341.0428450861567 }, + version: '481', // [FlowNodeTypeEnum.chatNode] inputs: [ { key: 'model', @@ -366,6 +370,7 @@ export function form2AppWorkflow(data: AppSimpleEditFormType): WorkflowType { x: 918.5901682164496, y: -227.11542247619582 }, + version: '481', inputs: [ { key: 'datasets', @@ -484,6 +489,7 @@ export function form2AppWorkflow(data: AppSimpleEditFormType): WorkflowType { x: 500, y: 545 }, + version: '481', inputs: [ { key: 'datasets', @@ -597,6 +603,7 @@ export function form2AppWorkflow(data: AppSimpleEditFormType): WorkflowType { x: 500 + 500 * (i + 1), y: 545 }, + version: tool.version, inputs: tool.inputs, outputs: tool.outputs } @@ -625,6 +632,7 @@ export function form2AppWorkflow(data: AppSimpleEditFormType): WorkflowType { x: 1062.1738942532802, y: -223.65033022650476 }, + version: '481', inputs: [ { key: 'model', diff --git a/projects/app/src/web/core/workflow/adapt.ts b/projects/app/src/web/core/workflow/adapt.ts index b4e6dd15766a..86657ec873c8 100644 --- a/projects/app/src/web/core/workflow/adapt.ts +++ b/projects/app/src/web/core/workflow/adapt.ts @@ -39,6 +39,7 @@ export const getGlobalVariableNode = (nodes: FlowNodeItemType[], t: TFunction) = intro: '', unique: true, forbidDelete: true, + version: '481', inputs: [], outputs: [] }; @@ -418,6 +419,7 @@ export const v1Workflow2V2 = ( pluginId, pluginType: node.pluginType, parentId: node.parentId, + version: 'v2.0', inputs, outputs diff --git a/projects/app/src/web/core/workflow/utils.ts b/projects/app/src/web/core/workflow/utils.ts index dfc822bc1b82..6d3667442c52 100644 --- a/projects/app/src/web/core/workflow/utils.ts +++ b/projects/app/src/web/core/workflow/utils.ts @@ -24,7 +24,11 @@ import { } from '@fastgpt/global/core/workflow/utils'; import { getSystemVariables } from '../app/utils'; import { TFunction } from 'next-i18next'; -import { ReferenceValueProps } from '@fastgpt/global/core/workflow/type/io'; +import { + FlowNodeInputItemType, + FlowNodeOutputItemType, + ReferenceValueProps +} from '@fastgpt/global/core/workflow/type/io'; import { IfElseListItemType } from '@fastgpt/global/core/workflow/template/system/ifElse/type'; import { VariableConditionEnum } from '@fastgpt/global/core/workflow/template/system/ifElse/constant'; @@ -87,7 +91,8 @@ export const storeNode2FlowNode = ({ ...templateOutput, value: storeOutput.value }; - }) + }), + version: storeNode.version || '481' }; return { @@ -298,3 +303,34 @@ export const getWorkflowGlobalVariables = ( return [...globalVariables, ...systemVariables]; }; + +export type CombinedItemType = Partial & Partial; + +export const updateFlowNodeVersion = ( + node: FlowNodeItemType, + template: FlowNodeTemplateType +): FlowNodeItemType => { + function updateArrayBasedOnTemplate( + nodeArray: T[], + templateArray: T[] + ): T[] { + return templateArray.map((templateItem) => { + const nodeItem = nodeArray.find((item) => item.key === templateItem.key); + if (nodeItem) { + return { ...templateItem, ...nodeItem } as T; + } + return { ...templateItem }; + }); + } + + const updatedNode: FlowNodeItemType = { ...node, ...template, name: node.name }; + + if (node.inputs && template.inputs) { + updatedNode.inputs = updateArrayBasedOnTemplate(node.inputs, template.inputs); + } + if (node.outputs && template.outputs) { + updatedNode.outputs = updateArrayBasedOnTemplate(node.outputs, template.outputs); + } + + return updatedNode; +}; From cd876251b70deb2ccd6171f6de82e33a036ab948 Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Wed, 15 May 2024 10:19:51 +0800 Subject: [PATCH 06/27] External dataset (#1485) * fix: revert version * feat: external collection * import context * external ui * doc * fix: ts * clear invalid data * feat: rename sub name * fix: node if else edge remove * fix: init * api size * fix: if else node refresh --- .vscode/nextapi.code-snippets | 13 +- Dockerfile | 2 + docSite/content/docs/development/faq.md | 3 +- .../content/docs/development/upgrading/481.md | 3 +- packages/global/core/dataset/api.d.ts | 8 +- .../core/dataset/collection/constants.ts | 6 + packages/global/core/dataset/constants.ts | 17 +- packages/global/core/dataset/type.d.ts | 23 +- packages/global/core/dataset/utils.ts | 2 +- .../template/system/assignedAnswer.ts | 1 + .../service/core/dataset/collection/schema.ts | 25 +- packages/service/core/dataset/schema.ts | 3 +- packages/service/support/wallet/sub/schema.ts | 2 +- .../components/common/DndDrag/DragIcon.tsx | 6 +- projects/app/.eslintrc.json | 8 +- projects/app/i18n/en/common.json | 7 +- projects/app/i18n/en/dataset.json | 13 +- projects/app/i18n/zh/common.json | 6 +- projects/app/i18n/zh/dataset.json | 13 +- .../src/components/common/NextHead/index.tsx | 14 + .../core/dataset/DatasetTypeTag.tsx | 9 +- .../Flow/nodes/NodeIfElse/ListItem.tsx | 146 +++--- .../workflow/Flow/nodes/NodeIfElse/index.tsx | 9 +- .../components/PublishHistoriesSlider.tsx | 35 +- .../src/components/core/workflow/context.tsx | 4 +- projects/app/src/global/core/dataset/api.d.ts | 4 +- projects/app/src/pages/_app.tsx | 26 +- .../src/pages/api/admin/clearInvalidData.ts | 13 +- projects/app/src/pages/api/admin/initv481.ts | 32 +- .../src/pages/api/common/file/uploadImage.ts | 1 - .../src/pages/api/core/dataset/allDataset.ts | 54 +-- .../app/src/pages/api/core/dataset/list.ts | 2 +- .../app/src/pages/api/core/dataset/update.ts | 17 +- .../app/src/pages/api/core/plugin/list.ts | 2 +- .../app/src/pages/api/core/workflow/debug.ts | 2 +- .../pages/api/plugins/textEditor/v2/index.ts | 3 + .../app/src/pages/api/v1/chat/completions.ts | 3 + projects/app/src/pages/chat/share.tsx | 244 +++++----- .../components/CollectionCard/Context.tsx | 158 +++++++ .../CollectionCard/EmptyCollectionTip.tsx | 55 +++ .../components/CollectionCard/Header.tsx | 399 ++++++++++++++++ .../WebsiteConfig.tsx | 0 .../index.tsx} | 437 +----------------- .../detail/components/Import/Context.tsx | 302 ++++++++++++ .../detail/components/Import/Provider.tsx | 165 ------- .../Import/commonProgress/DataProcess.tsx | 15 +- .../Import/commonProgress/PreviewData.tsx | 11 +- .../Import/commonProgress/Upload.tsx | 10 +- .../Import/components/FileSourceSelector.tsx | 2 +- .../components/Import/components/Preview.tsx | 5 +- .../Import/components/PreviewChunks.tsx | 8 +- .../Import/components/PreviewRawText.tsx | 5 +- .../Import/diffSource/ExternalFile.tsx | 188 ++++++++ .../Import/diffSource/FileCustomText.tsx | 15 +- .../components/Import/diffSource/FileLink.tsx | 20 +- .../Import/diffSource/FileLocal.tsx | 17 +- .../Import/diffSource/TableLocal.tsx | 17 +- .../detail/components/Import/index.tsx | 153 +----- .../pages/dataset/detail/components/Info.tsx | 62 ++- .../dataset/detail/components/Slider.tsx | 52 +-- .../pages/dataset/detail/components/Test.tsx | 4 +- .../app/src/pages/dataset/detail/index.tsx | 79 ++-- .../dataset/list/component/CreateModal.tsx | 18 +- .../dataset/list/component/MoveModal.tsx | 5 +- projects/app/src/pages/dataset/list/index.tsx | 33 +- .../src/service/support/wallet/usage/push.ts | 2 +- projects/app/src/web/core/dataset/api.ts | 10 +- .../dataset/components/SelectCollections.tsx | 5 +- .../app/src/web/core/dataset/constants.ts | 7 +- .../dataset/context/datasetPageContext.tsx | 62 ++- .../core/dataset/context/datasetsContext.tsx | 11 + .../dataset/context/useDatasetContext.tsx | 18 - .../app/src/web/core/dataset/store/dataset.ts | 83 +--- projects/app/src/web/core/dataset/type.d.ts | 13 +- 74 files changed, 1878 insertions(+), 1349 deletions(-) create mode 100644 packages/global/core/dataset/collection/constants.ts create mode 100644 projects/app/src/components/common/NextHead/index.tsx create mode 100644 projects/app/src/pages/dataset/detail/components/CollectionCard/Context.tsx create mode 100644 projects/app/src/pages/dataset/detail/components/CollectionCard/EmptyCollectionTip.tsx create mode 100644 projects/app/src/pages/dataset/detail/components/CollectionCard/Header.tsx rename projects/app/src/pages/dataset/detail/components/{Import => CollectionCard}/WebsiteConfig.tsx (100%) rename projects/app/src/pages/dataset/detail/components/{CollectionCard.tsx => CollectionCard/index.tsx} (52%) create mode 100644 projects/app/src/pages/dataset/detail/components/Import/Context.tsx delete mode 100644 projects/app/src/pages/dataset/detail/components/Import/Provider.tsx create mode 100644 projects/app/src/pages/dataset/detail/components/Import/diffSource/ExternalFile.tsx create mode 100644 projects/app/src/web/core/dataset/context/datasetsContext.tsx delete mode 100644 projects/app/src/web/core/dataset/context/useDatasetContext.tsx diff --git a/.vscode/nextapi.code-snippets b/.vscode/nextapi.code-snippets index 19864409b03a..e597e0bac529 100644 --- a/.vscode/nextapi.code-snippets +++ b/.vscode/nextapi.code-snippets @@ -40,18 +40,11 @@ "", "type ContextType = {$1};", "", - "type ContextValueType = {};", - "", "export const Context = createContext({});", "", - "export const ContextProvider = ({", - " children,", - " value", - "}: {", - " children: ReactNode;", - " value: ContextValueType;", - "}) => {", - " return {children};", + "export const ContextProvider = ({ children }: { children: ReactNode }) => {", + " const contextValue: ContextType = {};", + " return {children};", "};", ], "description": "FastGPT usecontext template" diff --git a/Dockerfile b/Dockerfile index 2e97b1489d83..6a4be70a84db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,6 +36,8 @@ COPY --from=mainDeps /app/projects/$name/node_modules ./projects/$name/node_modu RUN [ -z "$proxy" ] || sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories RUN apk add --no-cache libc6-compat && npm install -g pnpm@8.6.0 + +ENV NODE_OPTIONS="--max-old-space-size=4096" RUN pnpm --filter=$name build # --------- runner ----------- diff --git a/docSite/content/docs/development/faq.md b/docSite/content/docs/development/faq.md index 8733145db18c..e1bada463e71 100644 --- a/docSite/content/docs/development/faq.md +++ b/docSite/content/docs/development/faq.md @@ -118,4 +118,5 @@ OneAPI 的 API Key 配置错误，需要修改`OPENAI_API_KEY`环境变量，并 ### bad_response_status_code bad response status code 503 1. 模型服务不可用 -2. .... \ No newline at end of file +2. 模型接口参数异常（温度、max token等可能不适配） +3. .... \ No newline at end of file diff --git a/docSite/content/docs/development/upgrading/481.md b/docSite/content/docs/development/upgrading/481.md index 1fa86b622772..21a391593079 100644 --- a/docSite/content/docs/development/upgrading/481.md +++ b/docSite/content/docs/development/upgrading/481.md @@ -35,4 +35,5 @@ curl --location --request POST 'https://{{host}}/api/admin/clearInvalidData' \ ## V4.8.1 更新说明 1. 新增 - 知识库重新选择向量模型重建 -2. 修复 - 定时器清理脏数据任务 \ No newline at end of file +2. 修复 - 工作流删除节点的动态输入和输出时候，没有正确的删除连接线，导致可能出现逻辑异常。 +3. 修复 - 定时器清理脏数据任务 \ No newline at end of file diff --git a/packages/global/core/dataset/api.d.ts b/packages/global/core/dataset/api.d.ts index 7a34b6c8c7d1..af0eb5e76167 100644 --- a/packages/global/core/dataset/api.d.ts +++ b/packages/global/core/dataset/api.d.ts @@ -11,14 +11,16 @@ export type DatasetUpdateBody = { intro?: string; permission?: DatasetSchemaType['permission']; agentModel?: LLMModelItemType; - websiteConfig?: DatasetSchemaType['websiteConfig']; status?: DatasetSchemaType['status']; + + websiteConfig?: DatasetSchemaType['websiteConfig']; + externalReadUrl?: DatasetSchemaType['externalReadUrl']; }; /* ================= collection ===================== */ export type DatasetCollectionChunkMetadataType = { parentId?: string; - trainingType?: `${TrainingModeEnum}`; + trainingType?: TrainingModeEnum; chunkSize?: number; chunkSplitter?: string; qaPrompt?: string; @@ -78,7 +80,7 @@ export type PostWebsiteSyncParams = { export type PushDatasetDataProps = { collectionId: string; data: PushDatasetDataChunkProps[]; - trainingMode: `${TrainingModeEnum}`; + trainingMode: TrainingModeEnum; prompt?: string; billId?: string; }; diff --git a/packages/global/core/dataset/collection/constants.ts b/packages/global/core/dataset/collection/constants.ts new file mode 100644 index 000000000000..0b0cda0b1376 --- /dev/null +++ b/packages/global/core/dataset/collection/constants.ts @@ -0,0 +1,6 @@ +/* sourceId = prefix-id; id=fileId;link url;externalId */ +export enum CollectionSourcePrefixEnum { + local = 'local', + link = 'link', + external = 'external' +} diff --git a/packages/global/core/dataset/constants.ts b/packages/global/core/dataset/constants.ts index 0e8be0f5d766..570fc1417691 100644 --- a/packages/global/core/dataset/constants.ts +++ b/packages/global/core/dataset/constants.ts @@ -2,23 +2,29 @@ export enum DatasetTypeEnum { folder = 'folder', dataset = 'dataset', - websiteDataset = 'websiteDataset' // depp link + websiteDataset = 'websiteDataset', // depp link + externalFile = 'externalFile' } export const DatasetTypeMap = { [DatasetTypeEnum.folder]: { icon: 'common/folderFill', - label: 'core.dataset.Folder Dataset', + label: 'Folder Dataset', collectionLabel: 'common.Folder' }, [DatasetTypeEnum.dataset]: { icon: 'core/dataset/commonDataset', - label: 'core.dataset.Common Dataset', + label: 'Common Dataset', collectionLabel: 'common.File' }, [DatasetTypeEnum.websiteDataset]: { icon: 'core/dataset/websiteDataset', - label: 'core.dataset.Website Dataset', + label: 'Website Dataset', collectionLabel: 'common.Website' + }, + [DatasetTypeEnum.externalFile]: { + icon: 'core/dataset/commonDataset', + label: 'External File', + collectionLabel: 'common.File' } }; @@ -77,7 +83,8 @@ export enum ImportDataSourceEnum { fileLocal = 'fileLocal', fileLink = 'fileLink', fileCustom = 'fileCustom', - csvTable = 'csvTable' + csvTable = 'csvTable', + externalFile = 'externalFile' } export enum TrainingModeEnum { diff --git a/packages/global/core/dataset/type.d.ts b/packages/global/core/dataset/type.d.ts index 7d89b322c2f2..65cd20a9290a 100644 --- a/packages/global/core/dataset/type.d.ts +++ b/packages/global/core/dataset/type.d.ts @@ -22,13 +22,16 @@ export type DatasetSchemaType = { vectorModel: string; agentModel: string; intro: string; - type: `${DatasetTypeEnum}`; + type: DatasetTypeEnum; status: `${DatasetStatusEnum}`; permission: `${PermissionTypeEnum}`; + + // metadata websiteConfig?: { url: string; selector: string; }; + externalReadUrl?: string; }; export type DatasetCollectionSchemaType = { @@ -42,16 +45,18 @@ export type DatasetCollectionSchemaType = { createTime: Date; updateTime: Date; - trainingType: `${TrainingModeEnum}`; + trainingType: TrainingModeEnum; chunkSize: number; chunkSplitter?: string; qaPrompt?: string; - fileId?: string; - rawLink?: string; + sourceId?: string; // relate CollectionSourcePrefixEnum + fileId?: string; // local file id + rawLink?: string; // link url rawTextLength?: number; hashRawText?: string; + externalSourceUrl?: string; // external import url metadata?: { webPageSelector?: string; relatedImgId?: string; // The id of the associated image collections @@ -93,7 +98,7 @@ export type DatasetTrainingSchemaType = { billId: string; expireAt: Date; lockTime: Date; - mode: `${TrainingModeEnum}`; + mode: TrainingModeEnum; model: string; prompt: string; dataId?: string; @@ -112,13 +117,19 @@ export type DatasetDataWithCollectionType = Omit { +export const predictDataLimitLength = (mode: TrainingModeEnum, data: any[]) => { if (mode === TrainingModeEnum.qa) return data.length * 20; if (mode === TrainingModeEnum.auto) return data.length * 5; return data.length; diff --git a/packages/global/core/workflow/template/system/assignedAnswer.ts b/packages/global/core/workflow/template/system/assignedAnswer.ts index 6f133b5571ac..26c6f6baed3f 100644 --- a/packages/global/core/workflow/template/system/assignedAnswer.ts +++ b/packages/global/core/workflow/template/system/assignedAnswer.ts @@ -18,6 +18,7 @@ export const AssignedAnswerModule: FlowNodeTemplateType = { intro: '该模块可以直接回复一段指定的内容。常用于引导、提示。非字符串内容传入时，会转成字符串进行输出。', version: '481', + isTool: true, inputs: [ { key: NodeInputKeyEnum.answerText, diff --git a/packages/service/core/dataset/collection/schema.ts b/packages/service/core/dataset/collection/schema.ts index e8cb6e53acc6..dea33a72b243 100644 --- a/packages/service/core/dataset/collection/schema.ts +++ b/packages/service/core/dataset/collection/schema.ts @@ -16,11 +16,6 @@ const DatasetCollectionSchema = new Schema({ ref: DatasetColCollectionName, default: null }, - userId: { - // abandoned - type: Schema.Types.ObjectId, - ref: 'user' - }, teamId: { type: Schema.Types.ObjectId, ref: TeamCollectionName, @@ -54,6 +49,7 @@ const DatasetCollectionSchema = new Schema({ default: () => new Date() }, + // chunk filed trainingType: { type: String, enum: Object.keys(TrainingTypeMap), @@ -70,20 +66,21 @@ const DatasetCollectionSchema = new Schema({ type: String }, + sourceId: String, + // local file collection fileId: { type: Schema.Types.ObjectId, ref: 'dataset.files' }, - rawLink: { - type: String - }, + // web link collection + rawLink: String, - rawTextLength: { - type: Number - }, - hashRawText: { - type: String - }, + // external collection + + // metadata + rawTextLength: Number, + hashRawText: String, + externalSourceUrl: String, // external import url metadata: { type: Object, default: {} diff --git a/packages/service/core/dataset/schema.ts b/packages/service/core/dataset/schema.ts index 4139e042b57a..95f81daf6a20 100644 --- a/packages/service/core/dataset/schema.ts +++ b/packages/service/core/dataset/schema.ts @@ -89,7 +89,8 @@ const DatasetSchema = new Schema({ default: 'body' } } - } + }, + externalReadUrl: String }); try { diff --git a/packages/service/support/wallet/sub/schema.ts b/packages/service/support/wallet/sub/schema.ts index b90cd9b4baf4..42709e8024a3 100644 --- a/packages/service/support/wallet/sub/schema.ts +++ b/packages/service/support/wallet/sub/schema.ts @@ -14,7 +14,7 @@ import { } from '@fastgpt/global/support/wallet/sub/constants'; import type { TeamSubSchema } from '@fastgpt/global/support/wallet/sub/type'; -export const subCollectionName = 'team.subscriptions'; +export const subCollectionName = 'team_subscriptions'; const SubSchema = new Schema({ teamId: { diff --git a/packages/web/components/common/DndDrag/DragIcon.tsx b/packages/web/components/common/DndDrag/DragIcon.tsx index 493a409d2d3a..36eedd13ce05 100644 --- a/packages/web/components/common/DndDrag/DragIcon.tsx +++ b/packages/web/components/common/DndDrag/DragIcon.tsx @@ -1,11 +1,11 @@ import { DragHandleIcon } from '@chakra-ui/icons'; -import { Box } from '@chakra-ui/react'; +import { Box, BoxProps } from '@chakra-ui/react'; import React from 'react'; import { DraggableProvided } from 'react-beautiful-dnd'; -const DragIcon = ({ provided }: { provided: DraggableProvided }) => { +const DragIcon = ({ provided, ...props }: { provided: DraggableProvided } & BoxProps) => { return ( - + ); diff --git a/projects/app/.eslintrc.json b/projects/app/.eslintrc.json index 3a5d07f926c9..be661eb5c4df 100644 --- a/projects/app/.eslintrc.json +++ b/projects/app/.eslintrc.json @@ -1,12 +1,6 @@ - { - "parser": "@typescript-eslint/parser", // 确保使用了 TypeScript 解析器 - "plugins": ["@typescript-eslint"], // 引入 TypeScript 插件 - "extends": "next/core-web-vitals", "rules": { - "react-hooks/rules-of-hooks": 0, - "@typescript-eslint/consistent-type-imports": "warn" // 或者 "error" 来强制执行 - + "react-hooks/rules-of-hooks": 0 } } diff --git a/projects/app/i18n/en/common.json b/projects/app/i18n/en/common.json index e8de25278ee6..14fe00397aea 100644 --- a/projects/app/i18n/en/common.json +++ b/projects/app/i18n/en/common.json @@ -1,4 +1,5 @@ { + "Add new": "Add new", "App": "App", "Export": "Export", "Folder": "Folder", @@ -509,18 +510,14 @@ "Choose Dataset": "Associate dataset", "Chunk amount": "Number of chunks", "Collection": "Dataset", - "Common Dataset": "Common dataset", - "Common Dataset Desc": "Can be built by importing files, web links, or manual entry", "Create dataset": "Create a dataset", "Dataset": "Dataset", "Dataset ID": "Dataset ID", "Dataset Type": "Dataset type", "Delete Confirm": "Confirm to delete this dataset? Data cannot be recovered after deletion, please confirm!", "Delete Website Tips": "Confirm to delete this site?", - "Empty Dataset": "", "Empty Dataset Tips": "No datasets yet, go create one!", "File collection": "File dataset", - "Folder Dataset": "Folder", "Folder placeholder": "This is a directory", "Go Dataset": "Go to dataset", "Intro Placeholder": "This dataset has no introduction~", @@ -540,8 +537,6 @@ "Table collection": "Table dataset", "Text collection": "Text dataset", "Total chunks": "Total chunks: {{total}}", - "Website Dataset": "Web site synchronization", - "Website Dataset Desc": "Web site synchronization allows you to use a web page link to build a dataset", "collection": { "Click top config website": "Click to configure website", "Collection name": "Dataset name", diff --git a/projects/app/i18n/en/dataset.json b/projects/app/i18n/en/dataset.json index 0c2bc01d815e..510955b1c4c1 100644 --- a/projects/app/i18n/en/dataset.json +++ b/projects/app/i18n/en/dataset.json @@ -1,6 +1,17 @@ { + "Common Dataset": "Common dataset", + "Common Dataset Desc": "Can be built by importing files, web links, or manual entry", "Confirm to rebuild embedding tip": "Are you sure to switch the knowledge base index? Switching index is a very heavy operation that requires re-indexing all the data in your knowledge base, which may take a long time. Please ensure that the remaining points in your account are sufficient.", + "External file": "External file", + "External file Dataset Desc": "You can import files from an external file library to build a knowledge base. Files are not stored twice", + "External id": "File id", + "External read url": "External read url", + "External url": "File read url", + "Folder Dataset": "Folder", "Rebuild embedding start tip": "The task of switching index models has begun", "Rebuilding index count": "Rebuilding count: {{count}}", - "The knowledge base has indexes that are being trained or being rebuilt": "The knowledge base has indexes that are being trained or being rebuilt" + "The knowledge base has indexes that are being trained or being rebuilt": "The knowledge base has indexes that are being trained or being rebuilt", + "Website Dataset": "Web site", + "Website Dataset Desc": "Web site synchronization allows you to use a web page link to build a dataset", + "filename": "filename" } diff --git a/projects/app/i18n/zh/common.json b/projects/app/i18n/zh/common.json index 55ed8ff9227d..ca14b35f8dc6 100644 --- a/projects/app/i18n/zh/common.json +++ b/projects/app/i18n/zh/common.json @@ -1,4 +1,5 @@ { + "Add new": "新增", "App": "应用", "Export": "导出", "Folder": "文件夹", @@ -509,8 +510,6 @@ "Choose Dataset": "关联知识库", "Chunk amount": "分段数", "Collection": "数据集", - "Common Dataset": "通用知识库", - "Common Dataset Desc": "可通过导入文件、网页链接或手动录入形式构建知识库", "Create dataset": "创建一个知识库", "Dataset": "知识库", "Dataset ID": "知识库 ID", @@ -520,7 +519,6 @@ "Empty Dataset": "", "Empty Dataset Tips": "还没有知识库，快去创建一个吧！", "File collection": "文件数据集", - "Folder Dataset": "文件夹", "Folder placeholder": "这是一个目录", "Go Dataset": "前往知识库", "Intro Placeholder": "这个知识库还没有介绍~", @@ -540,8 +538,6 @@ "Table collection": "表格数据集", "Text collection": "文本数据集", "Total chunks": "总分段: {{total}}", - "Website Dataset": "Web 站点同步", - "Website Dataset Desc": "Web 站点同步允许你直接使用一个网页链接构建知识库", "collection": { "Click top config website": "点击配置网站", "Collection name": "数据集名称", diff --git a/projects/app/i18n/zh/dataset.json b/projects/app/i18n/zh/dataset.json index 4289048f7cba..d07a16bbe4ea 100644 --- a/projects/app/i18n/zh/dataset.json +++ b/projects/app/i18n/zh/dataset.json @@ -1,6 +1,17 @@ { + "Common Dataset": "通用知识库", + "Common Dataset Desc": "可通过导入文件、网页链接或手动录入形式构建知识库", "Confirm to rebuild embedding tip": "确认为知识库切换索引？\n切换索引是一个非常重量的操作，需要对您知识库内所有数据进行重新索引，时间可能较长，请确保账号内剩余积分充足。", + "External File": "外部文件库", + "External file Dataset Desc": "可以从外部文件库导入文件构建知识库，文件不会进行二次存储", + "External id": "文件阅读ID", + "External read url": "外部预览地址", + "External url": "文件访问URL", + "Folder Dataset": "文件夹", "Rebuild embedding start tip": "切换索引模型任务已开始", "Rebuilding index count": "重建中索引数量: {{count}}", - "The knowledge base has indexes that are being trained or being rebuilt": "知识库有训练中或正在重建的索引" + "The knowledge base has indexes that are being trained or being rebuilt": "知识库有训练中或正在重建的索引", + "Website Dataset": "Web 站点同步", + "Website Dataset Desc": "Web 站点同步允许你直接使用一个网页链接构建知识库", + "filename": "文件名" } diff --git a/projects/app/src/components/common/NextHead/index.tsx b/projects/app/src/components/common/NextHead/index.tsx new file mode 100644 index 000000000000..a3f529bc0f1c --- /dev/null +++ b/projects/app/src/components/common/NextHead/index.tsx @@ -0,0 +1,14 @@ +import Head from 'next/head'; +import React from 'react'; + +const NextHead = ({ title, icon, desc }: { title?: string; icon?: string; desc?: string }) => { + return ( + + {title} + {desc && } + {icon && } + + ); +}; + +export default NextHead; diff --git a/projects/app/src/components/core/dataset/DatasetTypeTag.tsx b/projects/app/src/components/core/dataset/DatasetTypeTag.tsx index 55122d0946bf..f31adebfb40b 100644 --- a/projects/app/src/components/core/dataset/DatasetTypeTag.tsx +++ b/projects/app/src/components/core/dataset/DatasetTypeTag.tsx @@ -1,12 +1,12 @@ import { Box, Flex, FlexProps } from '@chakra-ui/react'; import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; import MyIcon from '@fastgpt/web/components/common/Icon'; -import { useTranslation } from 'next-i18next'; import React from 'react'; import { DatasetTypeMap } from '@fastgpt/global/core/dataset/constants'; +import { useI18n } from '@/web/context/I18n'; -const DatasetTypeTag = ({ type, ...props }: { type: `${DatasetTypeEnum}` } & FlexProps) => { - const { t } = useTranslation(); +const DatasetTypeTag = ({ type, ...props }: { type: DatasetTypeEnum } & FlexProps) => { + const { datasetT } = useI18n(); const item = DatasetTypeMap[type] || DatasetTypeMap['dataset']; @@ -22,7 +22,8 @@ const DatasetTypeTag = ({ type, ...props }: { type: `${DatasetTypeEnum}` } & Fle {...props} > - {t(item.label)} + {/* @ts-ignore */} + {datasetT(item.label)} ); }; diff --git a/projects/app/src/components/core/workflow/Flow/nodes/NodeIfElse/ListItem.tsx b/projects/app/src/components/core/workflow/Flow/nodes/NodeIfElse/ListItem.tsx index 5f23a34f7bab..33c8e457476d 100644 --- a/projects/app/src/components/core/workflow/Flow/nodes/NodeIfElse/ListItem.tsx +++ b/projects/app/src/components/core/workflow/Flow/nodes/NodeIfElse/ListItem.tsx @@ -50,16 +50,11 @@ const ListItem = ({ }) => { const { t } = useTranslation(); const { getZoom } = useReactFlow(); + const onDelEdge = useContextSelector(WorkflowContext, (v) => v.onDelEdge); + const handleId = getHandleId(nodeId, 'source', getElseIFLabel(conditionIndex)); - return ( - + const Render = useMemo(() => { + return ( - {ifElseList.length > 1 && } + 1 ? 'visible' : 'hidden'} + provided={provided} + /> {getElseIFLabel(conditionIndex)} @@ -109,6 +107,10 @@ const ListItem = ({ color={'myGray.400'} onClick={() => { onUpdateIfElseList(ifElseList.filter((_, index) => index !== conditionIndex)); + onDelEdge({ + nodeId, + sourceHandle: handleId + }); }} /> )} @@ -185,21 +187,21 @@ const ListItem = ({ onChange={(e) => { onUpdateIfElseList( ifElseList.map((ifElse, index) => { - if (index === conditionIndex) { - return { - ...ifElse, - list: ifElse.list.map((item, index) => { - if (index === i) { - return { - ...item, - value: e - }; - } - return item; - }) - }; - } - return ifElse; + return { + ...ifElse, + list: + index === conditionIndex + ? ifElse.list.map((item, index) => { + if (index === i) { + return { + ...item, + value: e + }; + } + return item; + }) + : ifElse.list + }; }) ); }} @@ -263,12 +265,38 @@ const ListItem = ({ {!snapshot.isDragging && ( )} + ); + }, [ + conditionIndex, + conditionItem.condition, + conditionItem.list, + getZoom, + handleId, + ifElseList, + nodeId, + onDelEdge, + onUpdateIfElseList, + provided, + snapshot.isDragging, + t + ]); + + return ( + + {Render} ); }; @@ -387,35 +415,39 @@ const ConditionValueInput = ({ return output.valueType; }, [nodeList, variable]); - if (valueType === WorkflowIOValueTypeEnum.boolean) { - return ( - - ); - } else { - return ( - onChange(e.target.value)} - /> - ); - } + const Render = useMemo(() => { + if (valueType === WorkflowIOValueTypeEnum.boolean) { + return ( + + ); + } else { + return ( + onChange(e.target.value)} + /> + ); + } + }, [condition, onChange, value, valueType]); + + return Render; }; diff --git a/projects/app/src/components/core/workflow/Flow/nodes/NodeIfElse/index.tsx b/projects/app/src/components/core/workflow/Flow/nodes/NodeIfElse/index.tsx index 026245d9862c..f393a0c36c5a 100644 --- a/projects/app/src/components/core/workflow/Flow/nodes/NodeIfElse/index.tsx +++ b/projects/app/src/components/core/workflow/Flow/nodes/NodeIfElse/index.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useMemo, useState } from 'react'; +import React, { useCallback, useMemo } from 'react'; import NodeCard from '../render/NodeCard'; import { useTranslation } from 'next-i18next'; import { Box, Button, Flex } from '@chakra-ui/react'; @@ -9,7 +9,7 @@ import { IfElseListItemType } from '@fastgpt/global/core/workflow/template/syste import { useContextSelector } from 'use-context-selector'; import { WorkflowContext } from '../../../context'; import Container from '../../components/Container'; -import DndDrag, { Draggable, DropResult } from '@fastgpt/web/components/common/DndDrag/index'; +import DndDrag, { Draggable } from '@fastgpt/web/components/common/DndDrag/index'; import { SourceHandle } from '../render/Handle'; import { getHandleId } from '@fastgpt/global/core/workflow/utils'; import ListItem from './ListItem'; @@ -19,6 +19,7 @@ const NodeIfElse = ({ data, selected }: NodeProps) => { const { t } = useTranslation(); const { nodeId, inputs = [] } = data; const onChangeNode = useContextSelector(WorkflowContext, (v) => v.onChangeNode); + const elseHandleId = getHandleId(nodeId, 'source', IfElseResultEnum.ELSE); const ifElseList = useMemo( () => @@ -49,7 +50,7 @@ const NodeIfElse = ({ data, selected }: NodeProps) => { - onDragEndCb={(list) => onUpdateIfElseList(list)} + onDragEndCb={(list: IfElseListItemType[]) => onUpdateIfElseList(list)} dataList={ifElseList} renderClone={(provided, snapshot, rubric) => ( ) => { diff --git a/projects/app/src/components/core/workflow/components/PublishHistoriesSlider.tsx b/projects/app/src/components/core/workflow/components/PublishHistoriesSlider.tsx index 80617fde5653..c38df1523b32 100644 --- a/projects/app/src/components/core/workflow/components/PublishHistoriesSlider.tsx +++ b/projects/app/src/components/core/workflow/components/PublishHistoriesSlider.tsx @@ -1,4 +1,4 @@ -import React, { useState } from 'react'; +import React, { useCallback, useState } from 'react'; import { getPublishList, postRevertVersion } from '@/web/core/app/versionApi'; import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination'; import CustomRightDrawer from '@fastgpt/web/components/common/MyDrawer/CustomRightDrawer'; @@ -14,6 +14,8 @@ import MyIcon from '@fastgpt/web/components/common/Icon'; import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; import { useConfirm } from '@fastgpt/web/hooks/useConfirm'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; +import { StoreNodeItemType } from '@fastgpt/global/core/workflow/type'; +import { StoreEdgeItemType } from '@fastgpt/global/core/workflow/type/edge'; const PublishHistoriesSlider = () => { const { t } = useTranslation(); @@ -45,29 +47,29 @@ const PublishHistoriesSlider = () => { setIsShowVersionHistories(false); }); - const onPreview = useMemoizedFn((data: AppVersionSchemaType) => { + const onPreview = useCallback((data: AppVersionSchemaType) => { setSelectedHistoryId(data._id); initData({ nodes: data.nodes, edges: data.edges }); - }); - const onCloseSlider = useMemoizedFn(() => { - setSelectedHistoryId(undefined); - initData({ - nodes: appDetail.modules, - edges: appDetail.edges - }); - onClose(); - }); + }, []); + const onCloseSlider = useCallback( + (data: { nodes: StoreNodeItemType[]; edges: StoreEdgeItemType[] }) => { + setSelectedHistoryId(undefined); + initData(data); + onClose(); + }, + [appDetail] + ); const { mutate: onRevert, isLoading: isReverting } = useRequest({ mutationFn: async (data: AppVersionSchemaType) => { if (!appId) return; await postRevertVersion(appId, { versionId: data._id, - editNodes: appDetail.modules, + editNodes: appDetail.modules, // old workflow editEdges: appDetail.edges }); @@ -77,7 +79,7 @@ const PublishHistoriesSlider = () => { edges: data.edges }); - onCloseSlider(); + onCloseSlider(data); } }); @@ -86,7 +88,12 @@ const PublishHistoriesSlider = () => { return ( <> + onCloseSlider({ + nodes: appDetail.modules, + edges: appDetail.edges + }) + } iconSrc="core/workflow/versionHistories" title={t('core.workflow.publish.histories')} maxW={'300px'} diff --git a/projects/app/src/components/core/workflow/context.tsx b/projects/app/src/components/core/workflow/context.tsx index a2ffd6d81bf5..faf5c1cd157f 100644 --- a/projects/app/src/components/core/workflow/context.tsx +++ b/projects/app/src/components/core/workflow/context.tsx @@ -430,8 +430,8 @@ const WorkflowContextProvider = ({ const initData = useMemoizedFn( async (e: { nodes: StoreNodeItemType[]; edges: StoreEdgeItemType[] }) => { - setNodes(e.nodes?.map((item) => storeNode2FlowNode({ item }))); - setEdges(e.edges?.map((item) => storeEdgesRenderEdge({ edge: item }))); + setNodes(e.nodes?.map((item) => storeNode2FlowNode({ item })) || []); + setEdges(e.edges?.map((item) => storeEdgesRenderEdge({ edge: item })) || []); } ); diff --git a/projects/app/src/global/core/dataset/api.d.ts b/projects/app/src/global/core/dataset/api.d.ts index 69ef8c9ecf9e..e5fb7cf9f1c3 100644 --- a/projects/app/src/global/core/dataset/api.d.ts +++ b/projects/app/src/global/core/dataset/api.d.ts @@ -14,7 +14,7 @@ import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants'; /* ================= dataset ===================== */ export type CreateDatasetParams = { parentId?: string; - type: `${DatasetTypeEnum}`; + type: DatasetTypeEnum; name: string; intro: string; avatar: string; @@ -76,7 +76,7 @@ export type SearchTestResponse = { /* =========== training =========== */ export type PostPreviewFilesChunksProps = { - type: `${ImportDataSourceEnum}`; + type: ImportDataSourceEnum; sourceId: string; chunkSize: number; overlapRatio: number; diff --git a/projects/app/src/pages/_app.tsx b/projects/app/src/pages/_app.tsx index 69f3d589d9ea..c8b133c8f12b 100644 --- a/projects/app/src/pages/_app.tsx +++ b/projects/app/src/pages/_app.tsx @@ -10,28 +10,22 @@ import I18nContextProvider from '@/web/context/I18n'; import { useInitApp } from '@/web/context/useInitApp'; import '@/web/styles/reset.scss'; +import NextHead from '@/components/common/NextHead'; function App({ Component, pageProps }: AppProps) { const { feConfigs, scripts, title } = useInitApp(); return ( <> - - {title} - - - - + {scripts?.map((item, i) => )} diff --git a/projects/app/src/pages/api/admin/clearInvalidData.ts b/projects/app/src/pages/api/admin/clearInvalidData.ts index c6e3a71b6b33..d3f85a6a75c9 100644 --- a/projects/app/src/pages/api/admin/clearInvalidData.ts +++ b/projects/app/src/pages/api/admin/clearInvalidData.ts @@ -58,17 +58,18 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) try { await connectToDatabase(); await authCert({ req, authRoot: true }); + const { start = -2, end = -360 * 24 } = req.body as { start: number; end: number }; (async () => { try { console.log('执行脏数据清理任务'); // 360天 ~ 2小时前 - const end = addHours(new Date(), -2); - const start = addHours(new Date(), -360 * 24); - await checkInvalidDatasetFiles(start, end); - await checkInvalidImg(start, end); - await checkInvalidDatasetData(start, end); - await checkInvalidVector(start, end); + const endTime = addHours(new Date(), start); + const startTime = addHours(new Date(), end); + await checkInvalidDatasetFiles(startTime, endTime); + await checkInvalidImg(startTime, endTime); + await checkInvalidDatasetData(startTime, endTime); + await checkInvalidVector(startTime, endTime); console.log('执行脏数据清理任务完毕'); } catch (error) { console.log('执行脏数据清理任务出错了'); diff --git a/projects/app/src/pages/api/admin/initv481.ts b/projects/app/src/pages/api/admin/initv481.ts index 3ef36186871a..bfa3894c6bf8 100644 --- a/projects/app/src/pages/api/admin/initv481.ts +++ b/projects/app/src/pages/api/admin/initv481.ts @@ -141,11 +141,18 @@ async function handler(req: NextApiRequest, res: NextApiResponse) { const collections = await connectionMongo.connection.db .listCollections({ name: 'team.members' }) .toArray(); + if (collections.length > 0) { const sourceCol = connectionMongo.connection.db.collection('team.members'); + const targetCol = connectionMongo.connection.db.collection('team_members'); - await sourceCol.rename('team_members', { dropTarget: true }); - console.log('success rename team.members -> team_members'); + if ((await targetCol.countDocuments()) > 1) { + // 除了root + console.log('team_members 中有数据，无法自动将 buffer.tts 迁移到 team_members，请手动操作'); + } else { + await sourceCol.rename('team_members', { dropTarget: true }); + console.log('success rename team.members -> team_members'); + } } } catch (error) { console.log('error： rename team.members -> team_members', error); @@ -170,6 +177,27 @@ async function handler(req: NextApiRequest, res: NextApiResponse) { console.log('error： rename team.tags -> team_tags', error); } + try { + const collections = await connectionMongo.connection.db + .listCollections({ name: 'team.subscriptions' }) + .toArray(); + if (collections.length > 0) { + const sourceCol = connectionMongo.connection.db.collection('team.subscriptions'); + const targetCol = connectionMongo.connection.db.collection('team_subscriptions'); + + if ((await targetCol.countDocuments()) > 0) { + console.log( + 'team_subscriptions 中有数据，无法自动将 team.subscriptions 迁移到 team_subscriptions，请手动操作' + ); + } else { + await sourceCol.rename('team_subscriptions', { dropTarget: true }); + console.log('success rename team.subscriptions -> team_subscriptions'); + } + } + } catch (error) { + console.log('error： rename team.subscriptions -> team_subscriptions', error); + } + jsonRes(res, { message: 'success' }); diff --git a/projects/app/src/pages/api/common/file/uploadImage.ts b/projects/app/src/pages/api/common/file/uploadImage.ts index e88ac33cc77f..637b70adaf31 100644 --- a/projects/app/src/pages/api/common/file/uploadImage.ts +++ b/projects/app/src/pages/api/common/file/uploadImage.ts @@ -28,7 +28,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) export const config = { api: { - sizeLimit: '10mb', bodyParser: { sizeLimit: '16mb' } diff --git a/projects/app/src/pages/api/core/dataset/allDataset.ts b/projects/app/src/pages/api/core/dataset/allDataset.ts index 0783b178e0ae..2440dda8303c 100644 --- a/projects/app/src/pages/api/core/dataset/allDataset.ts +++ b/projects/app/src/pages/api/core/dataset/allDataset.ts @@ -1,45 +1,31 @@ import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@fastgpt/service/common/response'; -import { connectToDatabase } from '@/service/mongo'; import { MongoDataset } from '@fastgpt/service/core/dataset/schema'; import { getVectorModel } from '@fastgpt/service/core/ai/model'; -import type { DatasetListItemType } from '@fastgpt/global/core/dataset/type.d'; +import type { DatasetSimpleItemType } from '@fastgpt/global/core/dataset/type.d'; import { mongoRPermission } from '@fastgpt/global/support/permission/utils'; import { authUserRole } from '@fastgpt/service/support/permission/auth/user'; import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; +import { NextAPI } from '@/service/middle/entry'; /* get all dataset by teamId or tmbId */ -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - await connectToDatabase(); - // 凭证校验 - const { teamId, tmbId, teamOwner, role } = await authUserRole({ req, authToken: true }); +async function handler( + req: NextApiRequest, + res: NextApiResponse +): Promise { + // 凭证校验 + const { teamId, tmbId, teamOwner, role } = await authUserRole({ req, authToken: true }); - const datasets = await MongoDataset.find({ - ...mongoRPermission({ teamId, tmbId, role }), - type: { $ne: DatasetTypeEnum.folder } - }).lean(); + const datasets = await MongoDataset.find({ + ...mongoRPermission({ teamId, tmbId, role }), + type: { $ne: DatasetTypeEnum.folder } + }).lean(); - const data = datasets.map((item) => ({ - _id: item._id, - parentId: item.parentId, - avatar: item.avatar, - name: item.name, - intro: item.intro, - type: item.type, - permission: item.permission, - vectorModel: getVectorModel(item.vectorModel), - canWrite: String(item.tmbId) === tmbId, - isOwner: teamOwner || String(item.tmbId) === tmbId - })); - - jsonRes(res, { - data - }); - } catch (err) { - jsonRes(res, { - code: 500, - error: err - }); - } + return datasets.map((item) => ({ + _id: item._id, + avatar: item.avatar, + name: item.name, + vectorModel: getVectorModel(item.vectorModel) + })); } + +export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/list.ts b/projects/app/src/pages/api/core/dataset/list.ts index 1bfddb1e36da..f31f7899cbe2 100644 --- a/projects/app/src/pages/api/core/dataset/list.ts +++ b/projects/app/src/pages/api/core/dataset/list.ts @@ -9,7 +9,7 @@ import { getVectorModel } from '@fastgpt/service/core/ai/model'; import { NextAPI } from '@/service/middle/entry'; async function handler(req: NextApiRequest, res: NextApiResponse) { - const { parentId, type } = req.query as { parentId?: string; type?: `${DatasetTypeEnum}` }; + const { parentId, type } = req.query as { parentId?: string; type?: DatasetTypeEnum }; // 凭证校验 const { teamId, tmbId, teamOwner, role, canWrite } = await authUserRole({ req, diff --git a/projects/app/src/pages/api/core/dataset/update.ts b/projects/app/src/pages/api/core/dataset/update.ts index 9d4b5ae4f89d..cdb8a09dc027 100644 --- a/projects/app/src/pages/api/core/dataset/update.ts +++ b/projects/app/src/pages/api/core/dataset/update.ts @@ -8,8 +8,18 @@ import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { id, parentId, name, avatar, intro, permission, agentModel, websiteConfig, status } = - req.body as DatasetUpdateBody; + const { + id, + parentId, + name, + avatar, + intro, + permission, + agentModel, + websiteConfig, + externalReadUrl, + status + } = req.body as DatasetUpdateBody; if (!id) { throw new Error('缺少参数'); @@ -33,7 +43,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ...(agentModel && { agentModel: agentModel.model }), ...(websiteConfig && { websiteConfig }), ...(status && { status }), - ...(intro && { intro }) + ...(intro && { intro }), + ...(externalReadUrl && { externalReadUrl }) } ); diff --git a/projects/app/src/pages/api/core/plugin/list.ts b/projects/app/src/pages/api/core/plugin/list.ts index 94f676b4a557..1d935bbf7e06 100644 --- a/projects/app/src/pages/api/core/plugin/list.ts +++ b/projects/app/src/pages/api/core/plugin/list.ts @@ -9,7 +9,7 @@ import { PluginListItemType } from '@fastgpt/global/core/plugin/controller'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { parentId, type } = req.query as { parentId?: string; type?: `${DatasetTypeEnum}` }; + const { parentId, type } = req.query as { parentId?: string; type?: DatasetTypeEnum }; const { teamId } = await authCert({ req, authToken: true }); diff --git a/projects/app/src/pages/api/core/workflow/debug.ts b/projects/app/src/pages/api/core/workflow/debug.ts index e4749d8b5d07..63c5096e7659 100644 --- a/projects/app/src/pages/api/core/workflow/debug.ts +++ b/projects/app/src/pages/api/core/workflow/debug.ts @@ -82,7 +82,7 @@ export default NextAPI(handler); export const config = { api: { bodyParser: { - sizeLimit: '10mb' + sizeLimit: '20mb' }, responseLimit: '20mb' } diff --git a/projects/app/src/pages/api/plugins/textEditor/v2/index.ts b/projects/app/src/pages/api/plugins/textEditor/v2/index.ts index c01824ce3d76..01477a9ae27a 100644 --- a/projects/app/src/pages/api/plugins/textEditor/v2/index.ts +++ b/projects/app/src/pages/api/plugins/textEditor/v2/index.ts @@ -43,6 +43,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< export const config = { api: { + bodyParser: { + sizeLimit: '16mb' + }, responseLimit: '16mb' } }; diff --git a/projects/app/src/pages/api/v1/chat/completions.ts b/projects/app/src/pages/api/v1/chat/completions.ts index 72c57a9703eb..179b9143bc73 100644 --- a/projects/app/src/pages/api/v1/chat/completions.ts +++ b/projects/app/src/pages/api/v1/chat/completions.ts @@ -523,6 +523,9 @@ const authHeaderRequest = async ({ export const config = { api: { + bodyParser: { + sizeLimit: '20mb' + }, responseLimit: '20mb' } }; diff --git a/projects/app/src/pages/chat/share.tsx b/projects/app/src/pages/chat/share.tsx index f82de422647e..e73586159abb 100644 --- a/projects/app/src/pages/chat/share.tsx +++ b/projects/app/src/pages/chat/share.tsx @@ -269,139 +269,141 @@ const OutLink = ({ }, []); return ( - + <> {appName || chatData.app?.name} - - {showHistory === '1' - ? ((children: React.ReactNode) => { - return isPc ? ( - {children} - ) : ( - + {showHistory === '1' + ? ((children: React.ReactNode) => { + return isPc ? ( + {children} + ) : ( + + + + {children} + + + ); + })( + ({ + id: item.chatId, + title: item.title, + customTitle: item.customTitle, + top: item.top + }))} onClose={onCloseSlider} - > - - - {children} - - - ); - })( - ({ - id: item.chatId, - title: item.title, - customTitle: item.customTitle, - top: item.top - }))} - onClose={onCloseSlider} - onChangeChat={(chatId) => { - router.replace({ - query: { - ...router.query, - chatId: chatId || '' + onChangeChat={(chatId) => { + router.replace({ + query: { + ...router.query, + chatId: chatId || '' + } + }); + if (!isPc) { + onCloseSlider(); } - }); - if (!isPc) { - onCloseSlider(); + }} + onDelHistory={({ chatId }) => + delOneHistory({ appId: chatData.appId, chatId, shareId, outLinkUid }) } - }} - onDelHistory={({ chatId }) => - delOneHistory({ appId: chatData.appId, chatId, shareId, outLinkUid }) - } - onClearHistory={() => { - clearHistories({ shareId, outLinkUid }); - router.replace({ - query: { - ...router.query, - chatId: '' - } - }); - }} - onSetHistoryTop={(e) => { - updateHistory({ - ...e, - appId: chatData.appId, - shareId, - outLinkUid - }); - }} - onSetCustomTitle={async (e) => { - updateHistory({ - appId: chatData.appId, - chatId: e.chatId, - title: e.title, - customTitle: e.title, - shareId, - outLinkUid - }); - }} - /> - ) - : null} + onClearHistory={() => { + clearHistories({ shareId, outLinkUid }); + router.replace({ + query: { + ...router.query, + chatId: '' + } + }); + }} + onSetHistoryTop={(e) => { + updateHistory({ + ...e, + appId: chatData.appId, + shareId, + outLinkUid + }); + }} + onSetCustomTitle={async (e) => { + updateHistory({ + appId: chatData.appId, + chatId: e.chatId, + title: e.title, + customTitle: e.title, + shareId, + outLinkUid + }); + }} + /> + ) + : null} - {/* chat container */} - - {/* header */} - - {/* chat box */} - - + {/* header */} + {}} - onStartChat={startChat} - onDelMessage={(e) => - delOneHistoryItem({ ...e, appId: chatData.appId, chatId, shareId, outLinkUid }) - } - appId={chatData.appId} - chatId={chatId} - shareId={shareId} - outLinkUid={outLinkUid} + appName={chatData.app.name} + history={chatData.history} + showHistory={showHistory === '1'} + onOpenSlider={onOpenSlider} /> - - - - + {/* chat box */} + + {}} + onStartChat={startChat} + onDelMessage={(e) => + delOneHistoryItem({ ...e, appId: chatData.appId, chatId, shareId, outLinkUid }) + } + appId={chatData.appId} + chatId={chatId} + shareId={shareId} + outLinkUid={outLinkUid} + /> + + + + + ); }; diff --git a/projects/app/src/pages/dataset/detail/components/CollectionCard/Context.tsx b/projects/app/src/pages/dataset/detail/components/CollectionCard/Context.tsx new file mode 100644 index 000000000000..6da143e7584d --- /dev/null +++ b/projects/app/src/pages/dataset/detail/components/CollectionCard/Context.tsx @@ -0,0 +1,158 @@ +import { useConfirm } from '@fastgpt/web/hooks/useConfirm'; +import { Dispatch, ReactNode, SetStateAction, useEffect, useState } from 'react'; +import { useTranslation } from 'next-i18next'; +import { createContext, useContextSelector } from 'use-context-selector'; +import { DatasetStatusEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; +import { useRequest } from '@fastgpt/web/hooks/useRequest'; +import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type'; +import { useDisclosure } from '@chakra-ui/react'; +import { checkTeamWebSyncLimit } from '@/web/support/user/team/api'; +import { postCreateTrainingUsage } from '@/web/support/wallet/usage/api'; +import { getDatasetCollections, postWebsiteSync } from '@/web/core/dataset/api'; +import dynamic from 'next/dynamic'; +import { usePagination } from '@fastgpt/web/hooks/usePagination'; +import { DatasetCollectionsListItemType } from '@/global/core/dataset/type'; +import { useRouter } from 'next/router'; +import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; + +const WebSiteConfigModal = dynamic(() => import('./WebsiteConfig')); + +type CollectionPageContextType = { + openWebSyncConfirm: () => void; + onOpenWebsiteModal: () => void; + collections: DatasetCollectionsListItemType[]; + Pagination: () => JSX.Element; + total: number; + getData: (e: number) => void; + isGetting: boolean; + pageNum: number; + pageSize: number; + searchText: string; + setSearchText: Dispatch>; +}; + +export const CollectionPageContext = createContext({ + openWebSyncConfirm: function (): () => void { + throw new Error('Function not implemented.'); + }, + onOpenWebsiteModal: function (): void { + throw new Error('Function not implemented.'); + }, + collections: [], + Pagination: function (): JSX.Element { + throw new Error('Function not implemented.'); + }, + total: 0, + getData: function (e: number): void { + throw new Error('Function not implemented.'); + }, + isGetting: false, + pageNum: 0, + pageSize: 0, + searchText: '', + setSearchText: function (value: SetStateAction): void { + throw new Error('Function not implemented.'); + } +}); + +const CollectionPageContextProvider = ({ children }: { children: ReactNode }) => { + const { t } = useTranslation(); + const router = useRouter(); + const { parentId = '' } = router.query as { parentId: string }; + + const { datasetDetail, datasetId, updateDataset } = useContextSelector( + DatasetPageContext, + (v) => v + ); + + // website config + const { openConfirm: openWebSyncConfirm, ConfirmModal: ConfirmWebSyncModal } = useConfirm({ + content: t('core.dataset.collection.Start Sync Tip') + }); + const { + isOpen: isOpenWebsiteModal, + onOpen: onOpenWebsiteModal, + onClose: onCloseWebsiteModal + } = useDisclosure(); + const { mutate: onUpdateDatasetWebsiteConfig } = useRequest({ + mutationFn: async (websiteConfig: DatasetSchemaType['websiteConfig']) => { + onCloseWebsiteModal(); + await checkTeamWebSyncLimit(); + const billId = await postCreateTrainingUsage({ + name: t('core.dataset.training.Website Sync'), + datasetId: datasetId + }); + await postWebsiteSync({ datasetId: datasetId, billId }); + + await updateDataset({ + id: datasetId, + websiteConfig, + status: DatasetStatusEnum.syncing + }); + + return; + }, + errorToast: t('common.Update Failed') + }); + + // collection list + const [searchText, setSearchText] = useState(''); + const { + data: collections, + Pagination, + total, + getData, + isLoading: isGetting, + pageNum, + pageSize + } = usePagination({ + api: getDatasetCollections, + pageSize: 20, + params: { + datasetId, + parentId, + searchText + }, + defaultRequest: false + }); + useEffect(() => { + getData(1); + }, [parentId]); + + const contextValue: CollectionPageContextType = { + openWebSyncConfirm: openWebSyncConfirm(onUpdateDatasetWebsiteConfig), + onOpenWebsiteModal, + + searchText, + setSearchText, + collections, + Pagination, + total, + getData, + isGetting, + pageNum, + pageSize + }; + + return ( + + {children} + {datasetDetail.type === DatasetTypeEnum.websiteDataset && ( + <> + {isOpenWebsiteModal && ( + + )} + + + )} + + ); +}; +export default CollectionPageContextProvider; diff --git a/projects/app/src/pages/dataset/detail/components/CollectionCard/EmptyCollectionTip.tsx b/projects/app/src/pages/dataset/detail/components/CollectionCard/EmptyCollectionTip.tsx new file mode 100644 index 000000000000..047c6bfa4e4e --- /dev/null +++ b/projects/app/src/pages/dataset/detail/components/CollectionCard/EmptyCollectionTip.tsx @@ -0,0 +1,55 @@ +import EmptyTip from '@fastgpt/web/components/common/EmptyTip'; +import React from 'react'; +import { useTranslation } from 'next-i18next'; +import { DatasetStatusEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; +import { Box, Flex } from '@chakra-ui/react'; +import { useContextSelector } from 'use-context-selector'; +import { CollectionPageContext } from './Context'; +import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; + +const EmptyCollectionTip = () => { + const { t } = useTranslation(); + const onOpenWebsiteModal = useContextSelector(CollectionPageContext, (v) => v.onOpenWebsiteModal); + const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail); + + return ( + <> + {(datasetDetail.type === DatasetTypeEnum.dataset || + datasetDetail.type === DatasetTypeEnum.externalFile) && ( + + )} + {datasetDetail.type === DatasetTypeEnum.websiteDataset && ( + + {datasetDetail.status === DatasetStatusEnum.syncing && ( + <>{t('core.dataset.status.syncing')} + )} + {datasetDetail.status === DatasetStatusEnum.active && ( + <> + {!datasetDetail?.websiteConfig?.url ? ( + <> + {t('core.dataset.collection.Website Empty Tip')} + {', '} + + {t('core.dataset.collection.Click top config website')} + + + ) : ( + <>{t('core.dataset.website.UnValid Website Tip')} + )} + + )} + + } + /> + )} + + ); +}; + +export default EmptyCollectionTip; diff --git a/projects/app/src/pages/dataset/detail/components/CollectionCard/Header.tsx b/projects/app/src/pages/dataset/detail/components/CollectionCard/Header.tsx new file mode 100644 index 000000000000..425c8540915e --- /dev/null +++ b/projects/app/src/pages/dataset/detail/components/CollectionCard/Header.tsx @@ -0,0 +1,399 @@ +import React, { useCallback, useRef } from 'react'; +import { Box, Flex, MenuButton, Button, Link, useTheme, useDisclosure } from '@chakra-ui/react'; +import { + getDatasetCollectionPathById, + postDatasetCollection, + putDatasetCollectionById +} from '@/web/core/dataset/api'; +import { useQuery } from '@tanstack/react-query'; +import { debounce } from 'lodash'; +import { useTranslation } from 'next-i18next'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import MyInput from '@/components/MyInput'; +import { useRequest } from '@fastgpt/web/hooks/useRequest'; +import { useRouter } from 'next/router'; +import { useSystemStore } from '@/web/common/system/useSystemStore'; +import MyMenu from '@fastgpt/web/components/common/MyMenu'; +import { useEditTitle } from '@/web/common/hooks/useEditTitle'; +import { + DatasetCollectionTypeEnum, + TrainingModeEnum, + DatasetTypeEnum, + DatasetTypeMap, + DatasetStatusEnum +} from '@fastgpt/global/core/dataset/constants'; +import EditFolderModal, { useEditFolder } from '../../../component/EditFolderModal'; +import { TabEnum } from '../../index'; +import ParentPath from '@/components/common/ParentPaths'; +import dynamic from 'next/dynamic'; +import { useUserStore } from '@/web/support/user/useUserStore'; +import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant'; + +import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants'; +import { useContextSelector } from 'use-context-selector'; +import { CollectionPageContext } from './Context'; +import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; + +const FileSourceSelector = dynamic(() => import('../Import/components/FileSourceSelector')); + +const Header = ({}: {}) => { + const { t } = useTranslation(); + const theme = useTheme(); + const { setLoading } = useSystemStore(); + const { userInfo } = useUserStore(); + const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail); + + const router = useRouter(); + const { parentId = '' } = router.query as { parentId: string; datasetId: string }; + const { isPc } = useSystemStore(); + + const lastSearch = useRef(''); + const { searchText, setSearchText, total, getData, pageNum, onOpenWebsiteModal } = + useContextSelector(CollectionPageContext, (v) => v); + + // change search + const debounceRefetch = useCallback( + debounce(() => { + getData(1); + lastSearch.current = searchText; + }, 300), + [] + ); + + const { data: paths = [] } = useQuery(['getDatasetCollectionPathById', parentId], () => + getDatasetCollectionPathById(parentId) + ); + + const { editFolderData, setEditFolderData } = useEditFolder(); + const { onOpenModal: onOpenCreateVirtualFileModal, EditModal: EditCreateVirtualFileModal } = + useEditTitle({ + title: t('dataset.Create manual collection'), + tip: t('dataset.Manual collection Tip'), + canEmpty: false + }); + const { + isOpen: isOpenFileSourceSelector, + onOpen: onOpenFileSourceSelector, + onClose: onCloseFileSourceSelector + } = useDisclosure(); + const { mutate: onCreateCollection } = useRequest({ + mutationFn: async ({ + name, + type, + callback, + ...props + }: { + name: string; + type: `${DatasetCollectionTypeEnum}`; + callback?: (id: string) => void; + trainingType?: TrainingModeEnum; + rawLink?: string; + chunkSize?: number; + }) => { + setLoading(true); + const id = await postDatasetCollection({ + parentId, + datasetId: datasetDetail._id, + name, + type, + ...props + }); + callback?.(id); + return id; + }, + onSuccess() { + getData(pageNum); + }, + onSettled() { + setLoading(false); + }, + + successToast: t('common.Create Success'), + errorToast: t('common.Create Failed') + }); + + return ( + + + ({ + parentId: path.parentId, + parentName: i === paths.length - 1 ? `${path.parentName}` : path.parentName + }))} + FirstPathDom={ + <> + + {t(DatasetTypeMap[datasetDetail?.type]?.collectionLabel)}({total}) + + {datasetDetail?.websiteConfig?.url && ( + + {t('core.dataset.website.Base Url')}: + + {datasetDetail.websiteConfig.url} + + + )} + + } + onClick={(e) => { + router.replace({ + query: { + ...router.query, + parentId: e + } + }); + }} + /> + + + {/* search input */} + {isPc && ( + + + } + onChange={(e) => { + setSearchText(e.target.value); + debounceRefetch(); + }} + onBlur={() => { + if (searchText === lastSearch.current) return; + getData(1); + }} + onKeyDown={(e) => { + if (searchText === lastSearch.current) return; + if (e.key === 'Enter') { + getData(1); + } + }} + /> + + )} + + {/* diff collection button */} + {userInfo?.team?.role !== TeamMemberRoleEnum.visitor && ( + <> + {datasetDetail?.type === DatasetTypeEnum.dataset && ( + + + + {t('dataset.collections.Create And Import')} + + + } + menuList={[ + { + label: ( + + + {t('Folder')} + + ), + onClick: () => setEditFolderData({}) + }, + { + label: ( + + + {t('core.dataset.Manual collection')} + + ), + onClick: () => { + onOpenCreateVirtualFileModal({ + defaultVal: '', + onSuccess: (name) => { + onCreateCollection({ name, type: DatasetCollectionTypeEnum.virtual }); + } + }); + } + }, + { + label: ( + + + {t('core.dataset.Text collection')} + + ), + onClick: onOpenFileSourceSelector + }, + { + label: ( + + + {t('core.dataset.Table collection')} + + ), + onClick: () => + router.replace({ + query: { + ...router.query, + currentTab: TabEnum.import, + source: ImportDataSourceEnum.csvTable + } + }) + } + ]} + /> + )} + {datasetDetail?.type === DatasetTypeEnum.websiteDataset && ( + <> + {datasetDetail?.websiteConfig?.url ? ( + + {datasetDetail.status === DatasetStatusEnum.active && ( + + )} + {datasetDetail.status === DatasetStatusEnum.syncing && ( + + + + {t('core.dataset.status.syncing')} + + + )} + + ) : ( + + )} + + )} + {datasetDetail?.type === DatasetTypeEnum.externalFile && ( + + + + {t('dataset.collections.Create And Import')} + + + } + menuList={[ + { + label: ( + + + {t('Folder')} + + ), + onClick: () => setEditFolderData({}) + }, + { + label: ( + + + {t('core.dataset.Text collection')} + + ), + onClick: () => + router.replace({ + query: { + ...router.query, + currentTab: TabEnum.import, + source: ImportDataSourceEnum.externalFile + } + }) + } + ]} + /> + )} + + )} + + {/* modal */} + {!!editFolderData && ( + setEditFolderData(undefined)} + editCallback={async (name) => { + try { + if (editFolderData.id) { + await putDatasetCollectionById({ + id: editFolderData.id, + name + }); + getData(pageNum); + } else { + onCreateCollection({ + name, + type: DatasetCollectionTypeEnum.folder + }); + } + } catch (error) { + return Promise.reject(error); + } + }} + isEdit={!!editFolderData.id} + name={editFolderData.name} + /> + )} + + {isOpenFileSourceSelector && } + + ); +}; + +export default Header; diff --git a/projects/app/src/pages/dataset/detail/components/Import/WebsiteConfig.tsx b/projects/app/src/pages/dataset/detail/components/CollectionCard/WebsiteConfig.tsx similarity index 100% rename from projects/app/src/pages/dataset/detail/components/Import/WebsiteConfig.tsx rename to projects/app/src/pages/dataset/detail/components/CollectionCard/WebsiteConfig.tsx diff --git a/projects/app/src/pages/dataset/detail/components/CollectionCard.tsx b/projects/app/src/pages/dataset/detail/components/CollectionCard/index.tsx similarity index 52% rename from projects/app/src/pages/dataset/detail/components/CollectionCard.tsx rename to projects/app/src/pages/dataset/detail/components/CollectionCard/index.tsx index 87896ef17cde..1bb48595327d 100644 --- a/projects/app/src/pages/dataset/detail/components/CollectionCard.tsx +++ b/projects/app/src/pages/dataset/detail/components/CollectionCard/index.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useState, useRef, useMemo, useEffect } from 'react'; +import React, { useState, useRef, useMemo } from 'react'; import { Box, Flex, @@ -9,47 +9,29 @@ import { Th, Td, Tbody, - Image, - MenuButton, - useDisclosure, - Button, - Link, - useTheme + MenuButton } from '@chakra-ui/react'; import { - getDatasetCollections, delDatasetCollectionById, putDatasetCollectionById, - postDatasetCollection, - getDatasetCollectionPathById, postLinkCollectionSync } from '@/web/core/dataset/api'; import { useQuery } from '@tanstack/react-query'; -import { debounce } from 'lodash'; import { useConfirm } from '@fastgpt/web/hooks/useConfirm'; import { useTranslation } from 'next-i18next'; import MyIcon from '@fastgpt/web/components/common/Icon'; -import MyInput from '@/components/MyInput'; import dayjs from 'dayjs'; import { useRequest } from '@fastgpt/web/hooks/useRequest'; import { useRouter } from 'next/router'; -import { useSystemStore } from '@/web/common/system/useSystemStore'; import MyMenu from '@fastgpt/web/components/common/MyMenu'; import { useEditTitle } from '@/web/common/hooks/useEditTitle'; -import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.d'; -import EmptyTip from '@fastgpt/web/components/common/EmptyTip'; import { DatasetCollectionTypeEnum, - TrainingModeEnum, - DatasetTypeEnum, - DatasetTypeMap, DatasetStatusEnum, DatasetCollectionSyncResultMap } from '@fastgpt/global/core/dataset/constants'; import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils'; -import EditFolderModal, { useEditFolder } from '../../component/EditFolderModal'; -import { TabEnum } from '..'; -import ParentPath from '@/components/common/ParentPaths'; +import { TabEnum } from '../../index'; import dynamic from 'next/dynamic'; import { useDrag } from '@/web/common/hooks/useDrag'; import SelectCollections from '@/web/core/dataset/components/SelectCollections'; @@ -58,27 +40,22 @@ import MyTooltip from '@/components/MyTooltip'; import { useUserStore } from '@/web/support/user/useUserStore'; import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant'; import { useDatasetStore } from '@/web/core/dataset/store/dataset'; -import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type'; import { DatasetCollectionSyncResultEnum } from '@fastgpt/global/core/dataset/constants'; import MyBox from '@fastgpt/web/components/common/MyBox'; -import { usePagination } from '@fastgpt/web/hooks/usePagination'; -import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants'; +import { useContextSelector } from 'use-context-selector'; +import { CollectionPageContext } from './Context'; +import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; -const WebSiteConfigModal = dynamic(() => import('./Import/WebsiteConfig'), {}); -const FileSourceSelector = dynamic(() => import('./Import/components/FileSourceSelector'), {}); +const Header = dynamic(() => import('./Header')); +const EmptyCollectionTip = dynamic(() => import('./EmptyCollectionTip')); const CollectionCard = () => { const BoxRef = useRef(null); - const lastSearch = useRef(''); const router = useRouter(); - const theme = useTheme(); const { toast } = useToast(); - const { parentId = '', datasetId } = router.query as { parentId: string; datasetId: string }; const { t } = useTranslation(); - const { isPc } = useSystemStore(); const { userInfo } = useUserStore(); - const [searchText, setSearchText] = useState(''); - const { datasetDetail, updateDataset, startWebsiteSync, loadDatasetDetail } = useDatasetStore(); + const { datasetDetail, loadDatasetDetail } = useContextSelector(DatasetPageContext, (v) => v); const { openConfirm: openDeleteConfirm, ConfirmModal: ConfirmDeleteModal } = useConfirm({ content: t('dataset.Confirm to delete the file'), @@ -88,66 +65,18 @@ const CollectionCard = () => { content: t('core.dataset.collection.Start Sync Tip') }); - const { - isOpen: isOpenFileSourceSelector, - onOpen: onOpenFileSourceSelector, - onClose: onCloseFileSourceSelector - } = useDisclosure(); - const { - isOpen: isOpenWebsiteModal, - onOpen: onOpenWebsiteModal, - onClose: onCloseWebsiteModal - } = useDisclosure(); - const { onOpenModal: onOpenCreateVirtualFileModal, EditModal: EditCreateVirtualFileModal } = - useEditTitle({ - title: t('dataset.Create manual collection'), - tip: t('dataset.Manual collection Tip'), - canEmpty: false - }); - const { onOpenModal: onOpenEditTitleModal, EditModal: EditTitleModal } = useEditTitle({ title: t('Rename') }); - const { editFolderData, setEditFolderData } = useEditFolder(); const [moveCollectionData, setMoveCollectionData] = useState<{ collectionId: string }>(); - const { - data: collections, - Pagination, - total, - getData, - isLoading: isGetting, - pageNum, - pageSize - } = usePagination({ - api: getDatasetCollections, - pageSize: 20, - params: { - datasetId, - parentId, - searchText - }, - defaultRequest: false, - onChange() { - if (BoxRef.current) { - BoxRef.current.scrollTop = 0; - } - } - }); + const { collections, Pagination, total, getData, isGetting, pageNum, pageSize } = + useContextSelector(CollectionPageContext, (v) => v); const { dragStartId, setDragStartId, dragTargetId, setDragTargetId } = useDrag(); - // change search - const debounceRefetch = useCallback( - debounce(() => { - getData(1); - lastSearch.current = searchText; - }, 300), - [] - ); - - // add file icon + // Ad file status icon const formatCollections = useMemo( () => collections.map((collection) => { @@ -180,37 +109,6 @@ const CollectionCard = () => { [collections, t] ); - const { mutate: onCreateCollection, isLoading: isCreating } = useRequest({ - mutationFn: async ({ - name, - type, - callback, - ...props - }: { - name: string; - type: `${DatasetCollectionTypeEnum}`; - callback?: (id: string) => void; - trainingType?: `${TrainingModeEnum}`; - rawLink?: string; - chunkSize?: number; - }) => { - const id = await postDatasetCollection({ - parentId, - datasetId, - name, - type, - ...props - }); - callback?.(id); - return id; - }, - onSuccess() { - getData(pageNum); - }, - - successToast: t('common.Create Success'), - errorToast: t('common.Create Failed') - }); const { mutate: onUpdateCollectionName } = useRequest({ mutationFn: ({ collectionId, name }: { collectionId: string; name: string }) => { return putDatasetCollectionById({ @@ -237,17 +135,7 @@ const CollectionCard = () => { successToast: t('common.Delete Success'), errorToast: t('common.Delete Failed') }); - const { mutate: onUpdateDatasetWebsiteConfig, isLoading: isUpdating } = useRequest({ - mutationFn: async (websiteConfig: DatasetSchemaType['websiteConfig']) => { - onCloseWebsiteModal(); - await updateDataset({ - id: datasetDetail._id, - websiteConfig - }); - return startWebsiteSync(); - }, - errorToast: t('common.Update Failed') - }); + const { mutate: onclickStartSync, isLoading: isSyncing } = useRequest({ mutationFn: (collectionId: string) => { return postLinkCollectionSync(collectionId); @@ -262,22 +150,13 @@ const CollectionCard = () => { errorToast: t('core.dataset.error.Start Sync Failed') }); - const { data: paths = [] } = useQuery(['getDatasetCollectionPathById', parentId], () => - getDatasetCollectionPathById(parentId) - ); - const hasTrainingData = useMemo( () => !!formatCollections.find((item) => item.trainingAmount > 0), [formatCollections] ); const isLoading = useMemo( - () => - isCreating || - isDeleting || - isUpdating || - isSyncing || - (isGetting && collections.length === 0), - [collections.length, isCreating, isDeleting, isGetting, isSyncing, isUpdating] + () => isDeleting || isSyncing || (isGetting && collections.length === 0), + [collections.length, isDeleting, isGetting, isSyncing] ); useQuery( @@ -285,7 +164,7 @@ const CollectionCard = () => { () => { getData(1); if (datasetDetail.status === DatasetStatusEnum.syncing) { - loadDatasetDetail(datasetId, true); + loadDatasetDetail(datasetDetail._id); } return null; }, @@ -295,207 +174,11 @@ const CollectionCard = () => { } ); - useEffect(() => { - getData(1); - }, [parentId]); - return ( {/* header */} - - - ({ - parentId: path.parentId, - parentName: i === paths.length - 1 ? `${path.parentName}` : path.parentName - }))} - FirstPathDom={ - <> - - {t(DatasetTypeMap[datasetDetail?.type]?.collectionLabel)}({total}) - - {datasetDetail?.websiteConfig?.url && ( - - {t('core.dataset.website.Base Url')}: - - {datasetDetail.websiteConfig.url} - - - )} - - } - onClick={(e) => { - router.replace({ - query: { - ...router.query, - parentId: e - } - }); - }} - /> - - - {isPc && ( - - - } - onChange={(e) => { - setSearchText(e.target.value); - debounceRefetch(); - }} - onBlur={() => { - if (searchText === lastSearch.current) return; - getData(1); - }} - onKeyDown={(e) => { - if (searchText === lastSearch.current) return; - if (e.key === 'Enter') { - getData(1); - } - }} - /> - - )} - {datasetDetail?.type === DatasetTypeEnum.dataset && ( - <> - {userInfo?.team?.role !== TeamMemberRoleEnum.visitor && ( - - - - {t('dataset.collections.Create And Import')} - - - } - menuList={[ - { - label: ( - - - {t('Folder')} - - ), - onClick: () => setEditFolderData({}) - }, - { - label: ( - - - {t('core.dataset.Manual collection')} - - ), - onClick: () => { - onOpenCreateVirtualFileModal({ - defaultVal: '', - onSuccess: (name) => { - onCreateCollection({ name, type: DatasetCollectionTypeEnum.virtual }); - } - }); - } - }, - { - label: ( - - - {t('core.dataset.Text collection')} - - ), - onClick: onOpenFileSourceSelector - }, - { - label: ( - - - {t('core.dataset.Table collection')} - - ), - onClick: () => - router.replace({ - query: { - ...router.query, - currentTab: TabEnum.import, - source: ImportDataSourceEnum.csvTable - } - }) - } - ]} - /> - )} - - )} - {datasetDetail?.type === DatasetTypeEnum.websiteDataset && ( - <> - {datasetDetail?.websiteConfig?.url ? ( - - {datasetDetail.status === DatasetStatusEnum.active && ( - - )} - {datasetDetail.status === DatasetStatusEnum.syncing && ( - - - - {t('core.dataset.status.syncing')} - - - )} - - ) : ( - - )} - - )} - +
{/* collection table */} { )} - {total === 0 && ( - - {datasetDetail.status === DatasetStatusEnum.syncing && ( - <>{t('core.dataset.status.syncing')} - )} - {datasetDetail.status === DatasetStatusEnum.active && ( - <> - {!datasetDetail?.websiteConfig?.url ? ( - <> - {t('core.dataset.collection.Website Empty Tip')} - {', '} - - {t('core.dataset.collection.Click top config website')} - - - ) : ( - <>{t('core.dataset.website.UnValid Website Tip')} - )} - - )} - - ) - } - /> - )} + {total === 0 && } - - {/* {isOpenFileImportModal && ( - { - getData(1); - onCloseFileImportModal(); - }} - onClose={onCloseFileImportModal} - /> - )} */} - {isOpenFileSourceSelector && } - {!!editFolderData && ( - setEditFolderData(undefined)} - editCallback={async (name) => { - try { - if (editFolderData.id) { - await putDatasetCollectionById({ - id: editFolderData.id, - name - }); - getData(pageNum); - } else { - onCreateCollection({ - name, - type: DatasetCollectionTypeEnum.folder - }); - } - } catch (error) { - return Promise.reject(error); - } - }} - isEdit={!!editFolderData.id} - name={editFolderData.name} - /> - )} + {!!moveCollectionData && ( setMoveCollectionData(undefined)} @@ -828,16 +441,6 @@ const CollectionCard = () => { }} /> )} - {isOpenWebsiteModal && ( - - )} ); diff --git a/projects/app/src/pages/dataset/detail/components/Import/Context.tsx b/projects/app/src/pages/dataset/detail/components/Import/Context.tsx new file mode 100644 index 000000000000..6074df1f28d5 --- /dev/null +++ b/projects/app/src/pages/dataset/detail/components/Import/Context.tsx @@ -0,0 +1,302 @@ +import { useRouter } from 'next/router'; +import { SetStateAction, useState } from 'react'; +import { useTranslation } from 'next-i18next'; +import { createContext, useContextSelector } from 'use-context-selector'; +import { ImportDataSourceEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; +import { useMyStep } from '@fastgpt/web/hooks/useStep'; +import { Box, Button, Flex, IconButton } from '@chakra-ui/react'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import { TabEnum } from '../Slider'; +import { ImportProcessWayEnum } from '@/web/core/dataset/constants'; +import { UseFormReturn, useForm } from 'react-hook-form'; +import { ImportSourceItemType } from '@/web/core/dataset/type'; +import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent'; +import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; + +type TrainingFiledType = { + chunkOverlapRatio: number; + maxChunkSize: number; + minChunkSize: number; + autoChunkSize: number; + chunkSize: number; + showChunkInput: boolean; + showPromptInput: boolean; + charsPointsPrice: number; + priceTip: string; + uploadRate: number; + chunkSizeField?: ChunkSizeFieldType; +}; +type DatasetImportContextType = { + importSource: ImportDataSourceEnum; + parentId: string | undefined; + activeStep: number; + goToNext: () => void; + + processParamsForm: UseFormReturn; + sources: ImportSourceItemType[]; + setSources: React.Dispatch>; +} & TrainingFiledType; + +type ChunkSizeFieldType = 'embeddingChunkSize'; +export type ImportFormType = { + mode: TrainingModeEnum; + way: ImportProcessWayEnum; + embeddingChunkSize: number; + customSplitChar: string; + qaPrompt: string; + webSelector: string; +}; + +export const DatasetImportContext = createContext({ + importSource: ImportDataSourceEnum.fileLocal, + goToNext: function (): void { + throw new Error('Function not implemented.'); + }, + activeStep: 0, + parentId: undefined, + + maxChunkSize: 0, + minChunkSize: 0, + showChunkInput: false, + showPromptInput: false, + sources: [], + setSources: function (value: SetStateAction): void { + throw new Error('Function not implemented.'); + }, + chunkSize: 0, + chunkOverlapRatio: 0, + uploadRate: 0, + //@ts-ignore + processParamsForm: undefined, + autoChunkSize: 0, + charsPointsPrice: 0, + priceTip: '' +}); + +const DatasetImportContextProvider = ({ children }: { children: React.ReactNode }) => { + const { t } = useTranslation(); + const router = useRouter(); + const { source = ImportDataSourceEnum.fileLocal, parentId } = (router.query || {}) as { + source: ImportDataSourceEnum; + parentId?: string; + }; + + const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail); + + // step + const modeSteps: Record = { + [ImportDataSourceEnum.fileLocal]: [ + { + title: t('core.dataset.import.Select file') + }, + { + title: t('core.dataset.import.Data Preprocessing') + }, + { + title: t('core.dataset.import.Upload data') + } + ], + [ImportDataSourceEnum.fileLink]: [ + { + title: t('core.dataset.import.Select file') + }, + { + title: t('core.dataset.import.Data Preprocessing') + }, + { + title: t('core.dataset.import.Upload data') + } + ], + [ImportDataSourceEnum.fileCustom]: [ + { + title: t('core.dataset.import.Select file') + }, + { + title: t('core.dataset.import.Data Preprocessing') + }, + { + title: t('core.dataset.import.Upload data') + } + ], + [ImportDataSourceEnum.csvTable]: [ + { + title: t('core.dataset.import.Select file') + }, + { + title: t('core.dataset.import.Data Preprocessing') + }, + { + title: t('core.dataset.import.Upload data') + } + ], + [ImportDataSourceEnum.externalFile]: [ + { + title: t('core.dataset.import.Select file') + }, + { + title: t('core.dataset.import.Data Preprocessing') + }, + { + title: t('core.dataset.import.Upload data') + } + ] + }; + const steps = modeSteps[source]; + const { activeStep, goToNext, goToPrevious, MyStep } = useMyStep({ + defaultStep: 0, + steps + }); + + // ----- + const vectorModel = datasetDetail.vectorModel; + const agentModel = datasetDetail.agentModel; + + const processParamsForm = useForm({ + defaultValues: { + mode: TrainingModeEnum.chunk, + way: ImportProcessWayEnum.auto, + embeddingChunkSize: vectorModel?.defaultToken || 512, + customSplitChar: '', + qaPrompt: Prompt_AgentQA.description, + webSelector: '' + } + }); + + const [sources, setSources] = useState([]); + + // watch form + const mode = processParamsForm.watch('mode'); + const way = processParamsForm.watch('way'); + const embeddingChunkSize = processParamsForm.watch('embeddingChunkSize'); + const customSplitChar = processParamsForm.watch('customSplitChar'); + + const modeStaticParams: Record = { + [TrainingModeEnum.auto]: { + chunkOverlapRatio: 0.2, + maxChunkSize: 2048, + minChunkSize: 100, + autoChunkSize: vectorModel?.defaultToken ? vectorModel?.defaultToken * 2 : 1024, + chunkSize: vectorModel?.defaultToken ? vectorModel?.defaultToken * 2 : 1024, + showChunkInput: false, + showPromptInput: false, + charsPointsPrice: agentModel.charsPointsPrice, + priceTip: t('core.dataset.import.Auto mode Estimated Price Tips', { + price: agentModel.charsPointsPrice + }), + uploadRate: 100 + }, + [TrainingModeEnum.chunk]: { + chunkSizeField: 'embeddingChunkSize' as ChunkSizeFieldType, + chunkOverlapRatio: 0.2, + maxChunkSize: vectorModel?.maxToken || 512, + minChunkSize: 100, + autoChunkSize: vectorModel?.defaultToken || 512, + chunkSize: embeddingChunkSize, + showChunkInput: true, + showPromptInput: false, + charsPointsPrice: vectorModel.charsPointsPrice, + priceTip: t('core.dataset.import.Embedding Estimated Price Tips', { + price: vectorModel.charsPointsPrice + }), + uploadRate: 150 + }, + [TrainingModeEnum.qa]: { + chunkOverlapRatio: 0, + maxChunkSize: 8000, + minChunkSize: 3000, + autoChunkSize: agentModel.maxContext * 0.55 || 6000, + chunkSize: agentModel.maxContext * 0.55 || 6000, + showChunkInput: false, + showPromptInput: true, + charsPointsPrice: agentModel.charsPointsPrice, + priceTip: t('core.dataset.import.QA Estimated Price Tips', { + price: agentModel?.charsPointsPrice + }), + uploadRate: 30 + } + }; + const selectModelStaticParam = modeStaticParams[mode]; + + const wayStaticPrams = { + [ImportProcessWayEnum.auto]: { + chunkSize: selectModelStaticParam.autoChunkSize, + customSplitChar: '' + }, + [ImportProcessWayEnum.custom]: { + chunkSize: modeStaticParams[mode].chunkSize, + customSplitChar + } + }; + + const chunkSize = wayStaticPrams[way].chunkSize; + + const contextValue = { + importSource: source, + parentId, + activeStep, + goToNext, + + processParamsForm, + ...selectModelStaticParam, + sources, + setSources, + chunkSize + }; + + return ( + + + {activeStep === 0 ? ( + + } + aria-label={''} + size={'smSquare'} + w={'26px'} + h={'26px'} + borderRadius={'50%'} + variant={'whiteBase'} + mr={2} + onClick={() => + router.replace({ + query: { + ...router.query, + currentTab: TabEnum.collectionCard + } + }) + } + /> + {t('common.Exit')} + + ) : ( + + )} + + + {/* step */} + + + + + + {children} + + ); +}; + +export default DatasetImportContextProvider; diff --git a/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx b/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx deleted file mode 100644 index 228e6684a298..000000000000 --- a/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx +++ /dev/null @@ -1,165 +0,0 @@ -import React, { useContext, createContext, useState, useMemo, useEffect } from 'react'; - -import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants'; -import { useTranslation } from 'next-i18next'; -import { DatasetItemType } from '@fastgpt/global/core/dataset/type'; -import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent'; -import { UseFormReturn, useForm } from 'react-hook-form'; -import { ImportProcessWayEnum } from '@/web/core/dataset/constants'; -import { ImportSourceItemType } from '@/web/core/dataset/type'; -import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants'; - -type ChunkSizeFieldType = 'embeddingChunkSize'; -export type FormType = { - mode: `${TrainingModeEnum}`; - way: `${ImportProcessWayEnum}`; - embeddingChunkSize: number; - customSplitChar: string; - qaPrompt: string; - webSelector: string; -}; - -type useImportStoreType = { - parentId?: string; - processParamsForm: UseFormReturn; - chunkSizeField?: ChunkSizeFieldType; - maxChunkSize: number; - minChunkSize: number; - showChunkInput: boolean; - showPromptInput: boolean; - sources: ImportSourceItemType[]; - setSources: React.Dispatch>; - chunkSize: number; - chunkOverlapRatio: number; - priceTip: string; - uploadRate: number; - importSource: `${ImportDataSourceEnum}`; -}; -const StateContext = createContext({ - processParamsForm: {} as any, - sources: [], - setSources: function (value: React.SetStateAction): void { - throw new Error('Function not implemented.'); - }, - maxChunkSize: 0, - minChunkSize: 0, - showChunkInput: false, - showPromptInput: false, - chunkSizeField: 'embeddingChunkSize', - chunkSize: 0, - chunkOverlapRatio: 0, - priceTip: '', - uploadRate: 50, - importSource: ImportDataSourceEnum.fileLocal -}); - -export const useImportStore = () => useContext(StateContext); - -const Provider = ({ - importSource, - dataset, - parentId, - children -}: { - importSource: `${ImportDataSourceEnum}`; - dataset: DatasetItemType; - parentId?: string; - children: React.ReactNode; -}) => { - const vectorModel = dataset.vectorModel; - const agentModel = dataset.agentModel; - - const processParamsForm = useForm({ - defaultValues: { - mode: TrainingModeEnum.chunk, - way: ImportProcessWayEnum.auto, - embeddingChunkSize: vectorModel?.defaultToken || 512, - customSplitChar: '', - qaPrompt: Prompt_AgentQA.description, - webSelector: '' - } - }); - - const { t } = useTranslation(); - const [sources, setSources] = useState([]); - - // watch form - const mode = processParamsForm.watch('mode'); - const way = processParamsForm.watch('way'); - const embeddingChunkSize = processParamsForm.watch('embeddingChunkSize'); - const customSplitChar = processParamsForm.watch('customSplitChar'); - - const modeStaticParams = { - [TrainingModeEnum.auto]: { - chunkOverlapRatio: 0.2, - maxChunkSize: 2048, - minChunkSize: 100, - autoChunkSize: vectorModel?.defaultToken ? vectorModel?.defaultToken * 2 : 1024, - chunkSize: vectorModel?.defaultToken ? vectorModel?.defaultToken * 2 : 1024, - showChunkInput: false, - showPromptInput: false, - charsPointsPrice: agentModel.charsPointsPrice, - priceTip: t('core.dataset.import.Auto mode Estimated Price Tips', { - price: agentModel.charsPointsPrice - }), - uploadRate: 100 - }, - [TrainingModeEnum.chunk]: { - chunkSizeField: 'embeddingChunkSize' as ChunkSizeFieldType, - chunkOverlapRatio: 0.2, - maxChunkSize: vectorModel?.maxToken || 512, - minChunkSize: 100, - autoChunkSize: vectorModel?.defaultToken || 512, - chunkSize: embeddingChunkSize, - showChunkInput: true, - showPromptInput: false, - charsPointsPrice: vectorModel.charsPointsPrice, - priceTip: t('core.dataset.import.Embedding Estimated Price Tips', { - price: vectorModel.charsPointsPrice - }), - uploadRate: 150 - }, - [TrainingModeEnum.qa]: { - chunkOverlapRatio: 0, - maxChunkSize: 8000, - minChunkSize: 3000, - autoChunkSize: agentModel.maxContext * 0.55 || 6000, - chunkSize: agentModel.maxContext * 0.55 || 6000, - showChunkInput: false, - showPromptInput: true, - charsPointsPrice: agentModel.charsPointsPrice, - priceTip: t('core.dataset.import.QA Estimated Price Tips', { - price: agentModel?.charsPointsPrice - }), - uploadRate: 30 - } - }; - const selectModelStaticParam = useMemo(() => modeStaticParams[mode], [mode]); - - const wayStaticPrams = { - [ImportProcessWayEnum.auto]: { - chunkSize: selectModelStaticParam.autoChunkSize, - customSplitChar: '' - }, - [ImportProcessWayEnum.custom]: { - chunkSize: modeStaticParams[mode].chunkSize, - customSplitChar - } - }; - - const chunkSize = wayStaticPrams[way].chunkSize; - - const value: useImportStoreType = { - parentId, - processParamsForm, - ...selectModelStaticParam, - sources, - setSources, - chunkSize, - - importSource - }; - return {children}; -}; - -export default React.memo(Provider); diff --git a/projects/app/src/pages/dataset/detail/components/Import/commonProgress/DataProcess.tsx b/projects/app/src/pages/dataset/detail/components/Import/commonProgress/DataProcess.tsx index 99644e8bd16c..e0987b6571be 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/commonProgress/DataProcess.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/commonProgress/DataProcess.tsx @@ -20,23 +20,20 @@ import LeftRadio from '@fastgpt/web/components/common/Radio/LeftRadio'; import { TrainingTypeMap } from '@fastgpt/global/core/dataset/constants'; import { ImportProcessWayEnum } from '@/web/core/dataset/constants'; import MyTooltip from '@/components/MyTooltip'; -import { useImportStore } from '../Provider'; import { useSystemStore } from '@/web/common/system/useSystemStore'; import MyModal from '@fastgpt/web/components/common/MyModal'; import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent'; import Preview from '../components/Preview'; import Tag from '@fastgpt/web/components/common/Tag/index'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetImportContext } from '../Context'; -function DataProcess({ - showPreviewChunks = true, - goToNext -}: { - showPreviewChunks: boolean; - goToNext: () => void; -}) { +function DataProcess({ showPreviewChunks = true }: { showPreviewChunks: boolean }) { const { t } = useTranslation(); const { feConfigs } = useSystemStore(); + const { + goToNext, processParamsForm, chunkSizeField, minChunkSize, @@ -44,7 +41,7 @@ function DataProcess({ showPromptInput, maxChunkSize, priceTip - } = useImportStore(); + } = useContextSelector(DatasetImportContext, (v) => v); const { getValues, setValue, register } = processParamsForm; const [refresh, setRefresh] = useState(false); diff --git a/projects/app/src/pages/dataset/detail/components/Import/commonProgress/PreviewData.tsx b/projects/app/src/pages/dataset/detail/components/Import/commonProgress/PreviewData.tsx index 0c2070bd1c21..f018beb7ebd6 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/commonProgress/PreviewData.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/commonProgress/PreviewData.tsx @@ -2,15 +2,12 @@ import React from 'react'; import Preview from '../components/Preview'; import { Box, Button, Flex } from '@chakra-ui/react'; import { useTranslation } from 'next-i18next'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetImportContext } from '../Context'; -const PreviewData = ({ - showPreviewChunks, - goToNext -}: { - showPreviewChunks: boolean; - goToNext: () => void; -}) => { +const PreviewData = ({ showPreviewChunks }: { showPreviewChunks: boolean }) => { const { t } = useTranslation(); + const goToNext = useContextSelector(DatasetImportContext, (v) => v.goToNext); return ( diff --git a/projects/app/src/pages/dataset/detail/components/Import/commonProgress/Upload.tsx b/projects/app/src/pages/dataset/detail/components/Import/commonProgress/Upload.tsx index c7b99fa5fb8c..f5b441ff9a30 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/commonProgress/Upload.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/commonProgress/Upload.tsx @@ -11,7 +11,6 @@ import { Flex, Button } from '@chakra-ui/react'; -import { useImportStore, type FormType } from '../Provider'; import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants'; import { useTranslation } from 'next-i18next'; import MyIcon from '@fastgpt/web/components/common/Icon'; @@ -28,20 +27,23 @@ import { } from '@/web/core/dataset/api'; import Tag from '@fastgpt/web/components/common/Tag/index'; import { useI18n } from '@/web/context/I18n'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; +import { DatasetImportContext, type ImportFormType } from '../Context'; const Upload = () => { const { t } = useTranslation(); const { fileT } = useI18n(); const { toast } = useToast(); const router = useRouter(); - const { datasetDetail } = useDatasetStore(); + const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail); const { importSource, parentId, sources, setSources, processParamsForm, chunkSize } = - useImportStore(); + useContextSelector(DatasetImportContext, (v) => v); const { handleSubmit } = processParamsForm; const { mutate: startUpload, isLoading } = useRequest({ - mutationFn: async ({ mode, customSplitChar, qaPrompt, webSelector }: FormType) => { + mutationFn: async ({ mode, customSplitChar, qaPrompt, webSelector }: ImportFormType) => { if (sources.length === 0) return; const filterWaitingSources = sources.filter((item) => item.createStatus === 'waiting'); diff --git a/projects/app/src/pages/dataset/detail/components/Import/components/FileSourceSelector.tsx b/projects/app/src/pages/dataset/detail/components/Import/components/FileSourceSelector.tsx index 38af361d8a98..e6726ddb3063 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/components/FileSourceSelector.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/components/FileSourceSelector.tsx @@ -10,7 +10,7 @@ import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants'; const FileModeSelector = ({ onClose }: { onClose: () => void }) => { const { t } = useTranslation(); const router = useRouter(); - const [value, setValue] = useState<`${ImportDataSourceEnum}`>(ImportDataSourceEnum.fileLocal); + const [value, setValue] = useState(ImportDataSourceEnum.fileLocal); return ( import('./PreviewRawText')); const PreviewChunks = dynamic(() => import('./PreviewChunks')); const Preview = ({ showPreviewChunks }: { showPreviewChunks: boolean }) => { const { t } = useTranslation(); - const { sources } = useImportStore(); + const { sources } = useContextSelector(DatasetImportContext, (v) => v); const [previewRawTextSource, setPreviewRawTextSource] = useState(); const [previewChunkSource, setPreviewChunkSource] = useState(); diff --git a/projects/app/src/pages/dataset/detail/components/Import/components/PreviewChunks.tsx b/projects/app/src/pages/dataset/detail/components/Import/components/PreviewChunks.tsx index e0c1a6b7e3a6..65b30ce749ff 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/components/PreviewChunks.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/components/PreviewChunks.tsx @@ -4,11 +4,12 @@ import { ImportSourceItemType } from '@/web/core/dataset/type'; import { useQuery } from '@tanstack/react-query'; import MyRightDrawer from '@fastgpt/web/components/common/MyDrawer/MyRightDrawer'; import { getPreviewChunks } from '@/web/core/dataset/api'; -import { useImportStore } from '../Provider'; import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants'; import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter'; import { useToast } from '@fastgpt/web/hooks/useToast'; import { getErrText } from '@fastgpt/global/common/error/utils'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetImportContext } from '../Context'; const PreviewChunks = ({ previewSource, @@ -18,7 +19,10 @@ const PreviewChunks = ({ onClose: () => void; }) => { const { toast } = useToast(); - const { importSource, chunkSize, chunkOverlapRatio, processParamsForm } = useImportStore(); + const { importSource, chunkSize, chunkOverlapRatio, processParamsForm } = useContextSelector( + DatasetImportContext, + (v) => v + ); const { data = [], isLoading } = useQuery( ['previewSource'], diff --git a/projects/app/src/pages/dataset/detail/components/Import/components/PreviewRawText.tsx b/projects/app/src/pages/dataset/detail/components/Import/components/PreviewRawText.tsx index b995d4ed83a1..8c86d0b699d1 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/components/PreviewRawText.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/components/PreviewRawText.tsx @@ -4,10 +4,11 @@ import { ImportSourceItemType } from '@/web/core/dataset/type'; import { useQuery } from '@tanstack/react-query'; import { getPreviewFileContent } from '@/web/common/file/api'; import MyRightDrawer from '@fastgpt/web/components/common/MyDrawer/MyRightDrawer'; -import { useImportStore } from '../Provider'; import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants'; import { useToast } from '@fastgpt/web/hooks/useToast'; import { getErrText } from '@fastgpt/global/common/error/utils'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetImportContext } from '../Context'; const PreviewRawText = ({ previewSource, @@ -17,7 +18,7 @@ const PreviewRawText = ({ onClose: () => void; }) => { const { toast } = useToast(); - const { importSource } = useImportStore(); + const { importSource } = useContextSelector(DatasetImportContext, (v) => v); const { data, isLoading } = useQuery( ['previewSource', previewSource?.dbFileId], diff --git a/projects/app/src/pages/dataset/detail/components/Import/diffSource/ExternalFile.tsx b/projects/app/src/pages/dataset/detail/components/Import/diffSource/ExternalFile.tsx new file mode 100644 index 000000000000..f00f49bdad19 --- /dev/null +++ b/projects/app/src/pages/dataset/detail/components/Import/diffSource/ExternalFile.tsx @@ -0,0 +1,188 @@ +import React, { useEffect } from 'react'; +import dynamic from 'next/dynamic'; +import { useTranslation } from 'next-i18next'; +import { useFieldArray, useForm } from 'react-hook-form'; +import { + Box, + Button, + Flex, + Table, + Thead, + Tbody, + Tr, + Th, + Td, + TableContainer, + Input +} from '@chakra-ui/react'; +import { getNanoid } from '@fastgpt/global/common/string/tools'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import Loading from '@fastgpt/web/components/common/MyLoading'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetImportContext } from '../Context'; +import { getFileIcon } from '@fastgpt/global/common/file/icon'; +import { useI18n } from '@/web/context/I18n'; +import { SmallAddIcon } from '@chakra-ui/icons'; + +const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), { + loading: () => +}); +const Upload = dynamic(() => import('../commonProgress/Upload')); + +const ExternalFileCollection = () => { + const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep); + + return ( + <> + {activeStep === 0 && } + {activeStep === 1 && } + {activeStep === 2 && } + + ); +}; + +export default React.memo(ExternalFileCollection); + +const CustomLinkInput = () => { + const { t } = useTranslation(); + const { datasetT, commonT } = useI18n(); + const { goToNext, sources, setSources } = useContextSelector(DatasetImportContext, (v) => v); + const { register, reset, handleSubmit, control } = useForm<{ + list: { + sourceName: string; + sourceUrl: string; + externalId: string; + }[]; + }>({ + defaultValues: { + list: [ + { + sourceName: '', + sourceUrl: '', + externalId: '' + } + ] + } + }); + + const { + fields: list, + append, + remove, + update + } = useFieldArray({ + control, + name: 'list' + }); + + useEffect(() => { + if (sources.length > 0) { + reset({ + list: sources.map((item) => ({ + sourceName: item.sourceName, + sourceUrl: item.sourceUrl || '', + externalId: item.externalId || '' + })) + }); + } + }, []); + + return ( + + + + + + + + + + + + + {list.map((item, index) => ( + + + + + + + ))} + +
{datasetT('External url')}{datasetT('External id')}{datasetT('filename')}
+ + + + + + + remove(index)} + /> +
+ + + + + + + ); +}; diff --git a/projects/app/src/pages/dataset/detail/components/Import/diffSource/FileCustomText.tsx b/projects/app/src/pages/dataset/detail/components/Import/diffSource/FileCustomText.tsx index 5161376dcf21..befac68ef7a5 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/diffSource/FileCustomText.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/diffSource/FileCustomText.tsx @@ -1,24 +1,25 @@ import React, { useCallback, useEffect } from 'react'; -import { ImportDataComponentProps } from '@/web/core/dataset/type.d'; import dynamic from 'next/dynamic'; -import { useImportStore } from '../Provider'; import { useTranslation } from 'next-i18next'; import { useForm } from 'react-hook-form'; import { Box, Button, Flex, Input, Textarea } from '@chakra-ui/react'; import { getNanoid } from '@fastgpt/global/common/string/tools'; import Loading from '@fastgpt/web/components/common/MyLoading'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetImportContext } from '../Context'; const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), { loading: () => }); const Upload = dynamic(() => import('../commonProgress/Upload')); -const CustomTet = ({ activeStep, goToNext }: ImportDataComponentProps) => { +const CustomTet = () => { + const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep); return ( <> - {activeStep === 0 && } - {activeStep === 1 && } + {activeStep === 0 && } + {activeStep === 1 && } {activeStep === 2 && } ); @@ -26,9 +27,9 @@ const CustomTet = ({ activeStep, goToNext }: ImportDataComponentProps) => { export default React.memo(CustomTet); -const CustomTextInput = ({ goToNext }: { goToNext: () => void }) => { +const CustomTextInput = () => { const { t } = useTranslation(); - const { sources, setSources } = useImportStore(); + const { sources, goToNext, setSources } = useContextSelector(DatasetImportContext, (v) => v); const { register, reset, handleSubmit } = useForm({ defaultValues: { name: '', diff --git a/projects/app/src/pages/dataset/detail/components/Import/diffSource/FileLink.tsx b/projects/app/src/pages/dataset/detail/components/Import/diffSource/FileLink.tsx index c4b4d7a1209f..6a7d3349cb1d 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/diffSource/FileLink.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/diffSource/FileLink.tsx @@ -1,8 +1,5 @@ import React, { useEffect } from 'react'; -import { ImportDataComponentProps } from '@/web/core/dataset/type.d'; - import dynamic from 'next/dynamic'; -import { useImportStore } from '../Provider'; import { useTranslation } from 'next-i18next'; import { useForm } from 'react-hook-form'; import { Box, Button, Flex, Input, Link, Textarea } from '@chakra-ui/react'; @@ -12,17 +9,21 @@ import { LinkCollectionIcon } from '@fastgpt/global/core/dataset/constants'; import { useSystemStore } from '@/web/common/system/useSystemStore'; import { getDocPath } from '@/web/common/system/doc'; import Loading from '@fastgpt/web/components/common/MyLoading'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetImportContext } from '../Context'; const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), { loading: () => }); const Upload = dynamic(() => import('../commonProgress/Upload')); -const LinkCollection = ({ activeStep, goToNext }: ImportDataComponentProps) => { +const LinkCollection = () => { + const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep); + return ( <> - {activeStep === 0 && } - {activeStep === 1 && } + {activeStep === 0 && } + {activeStep === 1 && } {activeStep === 2 && } ); @@ -30,10 +31,13 @@ const LinkCollection = ({ activeStep, goToNext }: ImportDataComponentProps) => { export default React.memo(LinkCollection); -const CustomLinkImport = ({ goToNext }: { goToNext: () => void }) => { +const CustomLinkImport = () => { const { t } = useTranslation(); const { feConfigs } = useSystemStore(); - const { sources, setSources, processParamsForm } = useImportStore(); + const { goToNext, sources, setSources, processParamsForm } = useContextSelector( + DatasetImportContext, + (v) => v + ); const { register, reset, handleSubmit, watch } = useForm({ defaultValues: { link: '' diff --git a/projects/app/src/pages/dataset/detail/components/Import/diffSource/FileLocal.tsx b/projects/app/src/pages/dataset/detail/components/Import/diffSource/FileLocal.tsx index efa1692cac7f..c2f8325e4b34 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/diffSource/FileLocal.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/diffSource/FileLocal.tsx @@ -1,13 +1,14 @@ import React, { useCallback, useEffect, useMemo, useState } from 'react'; -import { ImportDataComponentProps, ImportSourceItemType } from '@/web/core/dataset/type.d'; +import { ImportSourceItemType } from '@/web/core/dataset/type.d'; import { Box, Button } from '@chakra-ui/react'; import FileSelector from '../components/FileSelector'; import { useTranslation } from 'next-i18next'; -import { useImportStore } from '../Provider'; import dynamic from 'next/dynamic'; import Loading from '@fastgpt/web/components/common/MyLoading'; import { RenderUploadFiles } from '../components/RenderFiles'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetImportContext } from '../Context'; const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), { loading: () => @@ -16,11 +17,13 @@ const Upload = dynamic(() => import('../commonProgress/Upload')); const fileType = '.txt, .docx, .csv, .xlsx, .pdf, .md, .html, .pptx'; -const FileLocal = ({ activeStep, goToNext }: ImportDataComponentProps) => { +const FileLocal = () => { + const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep); + return ( <> - {activeStep === 0 && } - {activeStep === 1 && } + {activeStep === 0 && } + {activeStep === 1 && } {activeStep === 2 && } ); @@ -28,9 +31,9 @@ const FileLocal = ({ activeStep, goToNext }: ImportDataComponentProps) => { export default React.memo(FileLocal); -const SelectFile = React.memo(function SelectFile({ goToNext }: { goToNext: () => void }) { +const SelectFile = React.memo(function SelectFile() { const { t } = useTranslation(); - const { sources, setSources } = useImportStore(); + const { goToNext, sources, setSources } = useContextSelector(DatasetImportContext, (v) => v); const [selectFiles, setSelectFiles] = useState( sources.map((source) => ({ isUploading: false, diff --git a/projects/app/src/pages/dataset/detail/components/Import/diffSource/TableLocal.tsx b/projects/app/src/pages/dataset/detail/components/Import/diffSource/TableLocal.tsx index b574cb3961e1..9c51d296a5c0 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/diffSource/TableLocal.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/diffSource/TableLocal.tsx @@ -1,24 +1,27 @@ import React, { useEffect, useMemo, useState } from 'react'; -import { ImportDataComponentProps, ImportSourceItemType } from '@/web/core/dataset/type.d'; +import { ImportSourceItemType } from '@/web/core/dataset/type.d'; import { Box, Button } from '@chakra-ui/react'; import FileSelector from '../components/FileSelector'; import { useTranslation } from 'next-i18next'; -import { useImportStore } from '../Provider'; import dynamic from 'next/dynamic'; import { fileDownload } from '@/web/common/file/utils'; import { RenderUploadFiles } from '../components/RenderFiles'; +import { useContextSelector } from 'use-context-selector'; +import { DatasetImportContext } from '../Context'; const PreviewData = dynamic(() => import('../commonProgress/PreviewData')); const Upload = dynamic(() => import('../commonProgress/Upload')); const fileType = '.csv'; -const FileLocal = ({ activeStep, goToNext }: ImportDataComponentProps) => { +const FileLocal = () => { + const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep); + return ( <> - {activeStep === 0 && } - {activeStep === 1 && } + {activeStep === 0 && } + {activeStep === 1 && } {activeStep === 2 && } ); @@ -32,9 +35,9 @@ const csvTemplate = `"第一列内容","第二列内容" "结合人工智能的演进历程,AIGC的发展大致可以分为三个阶段，即:早期萌芽阶段(20世纪50年代至90年代中期)、沉淀积累阶段(20世纪90年代中期至21世纪10年代中期),以及快速发展展阶段(21世纪10年代中期至今)。","" "AIGC发展分为几个阶段？","早期萌芽阶段(20世纪50年代至90年代中期)、沉淀积累阶段(20世纪90年代中期至21世纪10年代中期)、快速发展展阶段(21世纪10年代中期至今)"`; -const SelectFile = React.memo(function SelectFile({ goToNext }: { goToNext: () => void }) { +const SelectFile = React.memo(function SelectFile() { const { t } = useTranslation(); - const { sources, setSources } = useImportStore(); + const { goToNext, sources, setSources } = useContextSelector(DatasetImportContext, (v) => v); const [selectFiles, setSelectFiles] = useState( sources.map((source) => ({ isUploading: false, diff --git a/projects/app/src/pages/dataset/detail/components/Import/index.tsx b/projects/app/src/pages/dataset/detail/components/Import/index.tsx index f7250769079f..6e2c44a3ea63 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/index.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/index.tsx @@ -1,147 +1,42 @@ import React, { useMemo } from 'react'; -import { Box, Button, Flex, IconButton } from '@chakra-ui/react'; -import MyIcon from '@fastgpt/web/components/common/Icon'; -import { useTranslation } from 'next-i18next'; -import { useRouter } from 'next/router'; -import { TabEnum } from '../../index'; -import { useMyStep } from '@fastgpt/web/hooks/useStep'; +import { Box, Flex } from '@chakra-ui/react'; import dynamic from 'next/dynamic'; -import { useDatasetStore } from '@/web/core/dataset/store/dataset'; import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants'; -import Provider from './Provider'; +import { useContextSelector } from 'use-context-selector'; +import DatasetImportContextProvider, { DatasetImportContext } from './Context'; const FileLocal = dynamic(() => import('./diffSource/FileLocal')); const FileLink = dynamic(() => import('./diffSource/FileLink')); const FileCustomText = dynamic(() => import('./diffSource/FileCustomText')); const TableLocal = dynamic(() => import('./diffSource/TableLocal')); +const ExternalFileCollection = dynamic(() => import('./diffSource/ExternalFile')); const ImportDataset = () => { - const { t } = useTranslation(); - const router = useRouter(); - const { datasetDetail } = useDatasetStore(); - const { source = ImportDataSourceEnum.fileLocal, parentId } = (router.query || {}) as { - source: `${ImportDataSourceEnum}`; - parentId?: string; - }; - - const modeSteps: Record<`${ImportDataSourceEnum}`, { title: string }[]> = { - [ImportDataSourceEnum.fileLocal]: [ - { - title: t('core.dataset.import.Select file') - }, - { - title: t('core.dataset.import.Data Preprocessing') - }, - { - title: t('core.dataset.import.Upload data') - } - ], - [ImportDataSourceEnum.fileLink]: [ - { - title: t('core.dataset.import.Select file') - }, - { - title: t('core.dataset.import.Data Preprocessing') - }, - { - title: t('core.dataset.import.Upload data') - } - ], - [ImportDataSourceEnum.fileCustom]: [ - { - title: t('core.dataset.import.Select file') - }, - { - title: t('core.dataset.import.Data Preprocessing') - }, - { - title: t('core.dataset.import.Upload data') - } - ], - [ImportDataSourceEnum.csvTable]: [ - { - title: t('core.dataset.import.Select file') - }, - { - title: t('core.dataset.import.Data Preprocessing') - }, - { - title: t('core.dataset.import.Upload data') - } - ] - }; - const steps = modeSteps[source]; - - const { activeStep, goToNext, goToPrevious, MyStep } = useMyStep({ - defaultStep: 0, - steps - }); + const importSource = useContextSelector(DatasetImportContext, (v) => v.importSource); const ImportComponent = useMemo(() => { - if (source === ImportDataSourceEnum.fileLocal) return FileLocal; - if (source === ImportDataSourceEnum.fileLink) return FileLink; - if (source === ImportDataSourceEnum.fileCustom) return FileCustomText; - if (source === ImportDataSourceEnum.csvTable) return TableLocal; - }, [source]); + if (importSource === ImportDataSourceEnum.fileLocal) return FileLocal; + if (importSource === ImportDataSourceEnum.fileLink) return FileLink; + if (importSource === ImportDataSourceEnum.fileCustom) return FileCustomText; + if (importSource === ImportDataSourceEnum.csvTable) return TableLocal; + if (importSource === ImportDataSourceEnum.externalFile) return ExternalFileCollection; + }, [importSource]); return ImportComponent ? ( + + + + ) : null; +}; + +const Render = () => { + return ( - - {activeStep === 0 ? ( - - } - aria-label={''} - size={'smSquare'} - w={'26px'} - h={'26px'} - borderRadius={'50%'} - variant={'whiteBase'} - mr={2} - onClick={() => - router.replace({ - query: { - ...router.query, - currentTab: TabEnum.collectionCard - } - }) - } - /> - {t('common.Exit')} - - ) : ( - - )} - - - {/* step */} - - - - - - - - - - + + + - ) : null; + ); }; -export default React.memo(ImportDataset); +export default React.memo(Render); diff --git a/projects/app/src/pages/dataset/detail/components/Info.tsx b/projects/app/src/pages/dataset/detail/components/Info.tsx index b26bbd76cbc1..56654ff28167 100644 --- a/projects/app/src/pages/dataset/detail/components/Info.tsx +++ b/projects/app/src/pages/dataset/detail/components/Info.tsx @@ -23,13 +23,14 @@ import type { VectorModelItemType } from '@fastgpt/global/core/ai/model.d'; import { useContextSelector } from 'use-context-selector'; import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext'; import MyDivider from '@fastgpt/web/components/common/MyDivider/index'; +import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants'; const Info = ({ datasetId }: { datasetId: string }) => { const { t } = useTranslation(); const { datasetT } = useI18n(); - const { datasetDetail, loadDatasetDetail, loadDatasets, updateDataset } = useDatasetStore(); - const rebuildingCount = useContextSelector(DatasetPageContext, (v) => v.rebuildingCount); - const trainingCount = useContextSelector(DatasetPageContext, (v) => v.trainingCount); + const { datasetDetail, loadDatasetDetail, updateDataset, rebuildingCount, trainingCount } = + useContextSelector(DatasetPageContext, (v) => v); + const refetchDatasetTraining = useContextSelector( DatasetPageContext, (v) => v.refetchDatasetTraining @@ -82,9 +83,6 @@ const Info = ({ datasetId }: { datasetId: string }) => { ...data }); }, - onSuccess() { - loadDatasets(); - }, successToast: t('common.Update Success'), errorToast: t('common.Update Failed') }); @@ -117,7 +115,7 @@ const Info = ({ datasetId }: { datasetId: string }) => { }, onSuccess() { refetchDatasetTraining(); - loadDatasetDetail(datasetId, true); + loadDatasetDetail(datasetId); }, successToast: datasetT('Rebuild embedding start tip'), errorToast: t('common.Update Failed') @@ -128,16 +126,16 @@ const Info = ({ datasetId }: { datasetId: string }) => { return ( - + {t('core.dataset.Dataset ID')} {datasetDetail._id} - + {t('core.ai.model.Vector Model')} - + { - + {t('core.Max Token')} - {vectorModel.maxToken} + {vectorModel.maxToken} - + {t('core.ai.model.Dataset Agent Model')} - + { - + + + {datasetDetail.type === DatasetTypeEnum.externalFile && ( + <> + + + {datasetT('External read url')} + + + + + + )} - + {t('core.dataset.Avatar')} - + { - + {t('core.dataset.Name')} - + - {t('common.Intro')} -