From ff024286d03804e611abcc01c58d0088383ec701 Mon Sep 17 00:00:00 2001 From: Timothy Carambat Date: Mon, 16 Dec 2024 11:50:34 -0800 Subject: [PATCH 1/3] Add new workspace user management endpoint (#2842) deprecate old endpoint which users workspace ID vs slug resolves #2838 --- server/endpoints/api/admin/index.js | 128 ++++++++++++++++++++++++++++ server/models/workspace.js | 11 +++ server/models/workspaceUsers.js | 6 ++ server/swagger/openapi.json | 89 +++++++++++++++++++ 4 files changed, 234 insertions(+) diff --git a/server/endpoints/api/admin/index.js b/server/endpoints/api/admin/index.js index c165a9cc02..18f59ee872 100644 --- a/server/endpoints/api/admin/index.js +++ b/server/endpoints/api/admin/index.js @@ -4,6 +4,7 @@ const { SystemSettings } = require("../../../models/systemSettings"); const { User } = require("../../../models/user"); const { Workspace } = require("../../../models/workspace"); const { WorkspaceChats } = require("../../../models/workspaceChats"); +const { WorkspaceUser } = require("../../../models/workspaceUsers"); const { canModifyAdmin } = require("../../../utils/helpers/admin"); const { multiUserMode, reqBody } = require("../../../utils/http"); const { validApiKey } = require("../../../utils/middleware/validApiKey"); @@ -420,6 +421,7 @@ function apiAdminEndpoints(app) { } } ); + app.get( "/v1/admin/workspaces/:workspaceId/users", [validApiKey], @@ -474,12 +476,14 @@ function apiAdminEndpoints(app) { } } ); + app.post( "/v1/admin/workspaces/:workspaceId/update-users", [validApiKey], async (request, response) => { /* #swagger.tags = ['Admin'] + #swagger.deprecated = true #swagger.parameters['workspaceId'] = { in: 'path', description: 'id of the workspace in the database.', @@ -539,6 +543,130 @@ function apiAdminEndpoints(app) { } } ); + + app.post( + "/v1/admin/workspaces/:workspaceSlug/manage-users", + [validApiKey], + async (request, response) => { + /* + #swagger.tags = ['Admin'] + #swagger.parameters['workspaceSlug'] = { + in: 'path', + description: 'slug of the workspace in the database', + required: true, + type: 'string' + } + #swagger.description = 'Set workspace permissions to be accessible by the given user ids and admins. Methods are disabled until multi user mode is enabled via the UI.' + #swagger.requestBody = { + description: 'Array of user ids who will be given access to the target workspace. reset will remove all existing users from the workspace and only add the new users - default false.', + required: true, + content: { + "application/json": { + example: { + userIds: [1,2,4,12], + reset: false + } + } + } + } + #swagger.responses[200] = { + content: { + "application/json": { + schema: { + type: 'object', + example: { + success: true, + error: null, + users: [ + {"userId": 1, "username": "main-admin", "role": "admin"}, + {"userId": 2, "username": "sample-sam", "role": "default"} + ] + } + } + } + } + } + #swagger.responses[403] = { + schema: { + "$ref": "#/definitions/InvalidAPIKey" + } + } + #swagger.responses[401] = { + description: "Instance is not in Multi-User mode. Method denied", + } + */ + try { + if (!multiUserMode(response)) { + response.sendStatus(401).end(); + return; + } + + const { workspaceSlug } = request.params; + const { userIds: _uids, reset = false } = reqBody(request); + const userIds = ( + await User.where({ id: { in: _uids.map(Number) } }) + ).map((user) => user.id); + const workspace = await Workspace.get({ slug: String(workspaceSlug) }); + const workspaceUsers = await Workspace.workspaceUsers(workspace.id); + + if (!workspace) { + response + .status(404) + .json({ + success: false, + error: `Workspace ${workspaceSlug} not found`, + users: workspaceUsers, + }); + return; + } + + if (userIds.length === 0) { + response + .status(404) + .json({ + success: false, + error: `No valid user IDs provided.`, + users: workspaceUsers, + }); + return; + } + + // Reset all users in the workspace and add the new users as the only users in the workspace + if (reset) { + const { success, error } = await Workspace.updateUsers( + workspace.id, + userIds + ); + return response + .status(200) + .json({ + success, + error, + users: await Workspace.workspaceUsers(workspace.id), + }); + } + + // Add new users to the workspace if they are not already in the workspace + const existingUserIds = workspaceUsers.map((user) => user.userId); + const usersToAdd = userIds.filter( + (userId) => !existingUserIds.includes(userId) + ); + if (usersToAdd.length > 0) + await WorkspaceUser.createManyUsers(usersToAdd, workspace.id); + response + .status(200) + .json({ + success: true, + error: null, + users: await Workspace.workspaceUsers(workspace.id), + }); + } catch (e) { + console.error(e); + response.sendStatus(500).end(); + } + } + ); + app.post( "/v1/admin/workspace-chats", [validApiKey], diff --git a/server/models/workspace.js b/server/models/workspace.js index 47734ddd51..5bc9301948 100644 --- a/server/models/workspace.js +++ b/server/models/workspace.js @@ -243,6 +243,11 @@ const Workspace = { } }, + /** + * Get all users for a workspace. + * @param {number} workspaceId - The ID of the workspace to get users for. + * @returns {Promise>} A promise that resolves to an array of user objects. + */ workspaceUsers: async function (workspaceId) { try { const users = ( @@ -270,6 +275,12 @@ const Workspace = { } }, + /** + * Update the users for a workspace. Will remove all existing users and replace them with the new list. + * @param {number} workspaceId - The ID of the workspace to update. + * @param {number[]} userIds - An array of user IDs to add to the workspace. + * @returns {Promise<{success: boolean, error: string | null}>} A promise that resolves to an object containing the success status and an error message if applicable. + */ updateUsers: async function (workspaceId, userIds = []) { try { await WorkspaceUser.delete({ workspace_id: Number(workspaceId) }); diff --git a/server/models/workspaceUsers.js b/server/models/workspaceUsers.js index 5f66a1add4..c27dc858a0 100644 --- a/server/models/workspaceUsers.js +++ b/server/models/workspaceUsers.js @@ -17,6 +17,12 @@ const WorkspaceUser = { return; }, + /** + * Create many workspace users. + * @param {Array} userIds - An array of user IDs to create workspace users for. + * @param {number} workspaceId - The ID of the workspace to create workspace users for. + * @returns {Promise} A promise that resolves when the workspace users are created. + */ createManyUsers: async function (userIds = [], workspaceId) { if (userIds.length === 0) return; try { diff --git a/server/swagger/openapi.json b/server/swagger/openapi.json index 230398ada5..505d579628 100644 --- a/server/swagger/openapi.json +++ b/server/swagger/openapi.json @@ -635,6 +635,95 @@ } } } + }, + "deprecated": true + } + }, + "/v1/admin/workspaces/{workspaceSlug}/manage-users": { + "post": { + "tags": [ + "Admin" + ], + "description": "Set workspace permissions to be accessible by the given user ids and admins. Methods are disabled until multi user mode is enabled via the UI.", + "parameters": [ + { + "name": "workspaceSlug", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "slug of the workspace in the database" + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "object", + "example": { + "success": true, + "error": null, + "users": [ + { + "userId": 1, + "username": "main-admin", + "role": "admin" + }, + { + "userId": 2, + "username": "sample-sam", + "role": "default" + } + ] + } + } + } + } + }, + "401": { + "description": "Instance is not in Multi-User mode. Method denied" + }, + "403": { + "description": "Forbidden", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InvalidAPIKey" + } + }, + "application/xml": { + "schema": { + "$ref": "#/components/schemas/InvalidAPIKey" + } + } + } + }, + "404": { + "description": "Not Found" + }, + "500": { + "description": "Internal Server Error" + } + }, + "requestBody": { + "description": "Array of user ids who will be given access to the target workspace. reset will remove all existing users from the workspace and only add the new users - default false.", + "required": true, + "content": { + "application/json": { + "example": { + "userIds": [ + 1, + 2, + 4, + 12 + ], + "reset": false + } + } + } } } }, From d145602d5a3ffc5ed0003846058d9413e2ec9a9a Mon Sep 17 00:00:00 2001 From: wolfganghuse Date: Mon, 16 Dec 2024 21:03:51 +0100 Subject: [PATCH 2/3] Add attachments to GenericOpenAI prompt (#2831) * added attachments to genericopenai prompt * add devnote --------- Co-authored-by: timothycarambat --- .../utils/AiProviders/genericOpenAi/index.js | 50 ++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/server/utils/AiProviders/genericOpenAi/index.js b/server/utils/AiProviders/genericOpenAi/index.js index fe29023000..8d17aa257c 100644 --- a/server/utils/AiProviders/genericOpenAi/index.js +++ b/server/utils/AiProviders/genericOpenAi/index.js @@ -77,17 +77,65 @@ class GenericOpenAiLLM { return true; } + /** + * Generates appropriate content array for a message + attachments. + * + * ## Developer Note + * This function assumes the generic OpenAI provider is _actually_ OpenAI compatible. + * For example, Ollama is "OpenAI compatible" but does not support images as a content array. + * The contentString also is the base64 string WITH `data:image/xxx;base64,` prefix, which may not be the case for all providers. + * If your provider does not work exactly this way, then attachments will not function or potentially break vision requests. + * If you encounter this issue, you are welcome to open an issue asking for your specific provider to be supported. + * + * This function will **not** be updated for providers that **do not** support images as a content array like OpenAI does. + * Do not open issues to update this function due to your specific provider not being compatible. Open an issue to request support for your specific provider. + * @param {Object} props + * @param {string} props.userPrompt - the user prompt to be sent to the model + * @param {import("../../helpers").Attachment[]} props.attachments - the array of attachments to be sent to the model + * @returns {string|object[]} + */ + #generateContent({ userPrompt, attachments = [] }) { + if (!attachments.length) { + return userPrompt; + } + + const content = [{ type: "text", text: userPrompt }]; + for (let attachment of attachments) { + content.push({ + type: "image_url", + image_url: { + url: attachment.contentString, + detail: "high", + }, + }); + } + return content.flat(); + } + + /** + * Construct the user prompt for this model. + * @param {{attachments: import("../../helpers").Attachment[]}} param0 + * @returns + */ constructPrompt({ systemPrompt = "", contextTexts = [], chatHistory = [], userPrompt = "", + attachments = [], }) { const prompt = { role: "system", content: `${systemPrompt}${this.#appendContext(contextTexts)}`, }; - return [prompt, ...chatHistory, { role: "user", content: userPrompt }]; + return [ + prompt, + ...chatHistory, + { + role: "user", + content: this.#generateContent({ userPrompt, attachments }), + }, + ]; } async getChatCompletion(messages = null, { temperature = 0.7 }) { From ae510619f069ef2530bbba752a240d501dc2d047 Mon Sep 17 00:00:00 2001 From: Sean Hatfield Date: Tue, 17 Dec 2024 04:16:20 +0800 Subject: [PATCH 3/3] Purge cached docs and remove docs from all workspaces on vectorDB/embedder changes (#2819) * wip remove all docs clear vector db on embedder/vector db change * purge all cached docs and remove docs from workspaces on vectordb/embedder change * lint * remove unneeded console log * remove reset vector stores endpoint and move to server side updateENV with postUpdate check * reset embed module * remove unused import * simplify deletion process rescoped document deletion to be more general for speed, everything needs to be reset anyway fixed issue where unembedded docs not in any workspaces, but cached, were not removed * add back missing readme file update warning text modals --------- Co-authored-by: timothycarambat --- .../src/components/ChangeWarning/index.jsx | 74 +++++++++++-------- .../EmbeddingPreference/index.jsx | 2 +- .../GeneralSettings/VectorDatabase/index.jsx | 2 +- server/models/vectors.js | 34 ++++++--- server/utils/files/index.js | 11 +++ server/utils/helpers/index.js | 5 +- server/utils/helpers/updateENV.js | 23 ++++++ .../utils/vectorStore/resetAllVectorStores.js | 48 ++++++++++++ 8 files changed, 153 insertions(+), 46 deletions(-) create mode 100644 server/utils/vectorStore/resetAllVectorStores.js diff --git a/frontend/src/components/ChangeWarning/index.jsx b/frontend/src/components/ChangeWarning/index.jsx index 42b211baf4..2e0950a080 100644 --- a/frontend/src/components/ChangeWarning/index.jsx +++ b/frontend/src/components/ChangeWarning/index.jsx @@ -1,4 +1,4 @@ -import { Warning } from "@phosphor-icons/react"; +import { Warning, X } from "@phosphor-icons/react"; export default function ChangeWarningModal({ warningText = "", @@ -6,41 +6,55 @@ export default function ChangeWarningModal({ onConfirm, }) { return ( -
-
-
-
- -

Warning

-
+
+
+
+ +

+ WARNING - This action is irreversible +

-
-

- {warningText} + +

+
+
+

+ {warningText.split("\\n").map((line, index) => ( + + {line} +
+ + ))}

Are you sure you want to proceed?

- -
- - -
+
+
+ +
); diff --git a/frontend/src/pages/GeneralSettings/EmbeddingPreference/index.jsx b/frontend/src/pages/GeneralSettings/EmbeddingPreference/index.jsx index 893948472d..77853e0a99 100644 --- a/frontend/src/pages/GeneralSettings/EmbeddingPreference/index.jsx +++ b/frontend/src/pages/GeneralSettings/EmbeddingPreference/index.jsx @@ -361,7 +361,7 @@ export default function GeneralEmbeddingPreference() { )} diff --git a/frontend/src/pages/GeneralSettings/VectorDatabase/index.jsx b/frontend/src/pages/GeneralSettings/VectorDatabase/index.jsx index f7246de542..11e70d8fd7 100644 --- a/frontend/src/pages/GeneralSettings/VectorDatabase/index.jsx +++ b/frontend/src/pages/GeneralSettings/VectorDatabase/index.jsx @@ -308,7 +308,7 @@ export default function GeneralVectorDatabase() { )} diff --git a/server/models/vectors.js b/server/models/vectors.js index f6b79964a0..3653303da2 100644 --- a/server/models/vectors.js +++ b/server/models/vectors.js @@ -25,6 +25,19 @@ const DocumentVectors = { } }, + where: async function (clause = {}, limit) { + try { + const results = await prisma.document_vectors.findMany({ + where: clause, + take: limit || undefined, + }); + return results; + } catch (error) { + console.error("Where query failed", error); + return []; + } + }, + deleteForWorkspace: async function (workspaceId) { const documents = await Document.forWorkspace(workspaceId); const docIds = [...new Set(documents.map((doc) => doc.docId))]; @@ -40,27 +53,24 @@ const DocumentVectors = { } }, - where: async function (clause = {}, limit) { + deleteIds: async function (ids = []) { try { - const results = await prisma.document_vectors.findMany({ - where: clause, - take: limit || undefined, + await prisma.document_vectors.deleteMany({ + where: { id: { in: ids } }, }); - return results; + return true; } catch (error) { - console.error("Where query failed", error); - return []; + console.error("Delete IDs failed", error); + return false; } }, - deleteIds: async function (ids = []) { + delete: async function (clause = {}) { try { - await prisma.document_vectors.deleteMany({ - where: { id: { in: ids } }, - }); + await prisma.document_vectors.deleteMany({ where: clause }); return true; } catch (error) { - console.error("Delete IDs failed", error); + console.error("Delete failed", error); return false; } }, diff --git a/server/utils/files/index.js b/server/utils/files/index.js index 598884f999..625d8582cd 100644 --- a/server/utils/files/index.js +++ b/server/utils/files/index.js @@ -281,6 +281,16 @@ async function getWatchedDocumentFilenames(filenames = []) { }, {}); } +/** + * Purges the entire vector-cache folder and recreates it. + * @returns {void} + */ +function purgeEntireVectorCache() { + fs.rmSync(vectorCachePath, { recursive: true, force: true }); + fs.mkdirSync(vectorCachePath); + return; +} + module.exports = { findDocumentInDocuments, cachedVectorInformation, @@ -293,4 +303,5 @@ module.exports = { isWithin, documentsPath, hasVectorCachedFiles, + purgeEntireVectorCache, }; diff --git a/server/utils/helpers/index.js b/server/utils/helpers/index.js index e599078b6a..748e4fb1b1 100644 --- a/server/utils/helpers/index.js +++ b/server/utils/helpers/index.js @@ -52,10 +52,11 @@ /** * Gets the systems current vector database provider. + * @param {('pinecone' | 'chroma' | 'lancedb' | 'weaviate' | 'qdrant' | 'milvus' | 'zilliz' | 'astra') | null} getExactly - If provided, this will return an explit provider. * @returns { BaseVectorDatabaseProvider} */ -function getVectorDbClass() { - const vectorSelection = process.env.VECTOR_DB || "lancedb"; +function getVectorDbClass(getExactly = null) { + const vectorSelection = getExactly ?? process.env.VECTOR_DB ?? "lancedb"; switch (vectorSelection) { case "pinecone": const { Pinecone } = require("../vectorDbProviders/pinecone"); diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js index 3cfc13e6e1..948703dca2 100644 --- a/server/utils/helpers/updateENV.js +++ b/server/utils/helpers/updateENV.js @@ -1,3 +1,5 @@ +const { resetAllVectorStores } = require("../vectorStore/resetAllVectorStores"); + const KEY_MAPPING = { LLMProvider: { envKey: "LLM_PROVIDER", @@ -248,6 +250,7 @@ const KEY_MAPPING = { EmbeddingEngine: { envKey: "EMBEDDING_ENGINE", checks: [supportedEmbeddingModel], + postUpdate: [handleVectorStoreReset], }, EmbeddingBasePath: { envKey: "EMBEDDING_BASE_PATH", @@ -256,6 +259,7 @@ const KEY_MAPPING = { EmbeddingModelPref: { envKey: "EMBEDDING_MODEL_PREF", checks: [isNotEmpty], + postUpdate: [handleVectorStoreReset], }, EmbeddingModelMaxChunkLength: { envKey: "EMBEDDING_MODEL_MAX_CHUNK_LENGTH", @@ -276,6 +280,7 @@ const KEY_MAPPING = { VectorDB: { envKey: "VECTOR_DB", checks: [isNotEmpty, supportedVectorDB], + postUpdate: [handleVectorStoreReset], }, // Chroma Options @@ -878,6 +883,24 @@ function noRestrictedChars(input = "") { : null; } +async function handleVectorStoreReset(key, prevValue, nextValue) { + if (prevValue === nextValue) return; + if (key === "VectorDB") { + console.log( + `Vector configuration changed from ${prevValue} to ${nextValue} - resetting ${prevValue} namespaces` + ); + return await resetAllVectorStores({ vectorDbKey: prevValue }); + } + + if (key === "EmbeddingEngine" || key === "EmbeddingModelPref") { + console.log( + `${key} changed from ${prevValue} to ${nextValue} - resetting ${process.env.VECTOR_DB} namespaces` + ); + return await resetAllVectorStores({ vectorDbKey: process.env.VECTOR_DB }); + } + return false; +} + // This will force update .env variables which for any which reason were not able to be parsed or // read from an ENV file as this seems to be a complicating step for many so allowing people to write // to the process will at least alleviate that issue. It does not perform comprehensive validity checks or sanity checks diff --git a/server/utils/vectorStore/resetAllVectorStores.js b/server/utils/vectorStore/resetAllVectorStores.js new file mode 100644 index 0000000000..3bb9a5ec4c --- /dev/null +++ b/server/utils/vectorStore/resetAllVectorStores.js @@ -0,0 +1,48 @@ +const { Workspace } = require("../../models/workspace"); +const { Document } = require("../../models/documents"); +const { DocumentVectors } = require("../../models/vectors"); +const { EventLogs } = require("../../models/eventLogs"); +const { purgeEntireVectorCache } = require("../files"); +const { getVectorDbClass } = require("../helpers"); + +/** + * Resets all vector database and associated content: + * - Purges the entire vector-cache folder. + * - Deletes all document vectors from the database. + * - Deletes all documents from the database. + * - Deletes all vector db namespaces for each workspace. + * - Logs an event indicating the reset. + * @param {string} vectorDbKey - The _previous_ vector database provider name that we will be resetting. + * @returns {Promise} - True if successful, false otherwise. + */ +async function resetAllVectorStores({ vectorDbKey }) { + try { + const workspaces = await Workspace.where(); + purgeEntireVectorCache(); // Purges the entire vector-cache folder. + await DocumentVectors.delete(); // Deletes all document vectors from the database. + await Document.delete(); // Deletes all documents from the database. + await EventLogs.logEvent("workspace_vectors_reset", { + reason: "System vector configuration changed", + }); + + console.log( + "Resetting anythingllm managed vector namespaces for", + vectorDbKey + ); + const VectorDb = getVectorDbClass(vectorDbKey); + for (const workspace of workspaces) { + try { + await VectorDb["delete-namespace"]({ namespace: workspace.slug }); + } catch (e) { + console.error(e.message); + } + } + + return true; + } catch (error) { + console.error("Failed to reset vector stores:", error); + return false; + } +} + +module.exports = { resetAllVectorStores };