diff --git a/frontend/src/components/ChangeWarning/index.jsx b/frontend/src/components/ChangeWarning/index.jsx index 42b211baf4..2e0950a080 100644 --- a/frontend/src/components/ChangeWarning/index.jsx +++ b/frontend/src/components/ChangeWarning/index.jsx @@ -1,4 +1,4 @@ -import { Warning } from "@phosphor-icons/react"; +import { Warning, X } from "@phosphor-icons/react"; export default function ChangeWarningModal({ warningText = "", @@ -6,41 +6,55 @@ export default function ChangeWarningModal({ onConfirm, }) { return ( -
-
-
-
- -

Warning

-
+
+
+
+ +

+ WARNING - This action is irreversible +

-
-

- {warningText} + +

+
+
+

+ {warningText.split("\\n").map((line, index) => ( + + {line} +
+ + ))}

Are you sure you want to proceed?

- -
- - -
+
+
+ +
); diff --git a/frontend/src/pages/GeneralSettings/EmbeddingPreference/index.jsx b/frontend/src/pages/GeneralSettings/EmbeddingPreference/index.jsx index 893948472d..77853e0a99 100644 --- a/frontend/src/pages/GeneralSettings/EmbeddingPreference/index.jsx +++ b/frontend/src/pages/GeneralSettings/EmbeddingPreference/index.jsx @@ -361,7 +361,7 @@ export default function GeneralEmbeddingPreference() { )} diff --git a/frontend/src/pages/GeneralSettings/VectorDatabase/index.jsx b/frontend/src/pages/GeneralSettings/VectorDatabase/index.jsx index f7246de542..11e70d8fd7 100644 --- a/frontend/src/pages/GeneralSettings/VectorDatabase/index.jsx +++ b/frontend/src/pages/GeneralSettings/VectorDatabase/index.jsx @@ -308,7 +308,7 @@ export default function GeneralVectorDatabase() { )} diff --git a/server/endpoints/api/admin/index.js b/server/endpoints/api/admin/index.js index c165a9cc02..18f59ee872 100644 --- a/server/endpoints/api/admin/index.js +++ b/server/endpoints/api/admin/index.js @@ -4,6 +4,7 @@ const { SystemSettings } = require("../../../models/systemSettings"); const { User } = require("../../../models/user"); const { Workspace } = require("../../../models/workspace"); const { WorkspaceChats } = require("../../../models/workspaceChats"); +const { WorkspaceUser } = require("../../../models/workspaceUsers"); const { canModifyAdmin } = require("../../../utils/helpers/admin"); const { multiUserMode, reqBody } = require("../../../utils/http"); const { validApiKey } = require("../../../utils/middleware/validApiKey"); @@ -420,6 +421,7 @@ function apiAdminEndpoints(app) { } } ); + app.get( "/v1/admin/workspaces/:workspaceId/users", [validApiKey], @@ -474,12 +476,14 @@ function apiAdminEndpoints(app) { } } ); + app.post( "/v1/admin/workspaces/:workspaceId/update-users", [validApiKey], async (request, response) => { /* #swagger.tags = ['Admin'] + #swagger.deprecated = true #swagger.parameters['workspaceId'] = { in: 'path', description: 'id of the workspace in the database.', @@ -539,6 +543,130 @@ function apiAdminEndpoints(app) { } } ); + + app.post( + "/v1/admin/workspaces/:workspaceSlug/manage-users", + [validApiKey], + async (request, response) => { + /* + #swagger.tags = ['Admin'] + #swagger.parameters['workspaceSlug'] = { + in: 'path', + description: 'slug of the workspace in the database', + required: true, + type: 'string' + } + #swagger.description = 'Set workspace permissions to be accessible by the given user ids and admins. Methods are disabled until multi user mode is enabled via the UI.' + #swagger.requestBody = { + description: 'Array of user ids who will be given access to the target workspace. reset will remove all existing users from the workspace and only add the new users - default false.', + required: true, + content: { + "application/json": { + example: { + userIds: [1,2,4,12], + reset: false + } + } + } + } + #swagger.responses[200] = { + content: { + "application/json": { + schema: { + type: 'object', + example: { + success: true, + error: null, + users: [ + {"userId": 1, "username": "main-admin", "role": "admin"}, + {"userId": 2, "username": "sample-sam", "role": "default"} + ] + } + } + } + } + } + #swagger.responses[403] = { + schema: { + "$ref": "#/definitions/InvalidAPIKey" + } + } + #swagger.responses[401] = { + description: "Instance is not in Multi-User mode. Method denied", + } + */ + try { + if (!multiUserMode(response)) { + response.sendStatus(401).end(); + return; + } + + const { workspaceSlug } = request.params; + const { userIds: _uids, reset = false } = reqBody(request); + const userIds = ( + await User.where({ id: { in: _uids.map(Number) } }) + ).map((user) => user.id); + const workspace = await Workspace.get({ slug: String(workspaceSlug) }); + const workspaceUsers = await Workspace.workspaceUsers(workspace.id); + + if (!workspace) { + response + .status(404) + .json({ + success: false, + error: `Workspace ${workspaceSlug} not found`, + users: workspaceUsers, + }); + return; + } + + if (userIds.length === 0) { + response + .status(404) + .json({ + success: false, + error: `No valid user IDs provided.`, + users: workspaceUsers, + }); + return; + } + + // Reset all users in the workspace and add the new users as the only users in the workspace + if (reset) { + const { success, error } = await Workspace.updateUsers( + workspace.id, + userIds + ); + return response + .status(200) + .json({ + success, + error, + users: await Workspace.workspaceUsers(workspace.id), + }); + } + + // Add new users to the workspace if they are not already in the workspace + const existingUserIds = workspaceUsers.map((user) => user.userId); + const usersToAdd = userIds.filter( + (userId) => !existingUserIds.includes(userId) + ); + if (usersToAdd.length > 0) + await WorkspaceUser.createManyUsers(usersToAdd, workspace.id); + response + .status(200) + .json({ + success: true, + error: null, + users: await Workspace.workspaceUsers(workspace.id), + }); + } catch (e) { + console.error(e); + response.sendStatus(500).end(); + } + } + ); + app.post( "/v1/admin/workspace-chats", [validApiKey], diff --git a/server/models/vectors.js b/server/models/vectors.js index f6b79964a0..3653303da2 100644 --- a/server/models/vectors.js +++ b/server/models/vectors.js @@ -25,6 +25,19 @@ const DocumentVectors = { } }, + where: async function (clause = {}, limit) { + try { + const results = await prisma.document_vectors.findMany({ + where: clause, + take: limit || undefined, + }); + return results; + } catch (error) { + console.error("Where query failed", error); + return []; + } + }, + deleteForWorkspace: async function (workspaceId) { const documents = await Document.forWorkspace(workspaceId); const docIds = [...new Set(documents.map((doc) => doc.docId))]; @@ -40,27 +53,24 @@ const DocumentVectors = { } }, - where: async function (clause = {}, limit) { + deleteIds: async function (ids = []) { try { - const results = await prisma.document_vectors.findMany({ - where: clause, - take: limit || undefined, + await prisma.document_vectors.deleteMany({ + where: { id: { in: ids } }, }); - return results; + return true; } catch (error) { - console.error("Where query failed", error); - return []; + console.error("Delete IDs failed", error); + return false; } }, - deleteIds: async function (ids = []) { + delete: async function (clause = {}) { try { - await prisma.document_vectors.deleteMany({ - where: { id: { in: ids } }, - }); + await prisma.document_vectors.deleteMany({ where: clause }); return true; } catch (error) { - console.error("Delete IDs failed", error); + console.error("Delete failed", error); return false; } }, diff --git a/server/models/workspace.js b/server/models/workspace.js index 47734ddd51..5bc9301948 100644 --- a/server/models/workspace.js +++ b/server/models/workspace.js @@ -243,6 +243,11 @@ const Workspace = { } }, + /** + * Get all users for a workspace. + * @param {number} workspaceId - The ID of the workspace to get users for. + * @returns {Promise>} A promise that resolves to an array of user objects. + */ workspaceUsers: async function (workspaceId) { try { const users = ( @@ -270,6 +275,12 @@ const Workspace = { } }, + /** + * Update the users for a workspace. Will remove all existing users and replace them with the new list. + * @param {number} workspaceId - The ID of the workspace to update. + * @param {number[]} userIds - An array of user IDs to add to the workspace. + * @returns {Promise<{success: boolean, error: string | null}>} A promise that resolves to an object containing the success status and an error message if applicable. + */ updateUsers: async function (workspaceId, userIds = []) { try { await WorkspaceUser.delete({ workspace_id: Number(workspaceId) }); diff --git a/server/models/workspaceUsers.js b/server/models/workspaceUsers.js index 5f66a1add4..c27dc858a0 100644 --- a/server/models/workspaceUsers.js +++ b/server/models/workspaceUsers.js @@ -17,6 +17,12 @@ const WorkspaceUser = { return; }, + /** + * Create many workspace users. + * @param {Array} userIds - An array of user IDs to create workspace users for. + * @param {number} workspaceId - The ID of the workspace to create workspace users for. + * @returns {Promise} A promise that resolves when the workspace users are created. + */ createManyUsers: async function (userIds = [], workspaceId) { if (userIds.length === 0) return; try { diff --git a/server/swagger/openapi.json b/server/swagger/openapi.json index 230398ada5..505d579628 100644 --- a/server/swagger/openapi.json +++ b/server/swagger/openapi.json @@ -635,6 +635,95 @@ } } } + }, + "deprecated": true + } + }, + "/v1/admin/workspaces/{workspaceSlug}/manage-users": { + "post": { + "tags": [ + "Admin" + ], + "description": "Set workspace permissions to be accessible by the given user ids and admins. Methods are disabled until multi user mode is enabled via the UI.", + "parameters": [ + { + "name": "workspaceSlug", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "slug of the workspace in the database" + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "object", + "example": { + "success": true, + "error": null, + "users": [ + { + "userId": 1, + "username": "main-admin", + "role": "admin" + }, + { + "userId": 2, + "username": "sample-sam", + "role": "default" + } + ] + } + } + } + } + }, + "401": { + "description": "Instance is not in Multi-User mode. Method denied" + }, + "403": { + "description": "Forbidden", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InvalidAPIKey" + } + }, + "application/xml": { + "schema": { + "$ref": "#/components/schemas/InvalidAPIKey" + } + } + } + }, + "404": { + "description": "Not Found" + }, + "500": { + "description": "Internal Server Error" + } + }, + "requestBody": { + "description": "Array of user ids who will be given access to the target workspace. reset will remove all existing users from the workspace and only add the new users - default false.", + "required": true, + "content": { + "application/json": { + "example": { + "userIds": [ + 1, + 2, + 4, + 12 + ], + "reset": false + } + } + } } } }, diff --git a/server/utils/AiProviders/genericOpenAi/index.js b/server/utils/AiProviders/genericOpenAi/index.js index fe29023000..8d17aa257c 100644 --- a/server/utils/AiProviders/genericOpenAi/index.js +++ b/server/utils/AiProviders/genericOpenAi/index.js @@ -77,17 +77,65 @@ class GenericOpenAiLLM { return true; } + /** + * Generates appropriate content array for a message + attachments. + * + * ## Developer Note + * This function assumes the generic OpenAI provider is _actually_ OpenAI compatible. + * For example, Ollama is "OpenAI compatible" but does not support images as a content array. + * The contentString also is the base64 string WITH `data:image/xxx;base64,` prefix, which may not be the case for all providers. + * If your provider does not work exactly this way, then attachments will not function or potentially break vision requests. + * If you encounter this issue, you are welcome to open an issue asking for your specific provider to be supported. + * + * This function will **not** be updated for providers that **do not** support images as a content array like OpenAI does. + * Do not open issues to update this function due to your specific provider not being compatible. Open an issue to request support for your specific provider. + * @param {Object} props + * @param {string} props.userPrompt - the user prompt to be sent to the model + * @param {import("../../helpers").Attachment[]} props.attachments - the array of attachments to be sent to the model + * @returns {string|object[]} + */ + #generateContent({ userPrompt, attachments = [] }) { + if (!attachments.length) { + return userPrompt; + } + + const content = [{ type: "text", text: userPrompt }]; + for (let attachment of attachments) { + content.push({ + type: "image_url", + image_url: { + url: attachment.contentString, + detail: "high", + }, + }); + } + return content.flat(); + } + + /** + * Construct the user prompt for this model. + * @param {{attachments: import("../../helpers").Attachment[]}} param0 + * @returns + */ constructPrompt({ systemPrompt = "", contextTexts = [], chatHistory = [], userPrompt = "", + attachments = [], }) { const prompt = { role: "system", content: `${systemPrompt}${this.#appendContext(contextTexts)}`, }; - return [prompt, ...chatHistory, { role: "user", content: userPrompt }]; + return [ + prompt, + ...chatHistory, + { + role: "user", + content: this.#generateContent({ userPrompt, attachments }), + }, + ]; } async getChatCompletion(messages = null, { temperature = 0.7 }) { diff --git a/server/utils/files/index.js b/server/utils/files/index.js index 598884f999..625d8582cd 100644 --- a/server/utils/files/index.js +++ b/server/utils/files/index.js @@ -281,6 +281,16 @@ async function getWatchedDocumentFilenames(filenames = []) { }, {}); } +/** + * Purges the entire vector-cache folder and recreates it. + * @returns {void} + */ +function purgeEntireVectorCache() { + fs.rmSync(vectorCachePath, { recursive: true, force: true }); + fs.mkdirSync(vectorCachePath); + return; +} + module.exports = { findDocumentInDocuments, cachedVectorInformation, @@ -293,4 +303,5 @@ module.exports = { isWithin, documentsPath, hasVectorCachedFiles, + purgeEntireVectorCache, }; diff --git a/server/utils/helpers/index.js b/server/utils/helpers/index.js index e599078b6a..748e4fb1b1 100644 --- a/server/utils/helpers/index.js +++ b/server/utils/helpers/index.js @@ -52,10 +52,11 @@ /** * Gets the systems current vector database provider. + * @param {('pinecone' | 'chroma' | 'lancedb' | 'weaviate' | 'qdrant' | 'milvus' | 'zilliz' | 'astra') | null} getExactly - If provided, this will return an explit provider. * @returns { BaseVectorDatabaseProvider} */ -function getVectorDbClass() { - const vectorSelection = process.env.VECTOR_DB || "lancedb"; +function getVectorDbClass(getExactly = null) { + const vectorSelection = getExactly ?? process.env.VECTOR_DB ?? "lancedb"; switch (vectorSelection) { case "pinecone": const { Pinecone } = require("../vectorDbProviders/pinecone"); diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js index 3cfc13e6e1..948703dca2 100644 --- a/server/utils/helpers/updateENV.js +++ b/server/utils/helpers/updateENV.js @@ -1,3 +1,5 @@ +const { resetAllVectorStores } = require("../vectorStore/resetAllVectorStores"); + const KEY_MAPPING = { LLMProvider: { envKey: "LLM_PROVIDER", @@ -248,6 +250,7 @@ const KEY_MAPPING = { EmbeddingEngine: { envKey: "EMBEDDING_ENGINE", checks: [supportedEmbeddingModel], + postUpdate: [handleVectorStoreReset], }, EmbeddingBasePath: { envKey: "EMBEDDING_BASE_PATH", @@ -256,6 +259,7 @@ const KEY_MAPPING = { EmbeddingModelPref: { envKey: "EMBEDDING_MODEL_PREF", checks: [isNotEmpty], + postUpdate: [handleVectorStoreReset], }, EmbeddingModelMaxChunkLength: { envKey: "EMBEDDING_MODEL_MAX_CHUNK_LENGTH", @@ -276,6 +280,7 @@ const KEY_MAPPING = { VectorDB: { envKey: "VECTOR_DB", checks: [isNotEmpty, supportedVectorDB], + postUpdate: [handleVectorStoreReset], }, // Chroma Options @@ -878,6 +883,24 @@ function noRestrictedChars(input = "") { : null; } +async function handleVectorStoreReset(key, prevValue, nextValue) { + if (prevValue === nextValue) return; + if (key === "VectorDB") { + console.log( + `Vector configuration changed from ${prevValue} to ${nextValue} - resetting ${prevValue} namespaces` + ); + return await resetAllVectorStores({ vectorDbKey: prevValue }); + } + + if (key === "EmbeddingEngine" || key === "EmbeddingModelPref") { + console.log( + `${key} changed from ${prevValue} to ${nextValue} - resetting ${process.env.VECTOR_DB} namespaces` + ); + return await resetAllVectorStores({ vectorDbKey: process.env.VECTOR_DB }); + } + return false; +} + // This will force update .env variables which for any which reason were not able to be parsed or // read from an ENV file as this seems to be a complicating step for many so allowing people to write // to the process will at least alleviate that issue. It does not perform comprehensive validity checks or sanity checks diff --git a/server/utils/vectorStore/resetAllVectorStores.js b/server/utils/vectorStore/resetAllVectorStores.js new file mode 100644 index 0000000000..3bb9a5ec4c --- /dev/null +++ b/server/utils/vectorStore/resetAllVectorStores.js @@ -0,0 +1,48 @@ +const { Workspace } = require("../../models/workspace"); +const { Document } = require("../../models/documents"); +const { DocumentVectors } = require("../../models/vectors"); +const { EventLogs } = require("../../models/eventLogs"); +const { purgeEntireVectorCache } = require("../files"); +const { getVectorDbClass } = require("../helpers"); + +/** + * Resets all vector database and associated content: + * - Purges the entire vector-cache folder. + * - Deletes all document vectors from the database. + * - Deletes all documents from the database. + * - Deletes all vector db namespaces for each workspace. + * - Logs an event indicating the reset. + * @param {string} vectorDbKey - The _previous_ vector database provider name that we will be resetting. + * @returns {Promise} - True if successful, false otherwise. + */ +async function resetAllVectorStores({ vectorDbKey }) { + try { + const workspaces = await Workspace.where(); + purgeEntireVectorCache(); // Purges the entire vector-cache folder. + await DocumentVectors.delete(); // Deletes all document vectors from the database. + await Document.delete(); // Deletes all documents from the database. + await EventLogs.logEvent("workspace_vectors_reset", { + reason: "System vector configuration changed", + }); + + console.log( + "Resetting anythingllm managed vector namespaces for", + vectorDbKey + ); + const VectorDb = getVectorDbClass(vectorDbKey); + for (const workspace of workspaces) { + try { + await VectorDb["delete-namespace"]({ namespace: workspace.slug }); + } catch (e) { + console.error(e.message); + } + } + + return true; + } catch (error) { + console.error("Failed to reset vector stores:", error); + return false; + } +} + +module.exports = { resetAllVectorStores };