From 63128ece19f760e553a6aa91c1a621927ff3c44c Mon Sep 17 00:00:00 2001 From: Kris McGinnes Date: Tue, 3 Sep 2024 15:13:02 -0500 Subject: [PATCH] Setting to enable or disable DB query logging (#575) --- Changelog.md | 6 +- .../graph-explorer-proxy-server/src/env.ts | 2 +- .../src/logging.ts | 6 + .../src/node-server.ts | 29 +++- .../src/connector/fetchDatabaseRequest.ts | 8 +- .../src/connector/gremlin/gremlinExplorer.ts | 64 +++++-- .../openCypher/openCypherExplorer.ts | 83 ++++++--- .../src/connector/sparql/sparqlExplorer.ts | 68 +++++--- .../src/connector/useGEFetchTypes.ts | 14 +- .../core/StateProvider/configuration.test.ts | 159 ++++++++++++++++++ .../src/core/StateProvider/configuration.ts | 156 ++++++++++------- .../src/core/StateProvider/userPreferences.ts | 12 +- packages/graph-explorer/src/core/connector.ts | 8 +- .../graph-explorer/src/core/featureFlags.ts | 26 ++- .../src/utils/testing/randomData.ts | 110 +++++++++--- .../workspaces/Settings/SettingsGeneral.tsx | 33 +++- 16 files changed, 618 insertions(+), 166 deletions(-) create mode 100644 packages/graph-explorer/src/core/StateProvider/configuration.test.ts diff --git a/Changelog.md b/Changelog.md index b2a881bad..a8330db3a 100644 --- a/Changelog.md +++ b/Changelog.md @@ -10,8 +10,10 @@ exist ([#542](https://github.com/aws/graph-explorer/pull/542)) - **Added** global error page if the React app crashes ([#547](https://github.com/aws/graph-explorer/pull/547)) -- **Added** server logging of database queries when using the proxy server - ([#574](https://github.com/aws/graph-explorer/pull/574)) +- **Added** optional server logging of database queries when using the proxy + server which can be enabled within settings + ([#574](https://github.com/aws/graph-explorer/pull/574), + [#575](https://github.com/aws/graph-explorer/pull/575)) - **Improved** handling of server errors with more consistent logging ([#557](https://github.com/aws/graph-explorer/pull/557)) - **Transition** to Tailwind instead of EmotionCSS for styles, which should make diff --git a/packages/graph-explorer-proxy-server/src/env.ts b/packages/graph-explorer-proxy-server/src/env.ts index d0f33b5dd..09f111129 100644 --- a/packages/graph-explorer-proxy-server/src/env.ts +++ b/packages/graph-explorer-proxy-server/src/env.ts @@ -4,7 +4,7 @@ import { clientRoot } from "./paths.js"; import { z } from "zod"; /** Coerces a string to a boolean value in a case insensitive way. */ -const BooleanStringSchema = z +export const BooleanStringSchema = z .string() .refine(s => s.toLowerCase() === "true" || s.toLowerCase() === "false") .transform(s => s.toLowerCase() === "true"); diff --git a/packages/graph-explorer-proxy-server/src/logging.ts b/packages/graph-explorer-proxy-server/src/logging.ts index cd27303ac..fc0b47fda 100644 --- a/packages/graph-explorer-proxy-server/src/logging.ts +++ b/packages/graph-explorer-proxy-server/src/logging.ts @@ -73,6 +73,12 @@ export function requestLoggingMiddleware() { return; } + // Ignore CORS options requests + if (req.method === "OPTIONS") { + next(); + return; + } + // Wait for the request to complete. req.on("end", () => { logRequestAndResponse(req, res); diff --git a/packages/graph-explorer-proxy-server/src/node-server.ts b/packages/graph-explorer-proxy-server/src/node-server.ts index fff0c2379..1faef2b23 100644 --- a/packages/graph-explorer-proxy-server/src/node-server.ts +++ b/packages/graph-explorer-proxy-server/src/node-server.ts @@ -12,7 +12,7 @@ import { IncomingHttpHeaders } from "http"; import { logger as proxyLogger, requestLoggingMiddleware } from "./logging.js"; import { clientRoot, proxyServerRoot } from "./paths.js"; import { errorHandlingMiddleware, handleError } from "./error-handler.js"; -import { env } from "./env.js"; +import { BooleanStringSchema, env } from "./env.js"; const app = express(); @@ -23,6 +23,7 @@ interface DbQueryIncomingHttpHeaders extends IncomingHttpHeaders { "graph-db-connection-url"?: string; "aws-neptune-region"?: string; "service-type"?: string; + "db-query-logging-enabled"?: string; } interface LoggerIncomingHttpHeaders extends IncomingHttpHeaders { @@ -175,6 +176,9 @@ app.post("/sparql", (req, res, next) => { const headers = req.headers as DbQueryIncomingHttpHeaders; const queryId = headers["queryid"]; const graphDbConnectionUrl = headers["graph-db-connection-url"]; + const shouldLogDbQuery = BooleanStringSchema.default("false").parse( + headers["db-query-logging-enabled"] + ); const isIamEnabled = !!headers["aws-neptune-region"]; const region = isIamEnabled ? headers["aws-neptune-region"] : ""; const serviceType = isIamEnabled @@ -228,7 +232,11 @@ app.post("/sparql", (req, res, next) => { if (!queryString) { return res.status(400).send({ error: "[Proxy]SPARQL: Query not provided" }); } - proxyLogger.debug("[SPARQL] Received database query:\n%s", queryString); + + if (shouldLogDbQuery) { + proxyLogger.debug("[SPARQL] Received database query:\n%s", queryString); + } + const rawUrl = `${graphDbConnectionUrl}/sparql`; let body = `query=${encodeURIComponent(queryString)}`; if (queryId) { @@ -260,6 +268,9 @@ app.post("/gremlin", (req, res, next) => { const headers = req.headers as DbQueryIncomingHttpHeaders; const queryId = headers["queryid"]; const graphDbConnectionUrl = headers["graph-db-connection-url"]; + const shouldLogDbQuery = BooleanStringSchema.default("false").parse( + headers["db-query-logging-enabled"] + ); const isIamEnabled = !!headers["aws-neptune-region"]; const region = isIamEnabled ? headers["aws-neptune-region"] : ""; const serviceType = isIamEnabled @@ -274,7 +285,9 @@ app.post("/gremlin", (req, res, next) => { .send({ error: "[Proxy]Gremlin: query not provided" }); } - proxyLogger.debug("[Gremlin] Received database query:\n%s", queryString); + if (shouldLogDbQuery) { + proxyLogger.debug("[Gremlin] Received database query:\n%s", queryString); + } /// Function to cancel long running queries if the client disappears before completion async function cancelQuery() { @@ -336,6 +349,11 @@ app.post("/gremlin", (req, res, next) => { // POST endpoint for openCypher queries. app.post("/openCypher", (req, res, next) => { + const headers = req.headers as DbQueryIncomingHttpHeaders; + const shouldLogDbQuery = BooleanStringSchema.default("false").parse( + headers["db-query-logging-enabled"] + ); + const queryString = req.body.query; // Validate the input before making any external calls. if (!queryString) { @@ -344,9 +362,10 @@ app.post("/openCypher", (req, res, next) => { .send({ error: "[Proxy]OpenCypher: query not provided" }); } - proxyLogger.debug("[openCypher] Received database query:\n%s", queryString); + if (shouldLogDbQuery) { + proxyLogger.debug("[openCypher] Received database query:\n%s", queryString); + } - const headers = req.headers as DbQueryIncomingHttpHeaders; const rawUrl = `${headers["graph-db-connection-url"]}/openCypher`; const requestOptions = { method: "POST", diff --git a/packages/graph-explorer/src/connector/fetchDatabaseRequest.ts b/packages/graph-explorer/src/connector/fetchDatabaseRequest.ts index 1a48d8c11..f97e4ec48 100644 --- a/packages/graph-explorer/src/connector/fetchDatabaseRequest.ts +++ b/packages/graph-explorer/src/connector/fetchDatabaseRequest.ts @@ -1,6 +1,7 @@ import { type ConnectionConfig } from "@shared/types"; import { DEFAULT_SERVICE_TYPE } from "@/utils/constants"; import { anySignal } from "./utils/anySignal"; +import { FeatureFlags } from "@/core"; type NeptuneError = { code: string; @@ -76,11 +77,15 @@ async function decodeErrorSafely(response: Response): Promise { // Construct the request headers based on the connection settings function getAuthHeaders( connection: ConnectionConfig | undefined, + featureFlags: FeatureFlags, typeHeaders: HeadersInit | undefined ) { const headers: HeadersInit = {}; if (connection?.proxyConnection) { headers["graph-db-connection-url"] = connection.graphDbUrl || ""; + headers["db-query-logging-enabled"] = String( + featureFlags.allowLoggingDbQuery + ); } if (connection?.awsAuthEnabled) { headers["aws-neptune-region"] = connection.awsRegion || ""; @@ -105,13 +110,14 @@ function getFetchTimeoutSignal(connection: ConnectionConfig | undefined) { export async function fetchDatabaseRequest( connection: ConnectionConfig | undefined, + featureFlags: FeatureFlags, uri: URL | RequestInfo, options: RequestInit ) { // Apply connection settings to fetch options const fetchOptions: RequestInit = { ...options, - headers: getAuthHeaders(connection, options.headers), + headers: getAuthHeaders(connection, featureFlags, options.headers), signal: anySignal(getFetchTimeoutSignal(connection), options.signal), }; diff --git a/packages/graph-explorer/src/connector/gremlin/gremlinExplorer.ts b/packages/graph-explorer/src/connector/gremlin/gremlinExplorer.ts index 3615816b9..90ebe0dd8 100644 --- a/packages/graph-explorer/src/connector/gremlin/gremlinExplorer.ts +++ b/packages/graph-explorer/src/connector/gremlin/gremlinExplorer.ts @@ -7,11 +7,16 @@ import keywordSearch from "./queries/keywordSearch"; import { fetchDatabaseRequest } from "../fetchDatabaseRequest"; import { GraphSummary } from "./types"; import { v4 } from "uuid"; -import { Explorer } from "../useGEFetchTypes"; +import { Explorer, ExplorerRequestOptions } from "../useGEFetchTypes"; import { logger } from "@/utils"; import { createLoggerFromConnection } from "@/core/connector"; +import { FeatureFlags } from "@/core"; -function _gremlinFetch(connection: ConnectionConfig, options: any) { +function _gremlinFetch( + connection: ConnectionConfig, + featureFlags: FeatureFlags, + options?: ExplorerRequestOptions +) { return async (queryTemplate: string) => { logger.debug(queryTemplate); const body = JSON.stringify({ query: queryTemplate }); @@ -23,22 +28,29 @@ function _gremlinFetch(connection: ConnectionConfig, options: any) { headers.queryId = options.queryId; } - return fetchDatabaseRequest(connection, `${connection.url}/gremlin`, { - method: "POST", - headers, - body, - ...options, - }); + return fetchDatabaseRequest( + connection, + featureFlags, + `${connection.url}/gremlin`, + { + method: "POST", + headers, + body, + ...options, + } + ); }; } async function fetchSummary( connection: ConnectionConfig, - options: RequestInit + featureFlags: FeatureFlags, + options?: RequestInit ) { try { const response = await fetchDatabaseRequest( connection, + featureFlags, `${connection.url}/pg/statistics/summary?mode=detailed`, { method: "GET", @@ -54,33 +66,51 @@ async function fetchSummary( } } -export function createGremlinExplorer(connection: ConnectionConfig): Explorer { +export function createGremlinExplorer( + connection: ConnectionConfig, + featureFlags: FeatureFlags +): Explorer { const remoteLogger = createLoggerFromConnection(connection); return { connection: connection, async fetchSchema(options) { remoteLogger.info("[Gremlin Explorer] Fetching schema..."); - const summary = await fetchSummary(connection, options); - return fetchSchema(_gremlinFetch(connection, options), summary); + const summary = await fetchSummary(connection, featureFlags, options); + return fetchSchema( + _gremlinFetch(connection, featureFlags, options), + summary + ); }, async fetchVertexCountsByType(req, options) { remoteLogger.info("[Gremlin Explorer] Fetching vertex counts by type..."); - return fetchVertexTypeCounts(_gremlinFetch(connection, options), req); + return fetchVertexTypeCounts( + _gremlinFetch(connection, featureFlags, options), + req + ); }, async fetchNeighbors(req, options) { remoteLogger.info("[Gremlin Explorer] Fetching neighbors..."); - return fetchNeighbors(_gremlinFetch(connection, options), req); + return fetchNeighbors( + _gremlinFetch(connection, featureFlags, options), + req + ); }, async fetchNeighborsCount(req, options) { remoteLogger.info("[Gremlin Explorer] Fetching neighbors count..."); - return fetchNeighborsCount(_gremlinFetch(connection, options), req); + return fetchNeighborsCount( + _gremlinFetch(connection, featureFlags, options), + req + ); }, async keywordSearch(req, options) { options ??= {}; options.queryId = v4(); remoteLogger.info("[Gremlin Explorer] Fetching keyword search..."); - return keywordSearch(_gremlinFetch(connection, options), req); + return keywordSearch( + _gremlinFetch(connection, featureFlags, options), + req + ); }, - }; + } satisfies Explorer; } diff --git a/packages/graph-explorer/src/connector/openCypher/openCypherExplorer.ts b/packages/graph-explorer/src/connector/openCypher/openCypherExplorer.ts index 81f4b1579..2123e64ec 100644 --- a/packages/graph-explorer/src/connector/openCypher/openCypherExplorer.ts +++ b/packages/graph-explorer/src/connector/openCypher/openCypherExplorer.ts @@ -7,26 +7,37 @@ import { GraphSummary } from "./types"; import { fetchDatabaseRequest } from "../fetchDatabaseRequest"; import { ConnectionConfig } from "@shared/types"; import { DEFAULT_SERVICE_TYPE } from "@/utils/constants"; -import { Explorer } from "../useGEFetchTypes"; +import { Explorer, ExplorerRequestOptions } from "../useGEFetchTypes"; import { env, logger } from "@/utils"; import { createLoggerFromConnection } from "@/core/connector"; +import { FeatureFlags } from "@/core"; -function _openCypherFetch(connection: ConnectionConfig, options: any) { +function _openCypherFetch( + connection: ConnectionConfig, + featureFlags: FeatureFlags, + options?: ExplorerRequestOptions +) { return async (queryTemplate: string) => { logger.debug(queryTemplate); - return fetchDatabaseRequest(connection, `${connection.url}/openCypher`, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ query: queryTemplate }), - ...options, - }); + return fetchDatabaseRequest( + connection, + featureFlags, + `${connection.url}/openCypher`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ query: queryTemplate }), + ...options, + } + ); }; } export function createOpenCypherExplorer( - connection: ConnectionConfig + connection: ConnectionConfig, + featureFlags: FeatureFlags ): Explorer { const remoteLogger = createLoggerFromConnection(connection); const serviceType = connection.serviceType || DEFAULT_SERVICE_TYPE; @@ -34,44 +45,70 @@ export function createOpenCypherExplorer( connection: connection, async fetchSchema(options) { remoteLogger.info("[openCypher Explorer] Fetching schema..."); - const summary = await fetchSummary(serviceType, connection, options); - return fetchSchema(_openCypherFetch(connection, options), summary); + const summary = await fetchSummary( + serviceType, + connection, + featureFlags, + options + ); + return fetchSchema( + _openCypherFetch(connection, featureFlags, options), + summary + ); }, async fetchVertexCountsByType(req, options) { remoteLogger.info( "[openCypher Explorer] Fetching vertex counts by type..." ); - return fetchVertexTypeCounts(_openCypherFetch(connection, options), req); + return fetchVertexTypeCounts( + _openCypherFetch(connection, featureFlags, options), + req + ); }, async fetchNeighbors(req, options) { remoteLogger.info("[openCypher Explorer] Fetching neighbors..."); - return fetchNeighbors(_openCypherFetch(connection, options), req); + return fetchNeighbors( + _openCypherFetch(connection, featureFlags, options), + req + ); }, async fetchNeighborsCount(req, options) { remoteLogger.info("[openCypher Explorer] Fetching neighbors count..."); - return fetchNeighborsCount(_openCypherFetch(connection, options), req); + return fetchNeighborsCount( + _openCypherFetch(connection, featureFlags, options), + req + ); }, async keywordSearch(req, options) { remoteLogger.info("[openCypher Explorer] Fetching keyword search..."); - return keywordSearch(_openCypherFetch(connection, options), req); + return keywordSearch( + _openCypherFetch(connection, featureFlags, options), + req + ); }, - }; + } satisfies Explorer; } async function fetchSummary( serviceType: string, connection: ConnectionConfig, - options: RequestInit + featureFlags: FeatureFlags, + options?: RequestInit ) { try { const endpoint = serviceType === DEFAULT_SERVICE_TYPE ? `${connection.url}/pg/statistics/summary?mode=detailed` : `${connection.url}/summary?mode=detailed`; - const response = await fetchDatabaseRequest(connection, endpoint, { - method: "GET", - ...options, - }); + const response = await fetchDatabaseRequest( + connection, + featureFlags, + endpoint, + { + method: "GET", + ...options, + } + ); return ( (response.payload diff --git a/packages/graph-explorer/src/connector/sparql/sparqlExplorer.ts b/packages/graph-explorer/src/connector/sparql/sparqlExplorer.ts index c24e6ed8c..7f5d38ad4 100644 --- a/packages/graph-explorer/src/connector/sparql/sparqlExplorer.ts +++ b/packages/graph-explorer/src/connector/sparql/sparqlExplorer.ts @@ -1,6 +1,7 @@ import type { Criterion, Explorer, + ExplorerRequestOptions, KeywordSearchRequest, KeywordSearchResponse, NeighborsResponse, @@ -24,6 +25,7 @@ import { ConnectionConfig } from "@shared/types"; import { v4 } from "uuid"; import { env, logger } from "@/utils"; import { createLoggerFromConnection } from "@/core/connector"; +import { FeatureFlags } from "@/core"; const replaceBlankNodeFromSearch = ( blankNodes: BlankNodesMap, @@ -133,37 +135,49 @@ const storedBlankNodeNeighborsRequest = ( }); }; -function _sparqlFetch(connection: ConnectionConfig, options?: any) { +function _sparqlFetch( + connection: ConnectionConfig, + featureFlags: FeatureFlags, + options?: ExplorerRequestOptions +) { return async (queryTemplate: string) => { logger.debug(queryTemplate); const body = `query=${encodeURIComponent(queryTemplate)}`; - const headers = - options?.queryId && connection.proxyConnection === true + const queryId = options?.queryId; + const headers: Record = + queryId && connection.proxyConnection === true ? { accept: "application/sparql-results+json", "Content-Type": "application/x-www-form-urlencoded", - queryId: options.queryId, + queryId: queryId, } : { accept: "application/sparql-results+json", "Content-Type": "application/x-www-form-urlencoded", }; - return fetchDatabaseRequest(connection, `${connection.url}/sparql`, { - method: "POST", - headers, - body, - ...options, - }); + return fetchDatabaseRequest( + connection, + featureFlags, + `${connection.url}/sparql`, + { + method: "POST", + headers, + body, + ...options, + } + ); }; } async function fetchSummary( connection: ConnectionConfig, - options: RequestInit + featureFlags: FeatureFlags, + options?: RequestInit ) { try { const response = await fetchDatabaseRequest( connection, + featureFlags, `${connection.url}/rdf/statistics/summary?mode=detailed`, { method: "GET", @@ -181,6 +195,7 @@ async function fetchSummary( export function createSparqlExplorer( connection: ConnectionConfig, + featureFlags: FeatureFlags, blankNodes: BlankNodesMap ): Explorer { const remoteLogger = createLoggerFromConnection(connection); @@ -188,12 +203,18 @@ export function createSparqlExplorer( connection: connection, async fetchSchema(options) { remoteLogger.info("[SPARQL Explorer] Fetching schema..."); - const summary = await fetchSummary(connection, options); - return fetchSchema(_sparqlFetch(connection, options), summary); + const summary = await fetchSummary(connection, featureFlags, options); + return fetchSchema( + _sparqlFetch(connection, featureFlags, options), + summary + ); }, async fetchVertexCountsByType(req, options) { remoteLogger.info("[SPARQL Explorer] Fetching vertex counts by type..."); - return fetchClassCounts(_sparqlFetch(connection, options), req); + return fetchClassCounts( + _sparqlFetch(connection, featureFlags, options), + req + ); }, async fetchNeighbors(req, options) { remoteLogger.info("[SPARQL Explorer] Fetching neighbors..."); @@ -215,7 +236,7 @@ export function createSparqlExplorer( } const response = await fetchNeighbors( - _sparqlFetch(connection, options), + _sparqlFetch(connection, featureFlags, options), request ); const vertices = replaceBlankNodeFromNeighbors( @@ -238,7 +259,7 @@ export function createSparqlExplorer( if (bNode && !bNode.neighbors) { const response = await fetchBlankNodeNeighbors( - _sparqlFetch(connection, options), + _sparqlFetch(connection, featureFlags, options), { resourceURI: bNode.vertex.data.id, resourceClass: bNode.vertex.data.type, @@ -265,10 +286,13 @@ export function createSparqlExplorer( }; } - return fetchNeighborsCount(_sparqlFetch(connection, options), { - resourceURI: req.vertexId, - limit: req.limit, - }); + return fetchNeighborsCount( + _sparqlFetch(connection, featureFlags, options), + { + resourceURI: req.vertexId, + limit: req.limit, + } + ); }, async keywordSearch(req, options) { options ??= {}; @@ -286,7 +310,7 @@ export function createSparqlExplorer( }; const response = await keywordSearch( - _sparqlFetch(connection, options), + _sparqlFetch(connection, featureFlags, options), reqParams ); const vertices = replaceBlankNodeFromSearch( @@ -297,5 +321,5 @@ export function createSparqlExplorer( return { vertices }; }, - }; + } satisfies Explorer; } diff --git a/packages/graph-explorer/src/connector/useGEFetchTypes.ts b/packages/graph-explorer/src/connector/useGEFetchTypes.ts index 33415aade..be68ca1ee 100644 --- a/packages/graph-explorer/src/connector/useGEFetchTypes.ts +++ b/packages/graph-explorer/src/connector/useGEFetchTypes.ts @@ -211,27 +211,31 @@ export type ConfigurationWithConnection = Omit< > & Required>; +export type ExplorerRequestOptions = RequestInit & { + queryId?: string; +}; + /** * Abstracted interface to the common database queries used by * Graph Explorer. */ export type Explorer = { connection: ConnectionConfig; - fetchSchema: (options?: any) => Promise; + fetchSchema: (options?: ExplorerRequestOptions) => Promise; fetchVertexCountsByType: ( req: CountsByTypeRequest, - options?: any + options?: ExplorerRequestOptions ) => Promise; fetchNeighbors: ( req: NeighborsRequest, - options?: any + options?: ExplorerRequestOptions ) => Promise; fetchNeighborsCount: ( req: NeighborsCountRequest, - options?: any + options?: ExplorerRequestOptions ) => Promise; keywordSearch: ( req: KeywordSearchRequest, - options?: any + options?: ExplorerRequestOptions ) => Promise; }; diff --git a/packages/graph-explorer/src/core/StateProvider/configuration.test.ts b/packages/graph-explorer/src/core/StateProvider/configuration.test.ts new file mode 100644 index 000000000..c95ddd3c0 --- /dev/null +++ b/packages/graph-explorer/src/core/StateProvider/configuration.test.ts @@ -0,0 +1,159 @@ +import { + createRandomEdgePreferences, + createRandomRawConfiguration, + createRandomSchema, + createRandomVertexPreferences, +} from "@/utils/testing"; +import { mergeConfiguration } from "./configuration"; +import { RawConfiguration, VertexTypeConfig } from "../ConfigurationProvider"; +import DEFAULT_ICON_URL from "@/utils/defaultIconUrl"; +import { SchemaInference } from "./schema"; +import { UserStyling } from "./userPreferences"; +import { sanitizeText } from "@/utils"; + +const defaultVertexStyle = { + color: "#128EE5", + iconUrl: DEFAULT_ICON_URL, + iconImageType: "image/svg+xml", + displayNameAttribute: "id", + longDisplayNameAttribute: "types", +}; + +const defaultEdgeStyle = { + type: "unknown", + displayLabel: "Unknown", +}; + +describe("mergedConfiguration", () => { + it("should produce empty defaults when empty object is passed", () => { + const config = {} as RawConfiguration; + const result = mergeConfiguration(null, config, {}); + + expect(result).toEqual({ + connection: { + graphDbUrl: "", + queryEngine: "gremlin", + url: "", + }, + schema: { + edges: [], + vertices: [], + totalEdges: 0, + totalVertices: 0, + }, + }); + }); + + it("should produce empty schema when no schema provided", () => { + const config = createRandomRawConfiguration(); + const result = mergeConfiguration(null, config, {}); + + expect(result).toEqual({ + ...config, + connection: { + url: "", + graphDbUrl: "", + ...config.connection, + }, + schema: { + edges: [], + vertices: [], + totalEdges: 0, + totalVertices: 0, + }, + } satisfies RawConfiguration); + }); + + it("should use schema when provided", () => { + const config = createRandomRawConfiguration(); + const schema = createRandomSchema(); + const result = mergeConfiguration(schema, config, {}); + + const expectedSchema = { + ...schema, + vertices: schema.vertices + .map(v => ({ + ...defaultVertexStyle, + displayLabel: sanitizeText(v.type), + ...v, + })) + .sort(byType), + edges: schema.edges.map(e => { + return { + ...defaultEdgeStyle, + ...e, + }; + }), + } satisfies SchemaInference; + + expect(result.schema?.vertices).toEqual(expectedSchema.vertices); + expect(result.schema?.edges).toEqual(expectedSchema.edges); + expect(result.schema).toEqual(expectedSchema); + expect(result).toEqual({ + ...config, + connection: { + ...config.connection, + url: config.connection?.url ?? "", + graphDbUrl: config.connection?.graphDbUrl ?? "", + }, + schema: expectedSchema, + } satisfies RawConfiguration); + }); + + it("should use styling when provided", () => { + const config = createRandomRawConfiguration(); + const schema = createRandomSchema(); + const styling: UserStyling = { + vertices: schema.vertices.map(v => ({ + ...createRandomVertexPreferences(), + type: v.type, + })), + edges: schema.edges.map(v => ({ + ...createRandomEdgePreferences(), + type: v.type, + })), + }; + const result = mergeConfiguration(schema, config, styling); + + const expectedSchema = { + ...schema, + vertices: schema.vertices + .map(v => { + const style = styling.vertices?.find(s => s.type === v.type) ?? {}; + return { + ...defaultVertexStyle, + displayLabel: sanitizeText(v.type), + ...v, + ...style, + }; + }) + .sort(byType), + edges: schema.edges.map(e => { + const style = styling.edges?.find(s => s.type === e.type) ?? {}; + return { + ...defaultEdgeStyle, + ...e, + ...style, + }; + }), + } satisfies SchemaInference; + + expect(result.schema?.vertices).toEqual(expectedSchema.vertices); + expect(result.schema?.edges).toEqual(expectedSchema.edges); + expect(result.schema).toEqual(expectedSchema); + expect(result).toEqual({ + ...config, + connection: { + ...config.connection, + url: config.connection?.url ?? "", + graphDbUrl: config.connection?.graphDbUrl ?? "", + }, + schema: expectedSchema, + }); + }); +}); + +/** Sorts the configs by type name */ +function byType(a: VertexTypeConfig, b: VertexTypeConfig) { + return a.type.localeCompare(b.type); +} diff --git a/packages/graph-explorer/src/core/StateProvider/configuration.ts b/packages/graph-explorer/src/core/StateProvider/configuration.ts index 41b951839..47bbe16f8 100644 --- a/packages/graph-explorer/src/core/StateProvider/configuration.ts +++ b/packages/graph-explorer/src/core/StateProvider/configuration.ts @@ -9,12 +9,14 @@ import type { VertexTypeConfig, } from "../ConfigurationProvider"; import localForageEffect from "./localForageEffect"; -import { schemaAtom } from "./schema"; +import { activeSchemaSelector, SchemaInference } from "./schema"; import { EdgePreferences, + UserStyling, userStylingAtom, VertexPreferences, } from "./userPreferences"; +import isDefaultValue from "./isDefaultValue"; export const isStoreLoadedAtom = atom({ key: "store-loaded", @@ -33,78 +35,110 @@ export const configurationAtom = atom>({ effects: [localForageEffect()], }); +/** Gets or sets the config that is currently active. */ +export const activeConfigSelector = selector({ + key: "active-config-selector", + get({ get }) { + const configMap = get(configurationAtom); + const id = get(activeConfigurationAtom); + const activeConfig = id ? configMap.get(id) : null; + return activeConfig; + }, + set({ get, set }, newValue) { + const configId = get(activeConfigurationAtom); + if (!configId) { + return; + } + set(configurationAtom, prevConfigMap => { + const updatedConfigMap = new Map(prevConfigMap); + + // Handle reset value + if (!newValue || isDefaultValue(newValue)) { + updatedConfigMap.delete(configId); + return updatedConfigMap; + } + + // Update the map + updatedConfigMap.set(configId, newValue); + + return updatedConfigMap; + }); + }, +}); + export const mergedConfigurationSelector = selector({ key: "merged-configuration", get: ({ get }) => { - const activeConfig = get(activeConfigurationAtom); - const config = get(configurationAtom); - const currentConfig = activeConfig && config.get(activeConfig); + const currentConfig = get(activeConfigSelector); if (!currentConfig) { return null; } - const schema = get(schemaAtom); - const currentSchema = activeConfig ? schema.get(activeConfig) : null; + const currentSchema = get(activeSchemaSelector); const userStyling = get(userStylingAtom); - const configVLabels = currentConfig.schema?.vertices.map(v => v.type) || []; - const schemaVLabels = currentSchema?.vertices?.map(v => v.type) || []; - const allVertexLabels = uniq([...configVLabels, ...schemaVLabels]); - const mergedVertices = allVertexLabels - .map(vLabel => { - const configVertex = currentConfig.schema?.vertices.find( - v => v.type === vLabel - ); - const schemaVertex = currentSchema?.vertices.find( - v => v.type === vLabel - ); - const prefsVertex = userStyling.vertices?.find(v => v.type === vLabel); - - return mergeVertex(configVertex, schemaVertex, prefsVertex); - }) - .sort((a, b) => a.type.localeCompare(b.type)); - - const configELabels = currentConfig.schema?.edges.map(v => v.type) || []; - const schemaELabels = currentSchema?.edges?.map(v => v.type) || []; - const allEdgeLabels = uniq([...configELabels, ...schemaELabels]); - const mergedEdges = allEdgeLabels.map(vLabel => { - const configEdge = currentConfig.schema?.edges.find( + return mergeConfiguration(currentSchema, currentConfig, userStyling); + }, +}); + +export function mergeConfiguration( + currentSchema: SchemaInference | null | undefined, + currentConfig: RawConfiguration, + userStyling: UserStyling +): RawConfiguration { + const configVLabels = currentConfig.schema?.vertices.map(v => v.type) || []; + const schemaVLabels = currentSchema?.vertices?.map(v => v.type) || []; + const allVertexLabels = uniq([...configVLabels, ...schemaVLabels]); + const mergedVertices = allVertexLabels + .map(vLabel => { + const configVertex = currentConfig.schema?.vertices.find( v => v.type === vLabel ); - const schemaEdge = currentSchema?.edges.find(v => v.type === vLabel); - const prefsEdge = userStyling.edges?.find(v => v.type === vLabel); - return mergeEdge(configEdge, schemaEdge, prefsEdge); - }); + const schemaVertex = currentSchema?.vertices.find(v => v.type === vLabel); + const prefsVertex = userStyling.vertices?.find(v => v.type === vLabel); + + return mergeVertex(configVertex, schemaVertex, prefsVertex); + }) + .sort((a, b) => a.type.localeCompare(b.type)); + + const configELabels = currentConfig.schema?.edges.map(v => v.type) || []; + const schemaELabels = currentSchema?.edges?.map(v => v.type) || []; + const allEdgeLabels = uniq([...configELabels, ...schemaELabels]); + const mergedEdges = allEdgeLabels.map(vLabel => { + const configEdge = currentConfig.schema?.edges.find(v => v.type === vLabel); + const schemaEdge = currentSchema?.edges.find(v => v.type === vLabel); + const prefsEdge = userStyling.edges?.find(v => v.type === vLabel); + return mergeEdge(configEdge, schemaEdge, prefsEdge); + }); - return { - id: currentConfig.id, - displayLabel: currentConfig.displayLabel, - remoteConfigFile: currentConfig.remoteConfigFile, - __fileBase: currentConfig.__fileBase, - connection: { - ...(currentConfig.connection || {}), - // Remove trailing slash - url: currentConfig.connection?.url?.replace(/\/$/, "") || "", - queryEngine: currentConfig.connection?.queryEngine || "gremlin", - graphDbUrl: - currentConfig.connection?.graphDbUrl?.replace(/\/$/, "") || "", - }, - schema: { - vertices: mergedVertices, - edges: mergedEdges, - lastUpdate: currentSchema?.lastUpdate, - prefixes: - currentConfig.connection?.queryEngine === "sparql" - ? currentSchema?.prefixes - : undefined, - triedToSync: currentSchema?.triedToSync, - lastSyncFail: currentSchema?.lastSyncFail, - totalVertices: currentSchema?.totalVertices ?? 0, - totalEdges: currentSchema?.totalEdges ?? 0, - }, - }; - }, -}); + return { + id: currentConfig.id, + displayLabel: currentConfig.displayLabel, + remoteConfigFile: currentConfig.remoteConfigFile, + __fileBase: currentConfig.__fileBase, + connection: { + ...(currentConfig.connection || {}), + // Remove trailing slash + url: currentConfig.connection?.url?.replace(/\/$/, "") || "", + queryEngine: currentConfig.connection?.queryEngine || "gremlin", + graphDbUrl: + currentConfig.connection?.graphDbUrl?.replace(/\/$/, "") || "", + }, + schema: { + vertices: mergedVertices, + edges: mergedEdges, + lastUpdate: currentSchema?.lastUpdate, + prefixes: + currentConfig.connection?.queryEngine === "sparql" + ? currentSchema?.prefixes + : undefined, + triedToSync: currentSchema?.triedToSync, + lastSyncFail: currentSchema?.lastSyncFail, + totalVertices: currentSchema?.totalVertices ?? 0, + totalEdges: currentSchema?.totalEdges ?? 0, + }, + }; +} const mergeAttributes = ( config?: VertexTypeConfig | EdgeTypeConfig, diff --git a/packages/graph-explorer/src/core/StateProvider/userPreferences.ts b/packages/graph-explorer/src/core/StateProvider/userPreferences.ts index 72eaa3c2c..1529d6658 100644 --- a/packages/graph-explorer/src/core/StateProvider/userPreferences.ts +++ b/packages/graph-explorer/src/core/StateProvider/userPreferences.ts @@ -90,6 +90,11 @@ export type EdgePreferences = { targetArrowStyle?: ArrowStyle; }; +export type UserStyling = { + vertices?: Array; + edges?: Array; +}; + export type UserPreferences = { layout: { activeToggles: Set; @@ -99,13 +104,10 @@ export type UserPreferences = { }; detailsAutoOpenOnSelection?: boolean; }; - styling: { - vertices?: Array; - edges?: Array; - }; + styling: UserStyling; }; -export const userStylingAtom = atom({ +export const userStylingAtom = atom({ key: "user-styling", default: {}, effects: [localForageEffect()], diff --git a/packages/graph-explorer/src/core/connector.ts b/packages/graph-explorer/src/core/connector.ts index 023bd9db1..a5ed3ae7a 100644 --- a/packages/graph-explorer/src/core/connector.ts +++ b/packages/graph-explorer/src/core/connector.ts @@ -12,6 +12,7 @@ import { selector } from "recoil"; import { equalSelector } from "@/utils/recoilState"; import { ConnectionConfig } from "@shared/types"; import { logger } from "@/utils"; +import { featureFlagsSelector } from "./featureFlags"; /** * Active connection where the value will only change when one of the @@ -51,17 +52,18 @@ export const explorerSelector = selector({ key: "explorer", get: ({ get }) => { const connection = get(activeConnectionSelector); + const featureFlags = get(featureFlagsSelector); if (!connection) { return null; } switch (connection.queryEngine) { case "openCypher": - return createOpenCypherExplorer(connection); + return createOpenCypherExplorer(connection, featureFlags); case "sparql": - return createSparqlExplorer(connection, new Map()); + return createSparqlExplorer(connection, featureFlags, new Map()); default: - return createGremlinExplorer(connection); + return createGremlinExplorer(connection, featureFlags); } }, }); diff --git a/packages/graph-explorer/src/core/featureFlags.ts b/packages/graph-explorer/src/core/featureFlags.ts index 253bf653a..c9ffcf7b5 100644 --- a/packages/graph-explorer/src/core/featureFlags.ts +++ b/packages/graph-explorer/src/core/featureFlags.ts @@ -1,4 +1,4 @@ -import { atom } from "recoil"; +import { atom, selector } from "recoil"; import { asyncLocalForageEffect } from "./StateProvider/localForageEffect"; /** Shows Recoil diff logs in the browser console. */ @@ -14,3 +14,27 @@ export const showDebugActionsAtom = atom({ default: false, effects: [asyncLocalForageEffect("showDebugActions")], }); + +/** Shows debug actions in various places around the app. */ +export const allowLoggingDbQueryAtom = atom({ + key: "feature-flag-db-query-logging", + default: false, + effects: [asyncLocalForageEffect("allowLoggingDbQuery")], +}); + +export type FeatureFlags = { + showRecoilStateLogging: boolean; + showDebugActions: boolean; + allowLoggingDbQuery: boolean; +}; + +export const featureFlagsSelector = selector({ + key: "feature-flags", + get: ({ get }) => { + return { + showRecoilStateLogging: get(showRecoilStateLoggingAtom), + showDebugActions: get(showDebugActionsAtom), + allowLoggingDbQuery: get(allowLoggingDbQueryAtom), + } satisfies FeatureFlags; + }, +}); diff --git a/packages/graph-explorer/src/utils/testing/randomData.ts b/packages/graph-explorer/src/utils/testing/randomData.ts index 4a475451f..2ea223de2 100644 --- a/packages/graph-explorer/src/utils/testing/randomData.ts +++ b/packages/graph-explorer/src/utils/testing/randomData.ts @@ -1,6 +1,7 @@ import { AttributeConfig, EdgeTypeConfig, + FeatureFlags, RawConfiguration, Schema, VertexTypeConfig, @@ -10,12 +11,18 @@ import { Entities } from "@/core/StateProvider/entitiesSelector"; import { createArray, createRandomBoolean, + createRandomColor, createRandomInteger, createRandomName, createRandomUrlString, createRecord, randomlyUndefined, } from "@shared/utils/testing"; +import { + EdgePreferences, + UserStyling, + VertexPreferences, +} from "@/core/StateProvider/userPreferences"; /* @@ -36,12 +43,15 @@ affected by those values, regardless of what they are. * @returns A random AttributeConfig object. */ export function createRandomAttributeConfig(): AttributeConfig { + const dataType = randomlyUndefined(createRandomName("dataType")); + const hidden = randomlyUndefined(createRandomBoolean()); + const searchable = randomlyUndefined(createRandomBoolean()); return { name: createRandomName("name"), displayLabel: createRandomName("displayLabel"), - dataType: randomlyUndefined(createRandomName("dataType")), - hidden: randomlyUndefined(createRandomBoolean()), - searchable: randomlyUndefined(createRandomBoolean()), + ...(dataType && { dataType }), + ...(hidden && { hidden }), + ...(searchable && { searchable }), }; } @@ -50,11 +60,13 @@ export function createRandomAttributeConfig(): AttributeConfig { * @returns A random EdgeTypeConfig object. */ export function createRandomEdgeTypeConfig(): EdgeTypeConfig { + const displayLabel = randomlyUndefined(createRandomName("displayLabel")); + const hidden = randomlyUndefined(createRandomBoolean()); return { type: createRandomName("type"), attributes: createArray(6, createRandomAttributeConfig), - displayLabel: randomlyUndefined(createRandomName("displayLabel")), - hidden: randomlyUndefined(createRandomBoolean()), + ...(displayLabel && { displayLabel }), + ...(hidden && { hidden }), total: createRandomInteger(), }; } @@ -64,11 +76,13 @@ export function createRandomEdgeTypeConfig(): EdgeTypeConfig { * @returns A random VertexTypeConfig object. */ export function createRandomVertexTypeConfig(): VertexTypeConfig { + const displayLabel = randomlyUndefined(createRandomName("displayLabel")); + const hidden = randomlyUndefined(createRandomBoolean()); return { type: createRandomName("type"), attributes: createArray(6, createRandomAttributeConfig), - displayLabel: randomlyUndefined(createRandomName("displayLabel")), - hidden: randomlyUndefined(createRandomBoolean()), + ...(displayLabel && { displayLabel }), + ...(hidden && { hidden }), total: createRandomInteger(), }; } @@ -177,25 +191,83 @@ function pickRandomElement(array: T[]): T { export function createRandomRawConfiguration(): RawConfiguration { const isProxyConnection = createRandomBoolean(); const isIamEnabled = createRandomBoolean(); + const fetchTimeoutMs = randomlyUndefined(createRandomInteger()); + const nodeExpansionLimit = randomlyUndefined(createRandomInteger()); + const serviceType = randomlyUndefined( + pickRandomElement(["neptune-db", "neptune-graph"] as const) + ); return { id: createRandomName("id"), displayLabel: createRandomName("displayLabel"), connection: { url: createRandomUrlString(), - graphDbUrl: isProxyConnection ? createRandomUrlString() : undefined, + ...(isProxyConnection && { graphDbUrl: createRandomUrlString() }), queryEngine: pickRandomElement(["gremlin", "openCypher", "sparql"]), proxyConnection: isProxyConnection, - awsAuthEnabled: isIamEnabled ? createRandomBoolean() : undefined, - awsRegion: isIamEnabled - ? pickRandomElement(["us-west-1", "us-west-2", "us-east-1"]) - : undefined, - fetchTimeoutMs: randomlyUndefined(createRandomInteger()), - nodeExpansionLimit: randomlyUndefined(createRandomInteger()), - serviceType: pickRandomElement([ - "neptune-db", - "neptune-graph", - undefined, - ]), + ...(isIamEnabled && { awsAuthEnabled: createRandomBoolean() }), + ...(isIamEnabled && { + awsRegion: pickRandomElement(["us-west-1", "us-west-2", "us-east-1"]), + }), + ...(fetchTimeoutMs && { fetchTimeoutMs }), + ...(nodeExpansionLimit && { nodeExpansionLimit }), + ...(serviceType && { serviceType }), }, }; } + +export function createRandomVertexPreferences(): VertexPreferences { + const color = randomlyUndefined(createRandomColor()); + const borderColor = randomlyUndefined(createRandomColor()); + const iconUrl = randomlyUndefined(createRandomUrlString()); + const longDisplayNameAttribute = randomlyUndefined( + createRandomName("LongDisplayNameAttribute") + ); + const displayNameAttribute = randomlyUndefined( + createRandomName("DisplayNameAttribute") + ); + const displayLabel = randomlyUndefined(createRandomName("DisplayLabel")); + return { + type: createRandomName("VertexType"), + ...(displayLabel && { displayLabel }), + ...(displayNameAttribute && { displayNameAttribute }), + ...(longDisplayNameAttribute && { longDisplayNameAttribute }), + ...(color && { color }), + ...(borderColor && { borderColor }), + ...(iconUrl && { iconUrl }), + }; +} + +export function createRandomEdgePreferences(): EdgePreferences { + const displayLabel = randomlyUndefined(createRandomName("DisplayLabel")); + const displayNameAttribute = randomlyUndefined( + createRandomName("DisplayNameAttribute") + ); + const lineColor = randomlyUndefined(createRandomColor()); + const labelColor = randomlyUndefined(createRandomColor()); + const labelBorderColor = randomlyUndefined(createRandomColor()); + const lineThickness = randomlyUndefined(createRandomInteger(25)); + return { + type: createRandomName("EdgeType"), + ...(displayLabel && { displayLabel }), + ...(displayNameAttribute && { displayNameAttribute }), + ...(lineColor && { lineColor }), + ...(labelColor && { labelColor }), + ...(labelBorderColor && { labelBorderColor }), + ...(lineThickness && { lineThickness }), + }; +} + +export function createRandomUserStyling(): UserStyling { + return { + vertices: createArray(3, createRandomVertexPreferences), + edges: createArray(3, createRandomEdgePreferences), + }; +} + +export function createRandomFeatureFlags(): FeatureFlags { + return { + showRecoilStateLogging: createRandomBoolean(), + showDebugActions: createRandomBoolean(), + allowLoggingDbQuery: createRandomBoolean(), + }; +} diff --git a/packages/graph-explorer/src/workspaces/Settings/SettingsGeneral.tsx b/packages/graph-explorer/src/workspaces/Settings/SettingsGeneral.tsx index c9aa935d1..93757407d 100644 --- a/packages/graph-explorer/src/workspaces/Settings/SettingsGeneral.tsx +++ b/packages/graph-explorer/src/workspaces/Settings/SettingsGeneral.tsx @@ -1,5 +1,9 @@ import { useRecoilState } from "recoil"; -import { showDebugActionsAtom, showRecoilStateLoggingAtom } from "@/core"; +import { + allowLoggingDbQueryAtom, + showDebugActionsAtom, + showRecoilStateLoggingAtom, +} from "@/core"; import { Button, Checkbox, @@ -23,11 +27,37 @@ export default function SettingsGeneral() { const [isDebugOptionsEnabled, setIsDebugOptionsEnabled] = useRecoilState(showDebugActionsAtom); + const [allowLoggingDbQuery, setAllowLoggingDbQuery] = useRecoilState( + allowLoggingDbQueryAtom + ); + return ( <> General Settings + + Logging + + If you have encountered an issue it may be helpful to enable server + side logging of the database queries used by Graph Explorer. + + + This will not log any data returned by the database queries. + However, the node & edge labels, ID values, and any value filters + will be present in the queries. + + { + setAllowLoggingDbQuery(isSelected); + }} + > + Enable database query logging on proxy server + + + Save Configuration Data @@ -41,6 +71,7 @@ export default function SettingsGeneral() { Save Configuration + Load Configuration Data