diff --git a/server/app/header.tsx b/server/app/header.tsx index 33dc5b4..7be10a6 100644 --- a/server/app/header.tsx +++ b/server/app/header.tsx @@ -7,7 +7,7 @@ import { Bars3Icon, BellIcon, XMarkIcon } from "@heroicons/react/24/outline"; import clsx from "clsx"; import beerTap from "@/public/beer-tap.png"; -const navigation = [{ name: "Dashboard", href: "/query", current: true }]; +const navigation = [{ name: "Queries", href: "/query", current: true }]; const userNavigation = [ { name: "Your Profile", href: "#" }, { name: "Settings", href: "#" }, diff --git a/server/app/page.tsx b/server/app/page.tsx index a006c5e..b6d013b 100644 --- a/server/app/page.tsx +++ b/server/app/page.tsx @@ -58,7 +58,7 @@ export default async function Example() {
{isLoggedIn ? ( - Dashboard + Queries ) : ( Log in )} diff --git a/server/app/query/page.tsx b/server/app/query/page.tsx index 3da7f61..40a9d8c 100644 --- a/server/app/query/page.tsx +++ b/server/app/query/page.tsx @@ -1,4 +1,104 @@ +"use client"; + +import { useEffect, useState } from "react"; +import Link from "next/link"; + +import { StatusPill, RunTimePill } from "@/app/query/view/[id]/components"; +import { + StatusEvent, + RemoteServer, + RemoteServerNames, + IPARemoteServers, //hack until the queryId is stored in a DB + StatusEventByRemoteServer, + initialStatusEventByRemoteServer, +} from "@/app/query/servers"; +import { getQueryByUUID, Query } from "@/data/query"; + +type QueryData = { + statusEvent: StatusEventByRemoteServer; + query: Query; +}; +type DataByQuery = { + [queryID: string]: QueryData; +}; + export default function Page() { + const [queryIDs, setQueryIDs] = useState([]); + const [dataByQuery, setDataByQuery] = useState({}); + + const updateData = ( + query: Query, + remoteServer: RemoteServer, + statusEvent: StatusEvent, + ) => { + setDataByQuery((prev) => { + let _prev = prev; + if (!Object.hasOwn(prev, query.uuid)) { + // if queryID not in dataByQuery yet, + // add initial status before updating value. + // otherwise prev[query.uuid][statusEvent][remteServer.ServerName] + // doesn't exist, and cannot be updated. we need to fill in the + // nested structure, which `initialStatusEventByRemoteServer` does. + _prev = { + ..._prev, + [query.uuid]: { + statusEvent: initialStatusEventByRemoteServer, + query: query, + }, + }; + } + + return { + ..._prev, + [query.uuid]: { + ..._prev[query.uuid], + statusEvent: { + ..._prev[query.uuid].statusEvent, + [remoteServer.remoteServerName]: statusEvent, + }, + }, + }; + }); + }; + + useEffect(() => { + // poll runningQueries every second + (async () => { + const interval = setInterval(async () => { + const _queryIDs: string[] = + await IPARemoteServers[RemoteServerNames.Helper1].runningQueries(); + + setQueryIDs(_queryIDs); + }, 1000); // 1000 milliseconds = 1 second + return () => clearInterval(interval); + })(); + }, []); + + useEffect(() => { + (async () => { + setDataByQuery((prev) => { + return Object.fromEntries( + Object.keys(prev) + .filter((queryID) => queryIDs.includes(queryID)) + .map((queryID) => [queryID, prev[queryID]]), + ); + }); + + const promises = queryIDs.map(async (queryID) => { + const query: Query = await getQueryByUUID(queryID); + const remoteServerPromises = Object.values(IPARemoteServers).map( + async (remoteServer) => { + const statusEvent: StatusEvent = + await remoteServer.queryStatus(queryID); + updateData(query, remoteServer, statusEvent); + }, + ); + await Promise.all(remoteServerPromises); + }); + await Promise.all(promises); + })(); + }, [queryIDs]); + return ( <>
@@ -6,6 +106,94 @@ export default function Page() {

Current Queries

+ + {Object.keys(dataByQuery).length == 0 && ( +

+ None currently running. +

+ )} + + {Object.entries(dataByQuery).map(([queryID, queryData]) => { + const statusEventByRemoteServer = queryData.statusEvent; + const query = queryData.query; + + return ( +
+ +
+

+ Query: {query.displayId} +

+
+
+ {Object.values(IPARemoteServers).map( + (remoteServer: RemoteServer) => { + const statusEvent: StatusEvent | null = + statusEventByRemoteServer[ + remoteServer.remoteServerName + ]; + if (statusEvent === null) { + return ( +
+ ); + } + + return ( +
+
+ {remoteServer.toString()} Run Time +
+
+ +
+
+ ); + }, + )} +
+
+
+
+ {Object.values(IPARemoteServers).map( + (remoteServer: RemoteServer) => { + const statusEvent: StatusEvent | null = + statusEventByRemoteServer[ + remoteServer.remoteServerName + ]; + if (statusEvent === null) { + return ( +
+ ); + } + + return ( +
+
+ {remoteServer.remoteServerNameStr} Status +
+
+ +
+
+ ); + }, + )} +
+
+
+ +
+ ); + })}
diff --git a/server/app/query/servers.tsx b/server/app/query/servers.tsx index 610c3f7..85b3fbd 100644 --- a/server/app/query/servers.tsx +++ b/server/app/query/servers.tsx @@ -35,10 +35,18 @@ function getStatusFromString(statusString: string): Status { } } -export interface StatsDataPoint { - timestamp: string; - memoryRSSUsage: number; - cpuUsage: number; +export interface StatusEvent { + status: Status; + startTime: number; + endTime: number | null; +} + +function buildStatusEventFromJSON(statusJSON: any): StatusEvent { + return { + status: getStatusFromString(statusJSON.status), + startTime: statusJSON.start_time, + endTime: statusJSON.end_time ?? null, + }; } export type StatusByRemoteServer = { @@ -50,15 +58,6 @@ export const initialStatusByRemoteServer: StatusByRemoteServer = Object.values(RemoteServerNames).map((serverName) => [[serverName], null]), ); -export type StatsByRemoteServer = { - [key in RemoteServerNames]: StatsDataPoint[]; -}; - -export const initialStatsByRemoteServer: StatsByRemoteServer = - Object.fromEntries( - Object.values(RemoteServerNames).map((serverName) => [[serverName], []]), - ); - export type StartTimeByRemoteServer = { [key in RemoteServerNames]: number | null; }; @@ -67,6 +66,10 @@ export type EndTimeByRemoteServer = { [key in RemoteServerNames]: number | null; }; +export type StatusEventByRemoteServer = { + [key in RemoteServerNames]: StatusEvent | null; +}; + export const initialStartTimeByRemoteServer: StartTimeByRemoteServer = Object.fromEntries( Object.values(RemoteServerNames).map((serverName) => [[serverName], null]), @@ -77,6 +80,26 @@ export const initialEndTimeByRemoteServer: StartTimeByRemoteServer = Object.values(RemoteServerNames).map((serverName) => [[serverName], null]), ); +export const initialStatusEventByRemoteServer: StatusEventByRemoteServer = + Object.fromEntries( + Object.values(RemoteServerNames).map((serverName) => [[serverName], null]), + ); + +export interface StatsDataPoint { + timestamp: string; + memoryRSSUsage: number; + cpuUsage: number; +} + +export type StatsByRemoteServer = { + [key in RemoteServerNames]: StatsDataPoint[]; +}; + +export const initialStatsByRemoteServer: StatsByRemoteServer = + Object.fromEntries( + Object.values(RemoteServerNames).map((serverName) => [[serverName], []]), + ); + export class RemoteServer { protected baseURL: URL; remoteServerName: RemoteServerNames; @@ -96,6 +119,26 @@ export class RemoteServer { throw new Error("Not Implemented"); } + queryStatusURL(id: string): URL { + return new URL(`/start/${id}/status`, this.baseURL); + } + + async queryStatus(id: string): Promise { + const status_response = await fetch(this.queryStatusURL(id)); + const statusJSON = await status_response.json(); + return buildStatusEventFromJSON(statusJSON); + } + + runningQueriesURL(): URL { + return new URL(`/start/running-queries`, this.baseURL); + } + + async runningQueries(): Promise { + const queries_response = await fetch(this.runningQueriesURL()); + const queriesJSON = await queries_response.json(); + return queriesJSON.running_queries; + } + logURL(id: string): URL { return new URL(`/start/${id}/log-file`, this.baseURL); } @@ -187,44 +230,13 @@ export class RemoteServer { openStatusSocket( id: string, - setStatus: React.Dispatch>, - setStartTime: React.Dispatch>, - setEndTime: React.Dispatch>, + setStatusEvent: (statusEvent: StatusEvent) => void, ): WebSocket { const ws = this.statusSocket(id); - const updateStatus = (status: Status) => { - setStatus((prevStatus) => ({ - ...prevStatus, - [this.remoteServerName]: status, - })); - }; - - const updateStartTime = (runTime: number) => { - setStartTime((prevStartTime) => { - return { - ...prevStartTime, - [this.remoteServerName]: runTime, - }; - }); - }; - - const updateEndTime = (runTime: number) => { - setEndTime((prevEndTime) => { - return { - ...prevEndTime, - [this.remoteServerName]: runTime, - }; - }); - }; - ws.onmessage = (event) => { - const eventData = JSON.parse(event.data); - updateStartTime(eventData.start_time); - updateEndTime(eventData.end_time ?? null); - const statusString: string = eventData.status; - const status = getStatusFromString(statusString); - updateStatus(status); + const statusEvent = buildStatusEventFromJSON(JSON.parse(event.data)); + setStatusEvent(statusEvent); }; ws.onclose = (event) => { diff --git a/server/app/query/view/[id]/components.tsx b/server/app/query/view/[id]/components.tsx index 5aaf35a..6e43b9b 100644 --- a/server/app/query/view/[id]/components.tsx +++ b/server/app/query/view/[id]/components.tsx @@ -2,7 +2,7 @@ import { useEffect, useState, useRef } from "react"; import { Source_Code_Pro } from "next/font/google"; import clsx from "clsx"; import { ChevronDownIcon, ChevronRightIcon } from "@heroicons/react/24/solid"; -import { Status, ServerLog } from "@/app/query/servers"; +import { Status, StatusEvent, ServerLog } from "@/app/query/servers"; const sourceCodePro = Source_Code_Pro({ subsets: ["latin"] }); @@ -64,15 +64,7 @@ function secondsToTime(e: number) { return h + ":" + m + ":" + s; } -export function RunTimePill({ - status, - startTime, - endTime, -}: { - status: Status; - startTime: number | null; - endTime: number | null; -}) { +export function RunTimePill({ statusEvent }: { statusEvent: StatusEvent }) { const [runTime, setRunTime] = useState(null); const runTimeStr = runTime ? secondsToTime(runTime) : "N/A"; const intervalId = useRef | null>(null); @@ -83,22 +75,28 @@ export function RunTimePill({ // which runs the timer. if a new one is needed, it's created. clearTimeout(intervalId.current); } - if (startTime === null) { + if (statusEvent?.startTime === null) { setRunTime(null); } else { - if (endTime !== null) { - setRunTime(endTime - startTime); + if (statusEvent?.endTime !== null) { + setRunTime(statusEvent.endTime - statusEvent.startTime); } else { + setRunTime(Date.now() / 1000 - statusEvent.startTime); let newIntervalId = setInterval(() => { - setRunTime(Date.now() / 1000 - startTime); + setRunTime(Date.now() / 1000 - statusEvent.startTime); }, 1000); intervalId.current = newIntervalId; } } - }, [startTime, endTime]); + }, [statusEvent]); return ( -
+
{runTimeStr}
); diff --git a/server/app/query/view/[id]/page.tsx b/server/app/query/view/[id]/page.tsx index 083d5f1..024d8a3 100644 --- a/server/app/query/view/[id]/page.tsx +++ b/server/app/query/view/[id]/page.tsx @@ -8,20 +8,16 @@ import { LogViewer, } from "@/app/query/view/[id]/components"; import { - Status, + StatusEvent, ServerLog, RemoteServer, RemoteServerNames, RemoteServersType, IPARemoteServers, //hack until the queryId is stored in a DB - StatusByRemoteServer, StatsByRemoteServer, - StartTimeByRemoteServer, - EndTimeByRemoteServer, - initialStatusByRemoteServer, + StatusEventByRemoteServer, initialStatsByRemoteServer, - initialStartTimeByRemoteServer, - initialEndTimeByRemoteServer, + initialStatusEventByRemoteServer, } from "@/app/query/servers"; import { StatsComponent } from "@/app/query/view/[id]/charts"; import { JSONSafeParse } from "@/app/utils"; @@ -46,14 +42,20 @@ export default function QueryPage({ params }: { params: { id: string } }) { selectedRemoteServerLogs.includes(item.remoteServer.remoteServerNameStr), ); - const [statusByRemoteServer, setStatusByRemoteServer] = - useState(initialStatusByRemoteServer); const [statsByRemoteServer, setStatsByRemoteServer] = useState(initialStatsByRemoteServer); - const [startTimeByRemoteServer, setStartTimeByRemoteServer] = - useState(initialStartTimeByRemoteServer); - const [endTimeByRemoteServer, setEndTimeByRemoteServer] = - useState(initialEndTimeByRemoteServer); + const [statusEventByRemoteServer, setStatusEventByRemoteServer] = + useState(initialStatusEventByRemoteServer); + + const updateStatusEvent = ( + remoteServer: RemoteServer, + statusEvent: StatusEvent, + ) => { + setStatusEventByRemoteServer((prevStatus) => ({ + ...prevStatus, + [remoteServer.remoteServerName]: statusEvent, + })); + }; function flipLogsHidden() { setLogsHidden(!logsHidden); @@ -111,9 +113,7 @@ export default function QueryPage({ params }: { params: { id: string } }) { const loggingWs = remoteServer.openLogSocket(query.uuid, setLogs); const statusWs = remoteServer.openStatusSocket( query.uuid, - setStatusByRemoteServer, - setStartTimeByRemoteServer, - setEndTimeByRemoteServer, + (statusEvent) => updateStatusEvent(remoteServer, statusEvent), ); const statsWs = remoteServer.openStatsSocket( query.uuid, @@ -217,14 +217,11 @@ export default function QueryPage({ params }: { params: { id: string } }) {
{Object.values(IPARemoteServers).map( (remoteServer: RemoteServer) => { - const startTime = - startTimeByRemoteServer[remoteServer.remoteServerName]; - const endTime = - endTimeByRemoteServer[remoteServer.remoteServerName]; - - const status = - statusByRemoteServer[remoteServer.remoteServerName] ?? - Status.UNKNOWN; + const statusEvent: StatusEvent | null = + statusEventByRemoteServer[remoteServer.remoteServerName]; + if (statusEvent === null) { + return
; + } return (
- +
); @@ -273,9 +266,11 @@ export default function QueryPage({ params }: { params: { id: string } }) {
{Object.values(IPARemoteServers).map( (remoteServer: RemoteServer) => { - const status = - statusByRemoteServer[remoteServer.remoteServerName] ?? - Status.UNKNOWN; + const statusEvent: StatusEvent | null = + statusEventByRemoteServer[remoteServer.remoteServerName]; + if (statusEvent === null) { + return
; + } return (
- +
); diff --git a/server/data/query.ts b/server/data/query.ts index f199543..289abf9 100644 --- a/server/data/query.ts +++ b/server/data/query.ts @@ -45,6 +45,9 @@ export async function getQuery(displayId: string): Promise { if (error) { console.error(error); + throw new Error( + `Error fetching query with displayId= ${displayId}: ${error.message}`, + ); } else if (status === 200) { if (data) { return processQueryData(data); @@ -55,6 +58,31 @@ export async function getQuery(displayId: string): Promise { throw new Error(`${displayId} not found.`); } +export async function getQueryByUUID(uuid: string): Promise { + const supabase = await buildSupabaseServerClient(); + + const { status, data, error } = await supabase + .from("queries") + .select("*") + .eq("uuid", uuid) + .limit(1) + .maybeSingle(); + + if (error) { + console.error(error); + throw new Error( + `Error fetching query with UUID= ${uuid}: ${error.message}`, + ); + } else if (status === 200) { + if (data) { + return processQueryData(data); + } else { + notFound(); + } + } + throw new Error(`${uuid} not found.`); +} + export async function createNewQuery( params: FormData, queryType: QueryType, diff --git a/sidecar/app/query/ipa.py b/sidecar/app/query/ipa.py index eb980a5..044c410 100644 --- a/sidecar/app/query/ipa.py +++ b/sidecar/app/query/ipa.py @@ -251,7 +251,7 @@ def run(self): for sidecar_url in sidecar_urls: url = urlunparse( sidecar_url._replace( - scheme="https", path=f"/start/ipa-helper/{self.query_id}/status" + scheme="https", path=f"/start/{self.query_id}/status" ), ) while True: diff --git a/sidecar/app/routes/start.py b/sidecar/app/routes/start.py index 25e8784..e65f093 100644 --- a/sidecar/app/routes/start.py +++ b/sidecar/app/routes/start.py @@ -25,7 +25,7 @@ class IncorrectRoleError(Exception): pass -@router.get("/capacity_available") +@router.get("/capacity-available") def capacity_available( request: Request, ): @@ -33,7 +33,7 @@ def capacity_available( return {"capacity_available": query_manager.capacity_available} -@router.get("/running_queries") +@router.get("/running-queries") def running_queries( request: Request, ): @@ -109,13 +109,13 @@ def start_ipa_helper( return {"message": "Process started successfully", "query_id": query_id} -@router.get("/ipa-helper/{query_id}/status") -def get_ipa_helper_status( +@router.get("/{query_id}/status") +def get_query_status( query_id: str, request: Request, ): query = get_query_from_query_id(request.app.state.QUERY_MANAGER, Query, query_id) - return {"status": query.status.name} + return query.status_event_json @router.get("/{query_id}/log-file") diff --git a/sidecar/app/routes/websockets.py b/sidecar/app/routes/websockets.py index d846841..64cf149 100644 --- a/sidecar/app/routes/websockets.py +++ b/sidecar/app/routes/websockets.py @@ -39,11 +39,9 @@ async def status_websocket( async with use_websocket(websocket) as websocket: while query.running: - query.logger.debug(f"{query_id=} Status: {query.status.name}") await websocket.send_json(query.status_event_json) await asyncio.sleep(1) - query.logger.debug(f"{query_id=} Status: {query.status.name}") await websocket.send_json(query.status_event_json) diff --git a/sidecar/tests/app/routes/test_start.py b/sidecar/tests/app/routes/test_start.py index 8f00236..e7e0478 100644 --- a/sidecar/tests/app/routes/test_start.py +++ b/sidecar/tests/app/routes/test_start.py @@ -34,20 +34,20 @@ def _running_query(): def test_capacity_available(): - response = client.get("/start/capacity_available") + response = client.get("/start/capacity-available") assert response.status_code == 200 assert response.json() == {"capacity_available": True} def test_not_capacity_available(running_query): assert running_query.query_id in app.state.QUERY_MANAGER.running_queries - response = client.get("/start/capacity_available") + response = client.get("/start/capacity-available") assert response.status_code == 200 assert response.json() == {"capacity_available": False} def test_running_queries(running_query): - response = client.get("/start/running_queries") + response = client.get("/start/running-queries") assert response.status_code == 200 assert response.json() == {"running_queries": [running_query.query_id]} @@ -130,16 +130,29 @@ def test_start_ipa_query_as_helper(mock_role): ) -def test_get_ipa_helper_status_not_found(): +def test_get_status_not_found(): query_id = str(uuid4()) - response = client.get(f"/start/ipa-helper/{query_id}/status") + response = client.get(f"/start/{query_id}/status") assert response.status_code == 404 -def test_get_ipa_helper_status(running_query): - response = client.get(f"/start/ipa-helper/{running_query.query_id}/status") +def test_get_status_running(running_query): + response = client.get(f"/start/{running_query.query_id}/status") assert response.status_code == 200 - assert response.json() == {"status": str(Status.STARTING.name)} + status_event_json = response.json() + assert status_event_json["status"] == str(Status.STARTING.name) + assert "start_time" in status_event_json + assert "end_time" not in status_event_json + + +def test_get_status_complete(running_query): + running_query.status = Status.COMPLETE + response = client.get(f"/start/{running_query.query_id}/status") + assert response.status_code == 200 + status_event_json = response.json() + assert status_event_json["status"] == str(Status.COMPLETE.name) + assert "start_time" in status_event_json + assert "end_time" in status_event_json def test_get_ipa_helper_log_file_not_found():