[Utilization] Set up Job level profile, and test detail mechanism

torchci/components/SimpleDropList.tsx

@@ -0,0 +1,50 @@
+import {
+  FormControl,
+  MenuItem,
+  Select,
+  SelectChangeEvent,
+} from "@mui/material";
+import { useEffect, useState } from "react";
+
+export default function SimpleDropList({
+  onChange,
+  defaultValue,
+  options,
+}: {
+  onChange: (value: string) => void;
+  defaultValue?: string;
+  options: { value: string; name: string }[];
+}) {
+  const [value, setValue] = useState<string>("");
+
+  useEffect(() => {
+    if (defaultValue) {
+      setValue(defaultValue);
+    } else {
+      setValue("unkown");
+    }
+  }, [defaultValue]);
+
+  const handleChange = (event: SelectChangeEvent) => {
+    setValue(event.target.value);
+    onChange(event.target.value);
+  };
+  return (
+    <div>
+      <FormControl sx={{ m: 1, minWidth: 80 }}>
+        <Select value={value} onChange={handleChange} autoWidth>
+          {defaultValue == undefined ? (
+            <MenuItem value="unkown"></MenuItem>
+          ) : null}
+          {options.map((option, idx) => {
+            return (
+              <MenuItem key={idx} value={option.value}>
+                {option.name}
+              </MenuItem>
+            );
+          })}
+        </Select>
+      </FormControl>
+    </div>
+  );
+}

torchci/components/charts/line_rect_chart/LineRectChart.tsx

@@ -11,13 +11,14 @@ import { processLineData, processRectData, setDimensions } from "./lib/utils";
 import styles from "./LineChart.module.css";
 
 type Props = {
-  onDataChange?: (data: any) => void;
+  onClickedRect?: (id: string) => void;
   inputLines?: TimeSeriesWrapper[];
   rects?: {
     name: string;
     start_at: string;
     end_at: string;
     color?: string;
+    opacity?: number;
   }[];
   chartWidth?: number;
   disableRect?: boolean;
@@ -27,7 +28,7 @@ type Props = {
 };
 
 const LineRectChart = ({
-  onDataChange = (data: any) => void {},
+  onClickedRect = (id: string) => void {},
   inputLines,
   rects,
   chartWidth,
@@ -144,6 +145,7 @@ const LineRectChart = ({
               setLineTooltip={setLineTooltip}
             />
             <RenderSvgRects
+              onClickedRect={onClickedRect}
               setRectTooltip={setRectTooltip}
               rectangles={rectangles}
               disableRect={disableRect}

torchci/components/charts/line_rect_chart/component/RenderLineChartComponents.module.css

@@ -17,13 +17,9 @@
   opacity: 0.7;
 }
 
-.rect {
-  opacity: 0.2;
-}
-
 .rect:hover {
   fill: blue;
-  opacity: 0.5;
+  opacity: 0.7;
 }
 .rect:active {
   fill: red;

torchci/components/charts/line_rect_chart/component/RenderSvgLine.tsx

@@ -17,9 +17,7 @@ const RenderSvgLines = ({
   const lineGenerator = d3
     .line<D3LineRecord>()
     .x((d: D3LineRecord) => scales.xScale(d.date))
-    .y((d: D3LineRecord) => scales.yScale(d.value))
-    .curve(d3.curveBasis);
-
+    .y((d: D3LineRecord) => scales.yScale(d.value));
   return (
     <g className="lines-group">
       {lines.map((line, i) => {

torchci/components/charts/line_rect_chart/component/RenderSvgRect.tsx

@@ -5,6 +5,7 @@ import { formatDate } from "../lib/utils";
 import styles from "./RenderLineChartComponents.module.css";
 
 const RenderSvgRects = ({
+  onClickedRect,
   setRectTooltip,
   rectangles,
   disableRect,
@@ -21,6 +22,7 @@ const RenderSvgRects = ({
       };
     }>
   >;
+  onClickedRect: (id: any) => void;
   rectangles: RectangleData[];
   disableRect?: boolean;
   dimensions: any;
@@ -45,7 +47,9 @@ const RenderSvgRects = ({
     rectData: RectangleData
   ) => {
     if (disableRect) return;
-    //onDataChange(): todo handle the onclick event
+
+    console.log("RenderSvgRects", rectData);
+    onClickedRect(rectData.name);
   };
 
   // handle rect svg events
@@ -68,6 +72,7 @@ const RenderSvgRects = ({
           <rect
             key={i}
             className={`${styles.rect} rect`}
+            opacity={rec.opacity ? rec.opacity : 0.5}
             fill={rec.color ? rec.color : getRandomColor(i)}
             id={rec.name}
             display={disableRect ? "none" : "block"}

torchci/components/charts/line_rect_chart/lib/types.ts

@@ -3,6 +3,7 @@ export interface RectangleData {
   start: Date;
   end: Date;
   color?: string;
+  opacity?: number;
 }
 
 export interface TimeData {

torchci/components/charts/line_rect_chart/lib/utils.ts

@@ -42,6 +42,7 @@ export function processRectData(
     start_at: string;
     end_at: string;
     color?: string;
+    opacity?: number;
   }[]
 ) {
   if (!rectangles || rectangles.length == 0) {
@@ -54,6 +55,7 @@ export function processRectData(
       start: convertDate(el.start_at),
       end: convertDate(el.end_at),
       color: el.color,
+      opacity: el.opacity,
     };
   });
 }

torchci/components/utilization/UtilizationPage.tsx

@@ -1,22 +1,24 @@
 import { PickerConfig } from "components/charts/line_rect_chart/lib/types";
 import LineRectChart from "components/charts/line_rect_chart/LineRectChart";
 import {
+  Segment,
   UtilizationAPIResponse,
   UtilizationMetadata,
 } from "lib/utilization/types";
 import { useEffect, useState } from "react";
-import { getIgnoredSegmentName } from "./helper";
-import styles from "./UtilizationPage.module.css";
+import { TestSectionView } from "./components/TestSectionView/TestSectionView";
+import JobUtilizationSummary from "./components/UtilizationJobSummary/UtilizationJobSummary";
+import { getIgnoredSegmentName, processStatsData } from "./helper";
+import { Divider, MainPage, Section } from "./styles";
+import { StatsInfo } from "./types";
 
 const lineFilters: PickerConfig[] = [
   { category: "all", types: [{ name: "all", tags: ["|"] }] },
-  { category: "all max", types: [{ name: "max", tags: ["max"] }] },
-  { category: "all average", types: [{ name: "avg", tags: ["avg"] }] },
   {
-    category: "gpu max",
+    category: "gpu",
     types: [
-      { name: "gpu util", tags: ["gpu", "max", "|util_percent"] },
-      { name: "gpu mem", tags: ["gpu", "max", "|mem_util_percent"] },
+      { name: "gpu util", tags: ["gpu", "|util_percent"] },
+      { name: "gpu mem", tags: ["gpu", "|mem_util_percent"] },
     ],
   },
   { category: "cpu", types: [{ name: "cpu", tags: ["cpu"] }] },
@@ -34,9 +36,15 @@ export const UtilizationPage = ({
   attempt: string;
   data: UtilizationAPIResponse;
 }) => {
-  const [testSegments, setTestSegments] = useState<any[]>([]);
+  const [testSegments, setTestSegments] = useState<Segment[]>([]);
   const [timeSeriesList, setTimeSeriesList] = useState<any[]>([]);
   const [metadata, setMetadata] = useState<any>();
+  const [summaryData, setSummaryData] = useState<any[]>([]);
+
+  // currently we only show data that is aggregated by max value during the data collection time interval.
+  // this makes sense for utilization to detect potential effieciency issues, later our ui
+  // can support other aggregation methods for analysis, it's very disruptive to add both in UI right now.
+  const aggregateType = "max";
 
   useEffect(() => {
     if (!data) {
@@ -45,6 +53,14 @@ export const UtilizationPage = ({
 
     const util_metadata = data.metadata as UtilizationMetadata;
     const lines = data.ts_list;
+
+    // currently we only show data that is aggregated by max value during the time interval
+    const filteredLines = lines.filter((line) =>
+      line.id.includes(aggregateType)
+    );
+
+    const jobStats: StatsInfo[] = processStatsData(filteredLines);
+
     const segments = util_metadata.segments;
     const filteredSeg = segments.filter((segment) => {
       for (const ignoreName of getIgnoredSegmentName()) {
@@ -55,90 +71,46 @@ export const UtilizationPage = ({
       return true;
     });
     setMetadata(util_metadata);
-    setTimeSeriesList(lines);
+    setTimeSeriesList(filteredLines);
     setTestSegments(filteredSeg);
+    setSummaryData(jobStats);
   }, [data]);
 
   return (
-    <div className={styles.page}>
-      {metadata && (
-        <div className={styles.section}>
-          <TestInformationSection
+    <MainPage>
+      <Section>
+        <div>
+          <JobUtilizationSummary
+            aggregateType={aggregateType}
+            metadata={metadata}
+            tableData={summaryData}
             workflowId={workflowId}
             jobId={jobId}
             attempt={attempt}
-            jobName={metadata.job_name}
-            workflowName={metadata.workflow_name}
           />
         </div>
-      )}
+      </Section>
       {timeSeriesList.length > 0 && (
-        <div className={styles.section}>
+        <Section>
           <h3>Utilization Time Series</h3>
-          <div className={styles.divider}></div>
+          <Divider />
           <LineRectChart
             inputLines={timeSeriesList}
             chartWidth={1200}
             disableLineTooltip={false}
             disableRect={true}
             lineFilterConfig={lineFilters}
           ></LineRectChart>
-        </div>
+        </Section>
       )}
-      {testSegments.length > 0 && (
-        <div className={styles.section}>
-          <h3>Detected Python test details</h3>
-          <div className={styles.divider}></div>
-          <LineRectChart
-            inputLines={timeSeriesList}
-            chartWidth={1200}
-            rects={testSegments}
-            disableLineTooltip={true}
-            disableRect={false}
-          ></LineRectChart>
-          <div>
-            <h4>Tests </h4>
-            {testSegments.map((segment) => {
-              return (
-                <div key={segment.name}>
-                  <div>{segment.name}</div>
-                </div>
-              );
-            })}
-          </div>
-        </div>
+      {testSegments.length > 0 && timeSeriesList.length > 0 && (
+        <Section>
+          <TestSectionView
+            testSegments={testSegments}
+            timeSeriesList={timeSeriesList}
+          />
+        </Section>
       )}
-    </div>
-  );
-};
-
-const TestInformationSection = ({
-  workflowId,
-  jobId,
-  attempt,
-  jobName,
-  workflowName,
-}: {
-  workflowId: string;
-  jobId: string;
-  attempt: string;
-  jobName: string;
-  workflowName: string;
-}) => {
-  return (
-    <div className={styles.section}>
-      <h1> Test Job Infomation</h1>
-      <div className={styles.divider}></div>
-      <div>
-        <div>
-          <span>Workflow(run)Id:</span>
-          {workflowId}
-        </div>
-        <div>Job Id: {jobId} </div>
-        <div>Attempt: {attempt}</div>
-        <div>Job Name: {jobName}</div>
-        <div>Workflow Name: {workflowName}</div>
-      </div>
-    </div>
+    </MainPage>
   );
 };