From 1bc53d738795aa653ecaa92ed781debb214ee2ef Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 31 Jan 2025 12:45:01 -0800 Subject: [PATCH] Add platform (arch) for ExecuTorch dashboard (#6234) Address part of https://github.com/pytorch/executorch/issues/7986 The historical context here is that the older schema for benchmark database didn't have a separate columns for `device`, i.e. `cpu`, and `arch`, i.e. `x86_64`. So, they are bundle together into a single one called `deviceArch`. This PR splits this field into `device` and `arch` field behind the scene, but keeps the display name in the same format, i.e. ` Apple iPhone 15 (iOS 18.0)` to maintain the same UX, which allows me to add a new platform (arch) dropdown list for ExecuTorch. The list has only 3 options `Android` and `iOS` and `All Platforms`, ~~and only one can be selected at a time (this is the part that fixes https://github.com/pytorch/executorch/issues/7986)~~ This also applies lint to these queries and fixes a bug in which sqlfluff doesn't recognize the configuration file at `.sqlfluff` I found that https://mui.com/x/react-data-grid/pagination/ only allows up to 100 rows before enforcing pagination. Having more requires the paid pro version. ### Testing https://torchci-git-fork-huydhn-tweak-et-dashboard-fbopensource.vercel.app/benchmark/llms?repoName=pytorch%2Fexecutorch shows a new platform dropdown list --- .lintrunner.toml | 7 ++++- tools/linter/adapters/sqlfluff_linter.py | 2 ++ .../oss_ci_benchmark_branches/params.json | 5 +-- .../oss_ci_benchmark_branches/query.sql | 18 +++++------ .../oss_ci_benchmark_llms/params.json | 5 +-- .../oss_ci_benchmark_llms/query.sql | 31 +++++++++---------- .../oss_ci_benchmark_names/params.json | 5 +-- .../oss_ci_benchmark_names/query.sql | 18 +++++------ .../benchmark/llms/SummaryPanel.tsx | 16 ++++++++-- torchci/components/benchmark/llms/common.tsx | 6 ++++ torchci/pages/benchmark/llms.tsx | 28 +++++++++++++++-- 11 files changed, 93 insertions(+), 48 deletions(-) diff --git a/.lintrunner.toml b/.lintrunner.toml index c8572a09ec..524df77b19 100644 --- a/.lintrunner.toml +++ b/.lintrunner.toml @@ -360,7 +360,12 @@ is_formatter = true [[linter]] code = 'SQLFLUFF' # include_patterns = ['torchci/clickhouse_queries/**/*.sql'] -include_patterns = ['torchci/clickhouse_queries/workflow_load/query.sql'] +include_patterns = [ + 'torchci/clickhouse_queries/workflow_load/query.sql', + 'torchci/clickhouse_queries/oss_ci_benchmark_branches/query.sql', + 'torchci/clickhouse_queries/oss_ci_benchmark_llms/query.sql', + 'torchci/clickhouse_queries/oss_ci_benchmark_names/query.sql', +] exclude_patterns = [ ] command = [ diff --git a/tools/linter/adapters/sqlfluff_linter.py b/tools/linter/adapters/sqlfluff_linter.py index e0e3526791..ca11f131bc 100644 --- a/tools/linter/adapters/sqlfluff_linter.py +++ b/tools/linter/adapters/sqlfluff_linter.py @@ -78,6 +78,8 @@ def check_file( [ "sqlfluff", "format", + "--config", + os.path.join(os.getcwd(), ".sqlfluff"), "--dialect", "clickhouse", tmp.name, diff --git a/torchci/clickhouse_queries/oss_ci_benchmark_branches/params.json b/torchci/clickhouse_queries/oss_ci_benchmark_branches/params.json index 8042a80354..4e4656b90c 100644 --- a/torchci/clickhouse_queries/oss_ci_benchmark_branches/params.json +++ b/torchci/clickhouse_queries/oss_ci_benchmark_branches/params.json @@ -1,6 +1,7 @@ { "params": { - "deviceArch": "String", + "arch": "String", + "device": "String", "dtypes": "Array(String)", "excludedMetrics": "Array(String)", "benchmarks": "Array(String)", @@ -11,4 +12,4 @@ "stopTime": "DateTime64(3)" }, "tests": [] -} \ No newline at end of file +} diff --git a/torchci/clickhouse_queries/oss_ci_benchmark_branches/query.sql b/torchci/clickhouse_queries/oss_ci_benchmark_branches/query.sql index b073945ff6..f04370970a 100644 --- a/torchci/clickhouse_queries/oss_ci_benchmark_branches/query.sql +++ b/torchci/clickhouse_queries/oss_ci_benchmark_branches/query.sql @@ -1,7 +1,7 @@ -- This query is used to get the list of branches and commits used by different -- OSS CI benchmark experiments. This powers HUD benchmarks dashboards -SELECT - DISTINCT replaceOne(head_branch, 'refs/heads/', '') AS head_branch, +SELECT DISTINCT + replaceOne(head_branch, 'refs/heads/', '') AS head_branch, head_sha, workflow_id AS id, toStartOfDay(fromUnixTimestamp(timestamp)) AS event_time @@ -32,17 +32,15 @@ WHERE OR empty({excludedMetrics: Array(String) }) ) AND notEmpty(metric_name) - -- NB: DEVICE (ARCH) is the display format used by HUD when grouping together these two fields AND ( - CONCAT( - device, - ' (', - IF(empty(arch), 'NVIDIA A100-SXM4-40GB', arch), - ')' - ) = {deviceArch: String } - OR {deviceArch: String } = '' + startsWith({device: String }, device) + OR {device: String } = '' ) AND notEmpty(device) + AND ( + arch LIKE concat('%', {arch: String }, '%') + OR {arch: String } = '' + ) ORDER BY head_branch, timestamp DESC diff --git a/torchci/clickhouse_queries/oss_ci_benchmark_llms/params.json b/torchci/clickhouse_queries/oss_ci_benchmark_llms/params.json index d370a9875b..5219bdb63f 100644 --- a/torchci/clickhouse_queries/oss_ci_benchmark_llms/params.json +++ b/torchci/clickhouse_queries/oss_ci_benchmark_llms/params.json @@ -1,8 +1,9 @@ { "params": { + "arch": "String", "branches": "Array(String)", "commits": "Array(String)", - "deviceArch": "String", + "device": "String", "dtypes": "Array(String)", "excludedMetrics": "Array(String)", "benchmarks": "Array(String)", @@ -14,4 +15,4 @@ "stopTime": "DateTime64(3)" }, "tests": [] -} \ No newline at end of file +} diff --git a/torchci/clickhouse_queries/oss_ci_benchmark_llms/query.sql b/torchci/clickhouse_queries/oss_ci_benchmark_llms/query.sql index d1bfdec706..b1015e60df 100644 --- a/torchci/clickhouse_queries/oss_ci_benchmark_llms/query.sql +++ b/torchci/clickhouse_queries/oss_ci_benchmark_llms/query.sql @@ -15,18 +15,18 @@ WITH benchmarks AS ( o.benchmark.dtype AS dtype, IF( empty(o.runners), - tupleElement(o.benchmark, 'extra_info') [ 'device' ], - tupleElement(o.runners [ 1 ], 'name') + tupleElement(o.benchmark, 'extra_info')['device'], + tupleElement(o.runners[1], 'name') ) AS device, IF( empty(o.runners), - tupleElement(o.benchmark, 'extra_info') [ 'arch' ], - tupleElement(o.runners [ 1 ], 'type') + tupleElement(o.benchmark, 'extra_info')['arch'], + tupleElement(o.runners[1], 'type') ) AS arch, IF( - tupleElement(o.benchmark, 'extra_info') [ 'compile' ] = '', + tupleElement(o.benchmark, 'extra_info')['compile'] = '', 'true', -- Default to true - tupleElement(o.benchmark, 'extra_info') [ 'compile' ] + tupleElement(o.benchmark, 'extra_info')['compile'] ) AS use_torch_compile, DATE_TRUNC( {granularity: String }, @@ -64,8 +64,9 @@ WITH benchmarks AS ( ) AND notEmpty(o.metric.name) ) -SELECT - DISTINCT workflow_id, + +SELECT DISTINCT + workflow_id, job_id, model, backend, @@ -85,17 +86,15 @@ WHERE has({branches: Array(String) }, head_branch) OR empty({branches: Array(String) }) ) - -- NB: DEVICE (ARCH) is the display format used by HUD when grouping together these two fields AND ( - CONCAT( - device, - ' (', - IF(empty(arch), 'NVIDIA A100-SXM4-40GB', arch), - ')' - ) = {deviceArch: String } - OR {deviceArch: String } = '' + startsWith({device: String }, device) + OR {device: String } = '' ) AND notEmpty(device) + AND ( + arch LIKE concat('%', {arch: String }, '%') + OR {arch: String } = '' + ) ORDER BY granularity_bucket DESC, workflow_id DESC, diff --git a/torchci/clickhouse_queries/oss_ci_benchmark_names/params.json b/torchci/clickhouse_queries/oss_ci_benchmark_names/params.json index 8042a80354..4e4656b90c 100644 --- a/torchci/clickhouse_queries/oss_ci_benchmark_names/params.json +++ b/torchci/clickhouse_queries/oss_ci_benchmark_names/params.json @@ -1,6 +1,7 @@ { "params": { - "deviceArch": "String", + "arch": "String", + "device": "String", "dtypes": "Array(String)", "excludedMetrics": "Array(String)", "benchmarks": "Array(String)", @@ -11,4 +12,4 @@ "stopTime": "DateTime64(3)" }, "tests": [] -} \ No newline at end of file +} diff --git a/torchci/clickhouse_queries/oss_ci_benchmark_names/query.sql b/torchci/clickhouse_queries/oss_ci_benchmark_names/query.sql index 74207c1763..c9c136630e 100644 --- a/torchci/clickhouse_queries/oss_ci_benchmark_names/query.sql +++ b/torchci/clickhouse_queries/oss_ci_benchmark_names/query.sql @@ -1,6 +1,6 @@ --- This query is used by HUD benchmarks dashboards to get the list of experiment names -SELECT - DISTINCT benchmark_name AS benchmark, +SELECT DISTINCT + benchmark_name AS benchmark, model_name AS model, model_backend AS backend, metric_name AS metric, @@ -34,17 +34,15 @@ WHERE OR empty({excludedMetrics: Array(String) }) ) AND notEmpty(metric_name) - -- NB: DEVICE (ARCH) is the display format used by HUD when grouping together these two fields AND ( - CONCAT( - device, - ' (', - IF(empty(arch), 'NVIDIA A100-SXM4-40GB', arch), - ')' - ) = {deviceArch: String } - OR {deviceArch: String } = '' + startsWith({device: String }, device) + OR {device: String } = '' ) AND notEmpty(device) + AND ( + arch LIKE concat('%', {arch: String }, '%') + OR {arch: String } = '' + ) ORDER BY benchmark, backend, diff --git a/torchci/components/benchmark/llms/SummaryPanel.tsx b/torchci/components/benchmark/llms/SummaryPanel.tsx index 82da1e2c60..127725708f 100644 --- a/torchci/components/benchmark/llms/SummaryPanel.tsx +++ b/torchci/components/benchmark/llms/SummaryPanel.tsx @@ -23,6 +23,7 @@ export function SummaryPanel({ modelName, backendName, metricNames, + archName, lPerfData, rPerfData, }: { @@ -33,6 +34,7 @@ export function SummaryPanel({ modelName: string; backendName: string; metricNames: string[]; + archName: string; lPerfData: BranchAndCommitPerfData; rPerfData: BranchAndCommitPerfData; }) { @@ -75,13 +77,15 @@ export function SummaryPanel({ params.value.backend !== undefined ? `&backendName=${encodeURIComponent(params.value.backend)}` : ""; - const deviceArch = `${params.value.device} (${params.value.arch})`; + const deviceName = `${params.value.device} (${params.value.arch})`; const url = `/benchmark/llms?startTime=${startTime}&stopTime=${stopTime}&granularity=${granularity}&repoName=${encodeURIComponent( repoName )}&modelName=${encodeURIComponent( model - )}${backend}${dtype}&deviceName=${encodeURIComponent(deviceArch)}`; + )}${backend}${dtype}&deviceName=${encodeURIComponent( + deviceName + )}&archName=${encodeURIComponent(archName)}`; return ( @@ -215,17 +219,23 @@ export function SummaryPanel({ ] ); + // TODO (huydhn): Table bigger than 100 rows requires x-data-grid-pro return ( 99 + ? 99 * ROW_HEIGHT + : data.length * ROW_HEIGHT + ROW_GAP + } > el.name }} + showFooter={true} /> diff --git a/torchci/components/benchmark/llms/common.tsx b/torchci/components/benchmark/llms/common.tsx index 38bc9dadea..32f86d9ed1 100644 --- a/torchci/components/benchmark/llms/common.tsx +++ b/torchci/components/benchmark/llms/common.tsx @@ -35,9 +35,15 @@ export const METRIC_DISPLAY_SHORT_HEADERS: { [k: string]: string } = { "compilation_time(s)": "CompTime", }; export const DEFAULT_DEVICE_NAME = "All Devices"; +export const DEFAULT_ARCH_NAME = "All Platforms"; export const DEFAULT_DTYPE_NAME = "All DType"; export const DEFAULT_BACKEND_NAME = "All Backends"; +// Only used by ExecuTorch for now +export const ARCH_NAMES: { [k: string]: string[] } = { + "pytorch/executorch": ["Android", "iOS"], +}; + // Relative thresholds export const RELATIVE_THRESHOLD = 0.05; diff --git a/torchci/pages/benchmark/llms.tsx b/torchci/pages/benchmark/llms.tsx index 8568df6728..b69671facd 100644 --- a/torchci/pages/benchmark/llms.tsx +++ b/torchci/pages/benchmark/llms.tsx @@ -7,6 +7,8 @@ import { MAIN_BRANCH, } from "components/benchmark/common"; import { + ARCH_NAMES, + DEFAULT_ARCH_NAME, DEFAULT_BACKEND_NAME, DEFAULT_DEVICE_NAME, DEFAULT_DTYPE_NAME, @@ -45,6 +47,7 @@ function Report({ backendName, dtypeName, deviceName, + archName, metricNames, lBranchAndCommit, rBranchAndCommit, @@ -58,6 +61,7 @@ function Report({ backendName: string; dtypeName: string; deviceName: string; + archName: string; metricNames: string[]; lBranchAndCommit: BranchAndCommit; rBranchAndCommit: BranchAndCommit; @@ -146,6 +150,7 @@ function Report({ modelName={modelName} backendName={backendName} metricNames={metricNames} + archName={archName} lPerfData={{ ...lBranchAndCommit, data: lDataWithSpeedup, @@ -178,6 +183,7 @@ export default function Page() { const [backendName, setBackendName] = useState(DEFAULT_BACKEND_NAME); const [dtypeName, setDTypeName] = useState(DEFAULT_DTYPE_NAME); const [deviceName, setDeviceName] = useState(DEFAULT_DEVICE_NAME); + const [archName, setArchName] = useState(DEFAULT_ARCH_NAME); // Set the dropdown value what is in the param useEffect(() => { @@ -231,6 +237,12 @@ export default function Page() { setDeviceName(deviceName); } + // Set the default arch to Android for ExecuTorch as it has only 2 options Android and iOS + const archName: string = (router.query.archName as string) ?? undefined; + if (archName !== undefined) { + setArchName(archName); + } + const lBranch: string = (router.query.lBranch as string) ?? undefined; if (lBranch !== undefined) { setLBranch(lBranch); @@ -260,7 +272,8 @@ export default function Page() { const queryName = "oss_ci_benchmark_names"; const queryParams = { - deviceArch: deviceName === DEFAULT_DEVICE_NAME ? "" : deviceName, + arch: archName === DEFAULT_ARCH_NAME ? "" : archName, + device: deviceName === DEFAULT_DEVICE_NAME ? "" : deviceName, dtypes: dtypeName === DEFAULT_DTYPE_NAME ? [] @@ -326,7 +339,9 @@ export default function Page() { backendName )}&dtypeName=${encodeURIComponent( dtypeName - )}&deviceName=${encodeURIComponent(deviceName)}`} + )}&deviceName=${encodeURIComponent( + deviceName + )}&archName=${encodeURIComponent(archName)}`} /> @@ -365,6 +380,14 @@ export default function Page() { label={"DType"} /> )} + {repoName === "pytorch/executorch" && ( + + )}