From 0a89152fd4a5af08fa7f3a3bf799865b117d3a74 Mon Sep 17 00:00:00 2001 From: Sam Albers Date: Fri, 28 Jun 2024 11:24:47 -0700 Subject: [PATCH 1/7] update --- performance-release-report/renv.lock | 305 +++++++++++++-------------- 1 file changed, 151 insertions(+), 154 deletions(-) diff --git a/performance-release-report/renv.lock b/performance-release-report/renv.lock index 70e15270dfe..df9011d8297 100644 --- a/performance-release-report/renv.lock +++ b/performance-release-report/renv.lock @@ -1,6 +1,6 @@ { "R": { - "Version": "4.3.1", + "Version": "4.4.0", "Repositories": [ { "Name": "CRAN", @@ -11,9 +11,9 @@ "Packages": { "MASS": { "Package": "MASS", - "Version": "7.3-60", + "Version": "7.3-60.2", "Source": "Repository", - "Repository": "RSPM", + "Repository": "CRAN", "Requirements": [ "R", "grDevices", @@ -22,11 +22,11 @@ "stats", "utils" ], - "Hash": "a56a6365b3fa73293ea8d084be0d9bb0" + "Hash": "2f342c46163b0b54d7b64d1f798e2c78" }, "Matrix": { "Package": "Matrix", - "Version": "1.6-1.1", + "Version": "1.7-0", "Source": "Repository", "Repository": "CRAN", "Requirements": [ @@ -39,7 +39,7 @@ "stats", "utils" ], - "Hash": "1a00d4828f33a9d690806e98bd17150c" + "Hash": "1920b2f11133b12350024297d8a4ff4a" }, "R6": { "Package": "R6", @@ -63,18 +63,18 @@ }, "Rcpp": { "Package": "Rcpp", - "Version": "1.0.11", + "Version": "1.0.12", "Source": "Repository", - "Repository": "CRAN", + "Repository": "RSPM", "Requirements": [ "methods", "utils" ], - "Hash": "ae6cbbe1492f4de79c45fce06f967ce8" + "Hash": "5ea2700d21e038ace58269ecdbeb9ec0" }, "V8": { "Package": "V8", - "Version": "4.4.0", + "Version": "4.4.2", "Source": "Repository", "Repository": "CRAN", "Requirements": [ @@ -83,7 +83,7 @@ "jsonlite", "utils" ], - "Hash": "df924bedbdcaaf3b1d3ac7ed94f4d001" + "Hash": "ca98390ad1cef2a5a609597b49d3d042" }, "askpass": { "Package": "askpass", @@ -124,45 +124,47 @@ }, "bslib": { "Package": "bslib", - "Version": "0.5.1", + "Version": "0.7.0", "Source": "Repository", - "Repository": "CRAN", + "Repository": "RSPM", "Requirements": [ "R", "base64enc", "cachem", + "fastmap", "grDevices", "htmltools", "jquerylib", "jsonlite", + "lifecycle", "memoise", "mime", "rlang", "sass" ], - "Hash": "283015ddfbb9d7bf15ea9f0b5698f0d9" + "Hash": "8644cc53f43828f19133548195d7e59e" }, "cachem": { "Package": "cachem", - "Version": "1.0.8", + "Version": "1.1.0", "Source": "Repository", - "Repository": "RSPM", + "Repository": "CRAN", "Requirements": [ "fastmap", "rlang" ], - "Hash": "c35768291560ce302c0a6589f92e837d" + "Hash": "cd9a672193789068eb5a2aad65a0dedf" }, "cli": { "Package": "cli", - "Version": "3.6.1", + "Version": "3.6.2", "Source": "Repository", "Repository": "RSPM", "Requirements": [ "R", "utils" ], - "Hash": "89e6d8219950eac806ae0c489052048a" + "Hash": "1216ac65ac55ec0058a6f75d7ca0fd52" }, "colorspace": { "Package": "colorspace", @@ -180,10 +182,10 @@ }, "commonmark": { "Package": "commonmark", - "Version": "1.9.0", + "Version": "1.9.1", "Source": "Repository", "Repository": "RSPM", - "Hash": "d691c61bff84bd63c383874d2d0c3307" + "Hash": "5d8225445acb167abf7797de48b2ee3c" }, "conbenchcoms": { "Package": "conbenchcoms", @@ -206,38 +208,38 @@ }, "cpp11": { "Package": "cpp11", - "Version": "0.4.6", + "Version": "0.4.7", "Source": "Repository", - "Repository": "CRAN", + "Repository": "RSPM", "Requirements": [ "R" ], - "Hash": "707fae4bbf73697ec8d85f9d7076c061" + "Hash": "5a295d7d963cc5035284dcdbaf334f4e" }, "curl": { "Package": "curl", - "Version": "5.1.0", + "Version": "5.2.1", "Source": "Repository", "Repository": "CRAN", "Requirements": [ "R" ], - "Hash": "9123f3ef96a2c1a93927d828b2fe7d4c" + "Hash": "411ca2c03b1ce5f548345d2fc2685f7a" }, "digest": { "Package": "digest", - "Version": "0.6.33", + "Version": "0.6.35", "Source": "Repository", - "Repository": "CRAN", + "Repository": "RSPM", "Requirements": [ "R", "utils" ], - "Hash": "b18a9cf3c003977b0cc49d5e76ebe48d" + "Hash": "698ece7ba5a4fa4559e3d537e7ec3d31" }, "dplyr": { "Package": "dplyr", - "Version": "1.1.3", + "Version": "1.1.4", "Source": "Repository", "Repository": "CRAN", "Requirements": [ @@ -256,33 +258,22 @@ "utils", "vctrs" ], - "Hash": "e85ffbebaad5f70e1a2e2ef4302b4949" - }, - "ellipsis": { - "Package": "ellipsis", - "Version": "0.3.2", - "Source": "Repository", - "Repository": "RSPM", - "Requirements": [ - "R", - "rlang" - ], - "Hash": "bb0eec2fe32e88d9e2836c2f73ea2077" + "Hash": "fedd9d00c2944ff00a0e2696ccf048ec" }, "evaluate": { "Package": "evaluate", - "Version": "0.22", + "Version": "0.23", "Source": "Repository", - "Repository": "CRAN", + "Repository": "RSPM", "Requirements": [ "R", "methods" ], - "Hash": "66f39c7a21e03c4dcb2c2d21d738d603" + "Hash": "daf4a1246be12c1fa8c7705a0935c1a0" }, "fansi": { "Package": "fansi", - "Version": "1.0.5", + "Version": "1.0.6", "Source": "Repository", "Repository": "CRAN", "Requirements": [ @@ -290,21 +281,21 @@ "grDevices", "utils" ], - "Hash": "3e8583a60163b4bc1a80016e63b9959e" + "Hash": "962174cf2aeb5b9eea581522286a911f" }, "farver": { "Package": "farver", - "Version": "2.1.1", + "Version": "2.1.2", "Source": "Repository", - "Repository": "RSPM", - "Hash": "8106d78941f34855c440ddb946b8f7a5" + "Repository": "CRAN", + "Hash": "680887028577f3fa2a81e410ed0d6e42" }, "fastmap": { "Package": "fastmap", - "Version": "1.1.1", + "Version": "1.2.0", "Source": "Repository", - "Repository": "RSPM", - "Hash": "f7736a18de97dea803bde0a2daaafb27" + "Repository": "CRAN", + "Hash": "aa5e1cd11c2d15497494c5292d7ffcc8" }, "fontawesome": { "Package": "fontawesome", @@ -336,14 +327,14 @@ }, "fs": { "Package": "fs", - "Version": "1.6.3", + "Version": "1.6.4", "Source": "Repository", - "Repository": "CRAN", + "Repository": "RSPM", "Requirements": [ "R", "methods" ], - "Hash": "47b5f30c720c23999b913a1a635cf0bb" + "Hash": "15aeb8c27f5ea5161f9f6a641fafd93a" }, "generics": { "Package": "generics", @@ -358,11 +349,12 @@ }, "ggiraph": { "Package": "ggiraph", - "Version": "0.8.7", + "Version": "0.8.10", "Source": "Repository", "Repository": "CRAN", "Requirements": [ "Rcpp", + "cli", "ggplot2", "grid", "htmltools", @@ -374,13 +366,13 @@ "uuid", "vctrs" ], - "Hash": "cc4b14781abf296aab585c5c056d9685" + "Hash": "15748f4335af873289fbdd31610c3f96" }, "ggplot2": { "Package": "ggplot2", - "Version": "3.4.3", + "Version": "3.5.1", "Source": "Repository", - "Repository": "CRAN", + "Repository": "RSPM", "Requirements": [ "MASS", "R", @@ -399,22 +391,22 @@ "vctrs", "withr" ], - "Hash": "85846544c596e71f8f46483ab165da33" + "Hash": "44c6a2f8202d5b7e878ea274b1092426" }, "glue": { "Package": "glue", - "Version": "1.6.2", + "Version": "1.7.0", "Source": "Repository", "Repository": "RSPM", "Requirements": [ "R", "methods" ], - "Hash": "4f2596dfb05dac67b9dc558e5c6fba2e" + "Hash": "e0b3a53876554bd45879e596cdb10a52" }, "gt": { "Package": "gt", - "Version": "0.10.0", + "Version": "0.10.1", "Source": "Repository", "Repository": "CRAN", "Requirements": [ @@ -436,17 +428,17 @@ "rlang", "sass", "scales", - "tibble", "tidyselect", + "vctrs", "xml2" ], - "Hash": "21737c74811cccac01b5097bcb0f8b4c" + "Hash": "03009c105dfae79460b8eb9d8cf791e4" }, "gtable": { "Package": "gtable", - "Version": "0.3.4", + "Version": "0.3.5", "Source": "Repository", - "Repository": "CRAN", + "Repository": "RSPM", "Requirements": [ "R", "cli", @@ -455,7 +447,7 @@ "lifecycle", "rlang" ], - "Hash": "b29cf3031f49b04ab9c852c912547eef" + "Hash": "e18861963cbc65a27736e02b3cd3c4a0" }, "highr": { "Package": "highr", @@ -484,26 +476,25 @@ }, "htmltools": { "Package": "htmltools", - "Version": "0.5.6.1", + "Version": "0.5.8.1", "Source": "Repository", - "Repository": "CRAN", + "Repository": "RSPM", "Requirements": [ "R", "base64enc", "digest", - "ellipsis", "fastmap", "grDevices", "rlang", "utils" ], - "Hash": "1e12fe667316a76508898839ecfb2d00" + "Hash": "81d371a9cc60640e74e4ab6ac46dcedc" }, "htmlwidgets": { "Package": "htmlwidgets", - "Version": "1.6.2", + "Version": "1.6.4", "Source": "Repository", - "Repository": "CRAN", + "Repository": "RSPM", "Requirements": [ "grDevices", "htmltools", @@ -512,26 +503,28 @@ "rmarkdown", "yaml" ], - "Hash": "a865aa85bcb2697f47505bfd70422471" + "Hash": "04291cc45198225444a397606810ac37" }, "httr2": { "Package": "httr2", - "Version": "0.2.3", + "Version": "1.0.1", "Source": "Repository", - "Repository": "RSPM", + "Repository": "CRAN", "Requirements": [ "R", "R6", "cli", "curl", "glue", + "lifecycle", "magrittr", "openssl", "rappdirs", "rlang", + "vctrs", "withr" ], - "Hash": "193bb297368afbbb42dc85784a46b36e" + "Hash": "03d741c92fda96d98c3a3f22494e3b4a" }, "isoband": { "Package": "isoband", @@ -578,13 +571,13 @@ }, "jsonlite": { "Package": "jsonlite", - "Version": "1.8.7", + "Version": "1.8.8", "Source": "Repository", "Repository": "RSPM", "Requirements": [ "methods" ], - "Hash": "266a20443ca13c65688b2116d5220f76" + "Hash": "e1b9c55281c5adc4dd113652d9e26768" }, "juicyjuice": { "Package": "juicyjuice", @@ -598,9 +591,9 @@ }, "knitr": { "Package": "knitr", - "Version": "1.44", + "Version": "1.46", "Source": "Repository", - "Repository": "CRAN", + "Repository": "RSPM", "Requirements": [ "R", "evaluate", @@ -610,7 +603,7 @@ "xfun", "yaml" ], - "Hash": "60885b9f746c9dfaef110d070b5f7dc0" + "Hash": "6e008ab1d696a5283c79765fa7b56b47" }, "labeling": { "Package": "labeling", @@ -625,7 +618,7 @@ }, "lattice": { "Package": "lattice", - "Version": "0.21-9", + "Version": "0.22-6", "Source": "Repository", "Repository": "CRAN", "Requirements": [ @@ -636,11 +629,11 @@ "stats", "utils" ], - "Hash": "5558c61e0136e247252f5f952cdaad6a" + "Hash": "cc5ac1ba4c238c7ca9fa6a87ca11a7e2" }, "lifecycle": { "Package": "lifecycle", - "Version": "1.0.3", + "Version": "1.0.4", "Source": "Repository", "Repository": "RSPM", "Requirements": [ @@ -649,7 +642,7 @@ "glue", "rlang" ], - "Hash": "001cecbeac1cff9301bdc3775ee46a86" + "Hash": "b8552d117e1b808b09a832f589b79035" }, "lubridate": { "Package": "lubridate", @@ -676,7 +669,7 @@ }, "markdown": { "Package": "markdown", - "Version": "1.10", + "Version": "1.12", "Source": "Repository", "Repository": "CRAN", "Requirements": [ @@ -685,7 +678,7 @@ "utils", "xfun" ], - "Hash": "f00a677e0c006183f1232b6f1e0652d0" + "Hash": "765cf53992401b3b6c297b69e1edb8bd" }, "memoise": { "Package": "memoise", @@ -700,7 +693,7 @@ }, "mgcv": { "Package": "mgcv", - "Version": "1.9-0", + "Version": "1.9-1", "Source": "Repository", "Repository": "CRAN", "Requirements": [ @@ -713,7 +706,7 @@ "stats", "utils" ], - "Hash": "086028ca0460d0c368028d3bda58f31b" + "Hash": "110ee9d83b496279960e162ac97764ce" }, "mime": { "Package": "mime", @@ -727,18 +720,18 @@ }, "munsell": { "Package": "munsell", - "Version": "0.5.0", + "Version": "0.5.1", "Source": "Repository", "Repository": "RSPM", "Requirements": [ "colorspace", "methods" ], - "Hash": "6dfe8bf774944bd5595785e3229d8771" + "Hash": "4fd8900853b746af55b81fda99da7695" }, "nlme": { "Package": "nlme", - "Version": "3.1-163", + "Version": "3.1-164", "Source": "Repository", "Repository": "CRAN", "Requirements": [ @@ -748,17 +741,17 @@ "stats", "utils" ], - "Hash": "8d1938040a05566f4f7a14af4feadd6b" + "Hash": "a623a2239e642806158bc4dc3f51565d" }, "openssl": { "Package": "openssl", - "Version": "2.1.1", + "Version": "2.2.0", "Source": "Repository", "Repository": "CRAN", "Requirements": [ "askpass" ], - "Hash": "2a0dc8c6adfb6f032e4d4af82d258ab5" + "Hash": "2bcca3848e4734eb3b16103bc9aa4b8e" }, "pillar": { "Package": "pillar", @@ -849,18 +842,18 @@ }, "rlang": { "Package": "rlang", - "Version": "1.1.1", + "Version": "1.1.3", "Source": "Repository", "Repository": "RSPM", "Requirements": [ "R", "utils" ], - "Hash": "a85c767b55f0bf9b7ad16c6d7baee5bb" + "Hash": "42548638fae05fd9a9b5f3f437fbbbe2" }, "rmarkdown": { "Package": "rmarkdown", - "Version": "2.25", + "Version": "2.27", "Source": "Repository", "Repository": "CRAN", "Requirements": [ @@ -873,20 +866,19 @@ "jsonlite", "knitr", "methods", - "stringr", "tinytex", "tools", "utils", "xfun", "yaml" ], - "Hash": "d65e35823c817f09f4de424fcdfa812a" + "Hash": "27f9502e1cdbfa195f94e03b0f517484" }, "sass": { "Package": "sass", - "Version": "0.4.7", + "Version": "0.4.9", "Source": "Repository", - "Repository": "CRAN", + "Repository": "RSPM", "Requirements": [ "R6", "fs", @@ -894,37 +886,39 @@ "rappdirs", "rlang" ], - "Hash": "6bd4d33b50ff927191ec9acbf52fd056" + "Hash": "d53dbfddf695303ea4ad66f86e99b95d" }, "scales": { "Package": "scales", - "Version": "1.2.1", + "Version": "1.3.0", "Source": "Repository", "Repository": "RSPM", "Requirements": [ "R", "R6", "RColorBrewer", + "cli", "farver", + "glue", "labeling", "lifecycle", "munsell", "rlang", "viridisLite" ], - "Hash": "906cb23d2f1c5680b8ce439b44c6fa63" + "Hash": "c19df082ba346b0ffa6f833e92de34d1" }, "showtext": { "Package": "showtext", - "Version": "0.9-6", + "Version": "0.9-7", "Source": "Repository", - "Repository": "CRAN", + "Repository": "RSPM", "Requirements": [ "grDevices", "showtextdb", "sysfonts" ], - "Hash": "c0fd332d248b195bbcb94a0dfda37b0d" + "Hash": "ebc23fc796c28737ffe0a64e5404f3d1" }, "showtextdb": { "Package": "showtextdb", @@ -951,20 +945,20 @@ }, "stringi": { "Package": "stringi", - "Version": "1.7.12", + "Version": "1.8.4", "Source": "Repository", - "Repository": "RSPM", + "Repository": "CRAN", "Requirements": [ "R", "stats", "tools", "utils" ], - "Hash": "ca8bd84263c77310739d2cf64d84d7c9" + "Hash": "39e1144fd75428983dc3f63aa53dfa91" }, "stringr": { "Package": "stringr", - "Version": "1.5.0", + "Version": "1.5.1", "Source": "Repository", "Repository": "RSPM", "Requirements": [ @@ -977,7 +971,7 @@ "stringi", "vctrs" ], - "Hash": "671a4d384ae9d32fc47a14e98bfa3dc8" + "Hash": "960e2ae9e09656611e0b8214ad543207" }, "sys": { "Package": "sys", @@ -988,21 +982,22 @@ }, "sysfonts": { "Package": "sysfonts", - "Version": "0.8.8", + "Version": "0.8.9", "Source": "Repository", - "Repository": "CRAN", - "Hash": "7f4dac41a3e348ae12832167e6beb875" + "Repository": "RSPM", + "Hash": "7dfca1e9c5c278300b5ca6a1772072f7" }, "systemfonts": { "Package": "systemfonts", - "Version": "1.0.5", + "Version": "1.1.0", "Source": "Repository", "Repository": "CRAN", "Requirements": [ "R", - "cpp11" + "cpp11", + "lifecycle" ], - "Hash": "15b594369e70b975ba9f064295983499" + "Hash": "213b6b8ed5afbf934843e6c3b090d418" }, "tibble": { "Package": "tibble", @@ -1025,7 +1020,7 @@ }, "tidyr": { "Package": "tidyr", - "Version": "1.3.0", + "Version": "1.3.1", "Source": "Repository", "Repository": "RSPM", "Requirements": [ @@ -1044,13 +1039,13 @@ "utils", "vctrs" ], - "Hash": "e47debdc7ce599b070c8e78e8ac0cfcf" + "Hash": "915fb7ce036c22a6a33b5a8adb712eb1" }, "tidyselect": { "Package": "tidyselect", - "Version": "1.2.0", + "Version": "1.2.1", "Source": "Repository", - "Repository": "RSPM", + "Repository": "CRAN", "Requirements": [ "R", "cli", @@ -1060,54 +1055,54 @@ "vctrs", "withr" ], - "Hash": "79540e5fcd9e0435af547d885f184fd5" + "Hash": "829f27b9c4919c16b593794a6344d6c0" }, "timechange": { "Package": "timechange", - "Version": "0.2.0", + "Version": "0.3.0", "Source": "Repository", - "Repository": "CRAN", + "Repository": "RSPM", "Requirements": [ "R", "cpp11" ], - "Hash": "8548b44f79a35ba1791308b61e6012d7" + "Hash": "c5f3c201b931cd6474d17d8700ccb1c8" }, "tinytex": { "Package": "tinytex", - "Version": "0.47", + "Version": "0.51", "Source": "Repository", "Repository": "CRAN", "Requirements": [ "xfun" ], - "Hash": "8d4ccb733843e513c1c1cdd66a759f0d" + "Hash": "d44e2fcd2e4e076f0aac540208559d1d" }, "utf8": { "Package": "utf8", - "Version": "1.2.3", + "Version": "1.2.4", "Source": "Repository", - "Repository": "RSPM", + "Repository": "CRAN", "Requirements": [ "R" ], - "Hash": "1fe17157424bb09c48a8b3b550c753bc" + "Hash": "62b65c52671e6665f803ff02954446e9" }, "uuid": { "Package": "uuid", - "Version": "1.1-1", + "Version": "1.2-0", "Source": "Repository", - "Repository": "CRAN", + "Repository": "RSPM", "Requirements": [ "R" ], - "Hash": "3d78edfb977a69fc7a0341bee25e163f" + "Hash": "303c19bfd970bece872f93a824e323d9" }, "vctrs": { "Package": "vctrs", - "Version": "0.6.3", + "Version": "0.6.5", "Source": "Repository", - "Repository": "CRAN", + "Repository": "RSPM", "Requirements": [ "R", "cli", @@ -1115,7 +1110,7 @@ "lifecycle", "rlang" ], - "Hash": "d0ef2856b83dc33ea6e255caf6229ee2" + "Hash": "c03fa420630029418f7e6da3667aac4a" }, "viridisLite": { "Package": "viridisLite", @@ -1129,45 +1124,47 @@ }, "withr": { "Package": "withr", - "Version": "2.5.1", + "Version": "3.0.0", "Source": "Repository", - "Repository": "CRAN", + "Repository": "RSPM", "Requirements": [ "R", "grDevices", - "graphics", - "stats" + "graphics" ], - "Hash": "d77c6f74be05c33164e33fbc85540cae" + "Hash": "d31b6c62c10dcf11ec530ca6b0dd5d35" }, "xfun": { "Package": "xfun", - "Version": "0.40", + "Version": "0.44", "Source": "Repository", "Repository": "CRAN", "Requirements": [ + "grDevices", "stats", "tools" ], - "Hash": "be07d23211245fc7d4209f54c4e4ffc8" + "Hash": "317a0538d32f4a009658bcedb7923f4b" }, "xml2": { "Package": "xml2", - "Version": "1.3.5", + "Version": "1.3.6", "Source": "Repository", - "Repository": "CRAN", + "Repository": "RSPM", "Requirements": [ "R", - "methods" + "cli", + "methods", + "rlang" ], - "Hash": "6c40e5cfcc6aefd88110666e18c31f40" + "Hash": "1d0336142f4cd25d8d23cd3ba7a8fb61" }, "yaml": { "Package": "yaml", - "Version": "2.3.7", + "Version": "2.3.8", "Source": "Repository", "Repository": "RSPM", - "Hash": "0d0056cc5383fbc240ccd0cb584bf436" + "Hash": "29240487a071f535f5e5d5a323b7afbd" } } } From 36398dab530600f91faad0b2a0e12dac0e673949 Mon Sep 17 00:00:00 2001 From: Sam Albers Date: Fri, 28 Jun 2024 11:25:01 -0700 Subject: [PATCH 2/7] new z score less table --- performance-release-report/R/functions.R | 52 ++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/performance-release-report/R/functions.R b/performance-release-report/R/functions.R index 17ed2482dd1..c1477b07223 100644 --- a/performance-release-report/R/functions.R +++ b/performance-release-report/R/functions.R @@ -226,4 +226,56 @@ top_zscore_table <- function(.data, top_n = 20, direction = c("improvement", "re footnote = "MB/s = megabytes per second; ns = nanoseconds; i/s = iterations per second", locations = cells_body(columns = "unit") ) +} + +top_perf_table <- function(.data, top_n = 20, direction = c("improvement", "regression")) { + + direction <- match.arg(direction) + + if (direction == "improvement") { + .data <- .data %>% + arrange(desc(analysis_pairwise_percent_change)) + } else { + .data <- .data %>% + arrange(analysis_pairwise_percent_change) + } + + ## let's convert things to megabytes + .data <- .data %>% + mutate(across(ends_with("single_value_summary"), ~ case_when( + unit == "B/s" ~ .x/1000000, ## B/s -> MB/s + TRUE ~ .x + ))) %>% + mutate(unit = case_when( + unit == "B/s" ~ "MB/s", + TRUE ~ unit + )) + + .data %>% + head(top_n) %>% + mutate(name = glue("[{name}]({cb_url})")) %>% + select( + language, suite, name, params, analysis_pairwise_percent_change, baseline_single_value_summary, contender_single_value_summary, unit) %>% + arrange(language, suite, name, params) %>% + gt(rowname_col = "language", groupname_col = "suite") %>% + fmt_markdown(columns = "name") %>% + fmt_percent(columns = "analysis_pairwise_percent_change", scale_values = FALSE, decimals = 2) %>% + fmt_number(columns = ends_with("single_value_summary"), decimals = 0) %>% + cols_label( + language = "Language", + name = "Benchmark", + suite = "Suite", + params = "Params", + baseline_single_value_summary = "Baseline result", + contender_single_value_summary = "Contender result", + analysis_pairwise_percent_change = "Percent Change", + ) %>% + tab_spanner(columns = c("baseline_single_value_summary", "contender_single_value_summary", "unit"), label= "Results") %>% + tab_spanner(columns = starts_with("analysis_"), label= "Analysis") %>% + opt_table_font(font = google_font("Roboto Mono")) %>% + tab_options(table.font.size = "10px") %>% + tab_footnote( + footnote = "MB/s = megabytes per second; ns = nanoseconds; i/s = iterations per second", + locations = cells_body(columns = "unit") + ) } \ No newline at end of file From b2af5e19d340e96bb8700e0d13367f9f8488ae73 Mon Sep 17 00:00:00 2001 From: Sam Albers Date: Fri, 28 Jun 2024 11:55:52 -0700 Subject: [PATCH 3/7] use cloud machines --- .../performance-release-report.qmd | 55 ++++++------------- 1 file changed, 18 insertions(+), 37 deletions(-) diff --git a/performance-release-report/performance-release-report.qmd b/performance-release-report/performance-release-report.qmd index 5b02f1f0cb5..8adaa4faf8c 100644 --- a/performance-release-report/performance-release-report.qmd +++ b/performance-release-report/performance-release-report.qmd @@ -31,7 +31,7 @@ baseline_git_commit <- Sys.getenv("BASELINE_GIT_COMMIT") contender_git_commit <- Sys.getenv("CONTENDER_GIT_COMMIT") # baseline_git_commit <- '5bf86ab4d9e9bc5bb7e1c6e65a55d9f1723597bf' # contender_git_commit <- 'b7d2f7ffca66c868bd2fce5b3749c6caa002a7f0' -hardware_name <- c("ursa-i9-9960x", "ursa-thinkcentre-m75q") +hardware_name <- c("ec2-m5-4xlarge-us-east-2", "ec2-c6a-4xlarge-us-east-2") library(dplyr) library(ggplot2) @@ -97,7 +97,9 @@ if (!nzchar(baseline_git_commit) | !nzchar(contender_git_commit)) { #| results: 'asis' #| cache: !expr '!is_gha()' -run_comp <- find_runs(baseline_git_commit, contender_git_commit, hardware_name) +run_comp <- find_runs(baseline_git_commit, contender_git_commit, hardware_name) |> + filter(id != "5200ba71e40e462da1cdbb7ff57fcc50") ## old m5 that we don't want for comparisons + if (length(run_comp) == 0) { knit_exit("No runs found for the given commits. Please check that the commits are correct and that the benchmark runs have completed.") @@ -105,18 +107,19 @@ if (length(run_comp) == 0) { # Compare the baseline to the contender for # macrobenchmarks -ursa_i9_bm <- run_comp %>% - filter(hardware.name == "ursa-i9-9960x") %>% +m5_bm <- run_comp %>% + filter(hardware.name == "ec2-m5-4xlarge-us-east-2") %>% compare_baseline_to_contender() -macro_bm_df <- ursa_i9_bm %>% +macro_bm_df <- m5_bm %>% filter(baseline.language %in% c("Python", "R")) # microbenchmarks micro_bm_df <- run_comp %>% - filter(hardware.name == "ursa-thinkcentre-m75q") %>% + filter(hardware.name == "ec2-c6a-4xlarge-us-east-2") %>% compare_baseline_to_contender() %>% - bind_rows(ursa_i9_bm %>% filter(baseline.language %in% "JavaScript")) + filter(baseline.language %in% c("C++", "JavaScript", "Java")) |> + bind_rows(m5_bm %>% filter(baseline.language %in% "JavaScript")) ``` @@ -359,37 +362,35 @@ micro_bm_proced <- micro_bm_df %>% group_modify(~ tidy_compare(.x, .y)) %>% ungroup() %>% filter(!is.na(name)) %>% - filter(!is.na(analysis.lookback_z_score.regression_indicated)) %>% ## indicator of some empty data + # filter(!is.na(analysis.lookback_z_score.regression_indicated)) %>% ## indicator of some empty data ## this will enable the yaxis to be populated with names when params is NA. params is preferable because it is more specific mutate(params = ifelse(is.na(params), baseline.case_permutation, params)) %>% rowwise() %>% mutate(params = paste(strwrap(params, 10), collapse="\n")) %>% clean_names() %>% - select(language, baseline_benchmark_name, name, params, suite, analysis_pairwise_regression_indicated, analysis_pairwise_improvement_indicated, change, difference, pn_lab, analysis_lookback_z_score_z_score, analysis_lookback_z_score_z_threshold, analysis_pairwise_percent_change, baseline_single_value_summary, contender_single_value_summary, cb_url, unit) + select(language, baseline_benchmark_name, name, params, suite, analysis_pairwise_improvement_indicated, analysis_pairwise_regression_indicated, change, difference, pn_lab, analysis_pairwise_percent_change, baseline_single_value_summary, contender_single_value_summary, cb_url, unit) ``` There are currently `r nrow(micro_bm_proced)` microbenchmarks in the Arrow benchmarks. The following comparisons are also available to be viewed in the [Conbench UI](`r generate_compare_url(micro_bm_df)`). ```{r table-micro-bm-summary} -threshold <- unique(micro_bm_proced$analysis_lookback_z_score_z_threshold) -threshold <- threshold[!is.na(threshold)] micro_bm_proced %>% count(language, analysis_pairwise_regression_indicated, analysis_pairwise_improvement_indicated) %>% mutate(col_var = case_when( analysis_pairwise_regression_indicated == TRUE ~ "Regressions", analysis_pairwise_improvement_indicated == TRUE ~ "Improvements", + is.na(analysis_pairwise_regression_indicated) | is.na(analysis_pairwise_improvement_indicated) ~ "No comparison", TRUE ~ "Stable" )) %>% select(-all_of(starts_with("analysis_pairwise"))) %>% pivot_wider(names_from = col_var, values_from = n) %>% rowwise() %>% - mutate(Total = sum(c_across(c(Stable, Improvements, Regressions)))) %>% - mutate(`z-score threshold` = threshold, .after = language) %>% + mutate(Total = sum(c_across(c(Stable, Improvements, Regressions)), na.rm = TRUE)) %>% gt() %>% cols_label(language = "Language") %>% tab_spanner( label = "Number of microbenchmarks", - columns = c(Stable, Improvements, Regressions, Total) + columns = c(Stable, Improvements, Regressions, `No comparison`, Total) ) %>% opt_table_font(font = google_font("Roboto Mono")) ``` @@ -401,7 +402,7 @@ Because of the large number of benchmarks, the top 20 benchmark results that dev ## Largest 20 regressions between baseline and contender ```{r table-top-zscores-negative} -top_zscore_table(micro_bm_proced, direction = "regression") +top_perf_table(micro_bm_proced, direction = "regression") ``` @@ -412,31 +413,11 @@ top_zscore_table(micro_bm_proced, direction = "regression") ## Largest 20 improvements between baseline and contender ```{r table-top-zscores-positive} -top_zscore_table(micro_bm_proced, direction = "improvement") +top_perf_table(micro_bm_proced, direction = "improvement") ``` ::: -## z-score distribution - -Plotting the distribution of zscores for all microbenchmark results will help identify any systematic differences between the baseline and contender. The shape of the distribution of z-scores provides a sense of the overall performance of the contender relative to the baseline. Narrow distributions centered around 0 indicate that the contender is performing similarly to the baseline. Wider distributions indicate that the contender is performing differently than the baseline with left skewing indicating regressions and right skewing indicating improvements. - - -```{ojs} -Plot.plot({ - y: {grid: true}, - x: { - label: "z-score" - }, - color: {legend: false}, - width: 1000, - height: 400, - marks: [ - Plot.rectY(microBmProced, Plot.binX({y: "count"}, {x: "analysis_lookback_z_score_z_score", fill: "grey", tip: true})), - Plot.ruleY([0]) - ] -}) -``` ```{r ojs-defn} ojs_define(ojs_micro_bm_proced = micro_bm_proced) @@ -452,7 +433,7 @@ microBmProced = aq.from(transpose(ojs_micro_bm_proced)) ## Microbenchmark explorer {#micro-bm-explorer} -This microbenchmarks explorer allows you to filter the microbenchmark results by language, suite, and benchmark name and toggle regressions and improvements based on a threshold level of `r threshold` z-scores. Languages, suite and benchmark name need to be selected to show a benchmark plot. Additional benchmark parameters are displayed on the vertical axis resulting in each bar representing a case permutation. If a benchmark does not have additional parameters, the full case permutation string is displayed. Each bar can be clicked to open the Conbench UI page for that benchmark providing additional history and metadata for that case permutation. +This microbenchmarks explorer allows you to filter the microbenchmark results by language, suite, and benchmark name and toggle regressions and improvements based on a percent change between the baseline and contender |> . Languages, suite and benchmark name need to be selected to show a benchmark plot. Additional benchmark parameters are displayed on the vertical axis resulting in each bar representing a case permutation. If a benchmark does not have additional parameters, the full case permutation string is displayed. Each bar can be clicked to open the Conbench UI page for that benchmark providing additional history and metadata for that case permutation. ```{ojs filter-micro-bm} // Top level: are there regressions/improvements? From e4595bd331e7f30619653ff32d461e8c24b71980 Mon Sep 17 00:00:00 2001 From: Sam Albers Date: Fri, 28 Jun 2024 12:59:08 -0700 Subject: [PATCH 4/7] change commit defaults --- .github/workflows/performance-release-report.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/performance-release-report.yml b/.github/workflows/performance-release-report.yml index 2f9058b4f05..06170552907 100644 --- a/.github/workflows/performance-release-report.yml +++ b/.github/workflows/performance-release-report.yml @@ -33,8 +33,8 @@ permissions: env: ## payload vars - BASELINE_GIT_COMMIT: ${{ github.event.inputs.baseline_git_commit || '2dcee3f82c6cf54b53a64729fd81840efa583244' }} - CONTENDER_GIT_COMMIT: ${{ github.event.inputs.contender_git_commit || 'b5d26f833c5dfa1494adecccbcc9181bd31e3787' }} + BASELINE_GIT_COMMIT: ${{ github.event.inputs.baseline_git_commit || '7dd1d34074af176d9e861a360e135ae57b21cf96' }} + CONTENDER_GIT_COMMIT: ${{ github.event.inputs.contender_git_commit || 'a42df4baf09f9b4d168c5ad5139003ed7bdf2246' }} RC_LABEL: ${{ github.event.inputs.rc_label || 'manual' }} jobs: From a627bc8899f569a2544b2cb7aa0af2543529be4e Mon Sep 17 00:00:00 2001 From: Sam Albers Date: Fri, 28 Jun 2024 13:20:00 -0700 Subject: [PATCH 5/7] update R --- .github/workflows/performance-release-report.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/performance-release-report.yml b/.github/workflows/performance-release-report.yml index 06170552907..dd48dda749c 100644 --- a/.github/workflows/performance-release-report.yml +++ b/.github/workflows/performance-release-report.yml @@ -57,7 +57,7 @@ jobs: - name: Setup R uses: r-lib/actions/setup-r@v2 with: - r-version: '4.3.1' + r-version: '4.4.0' use-public-rspm: true # Needed due to https://github.com/r-lib/actions/issues/618 From eb1f97a161737a65583017b1ade44a787045d600 Mon Sep 17 00:00:00 2001 From: Sam Albers Date: Fri, 28 Jun 2024 13:35:05 -0700 Subject: [PATCH 6/7] bump renv --- performance-release-report/renv.lock | 9 +-- performance-release-report/renv/activate.R | 80 ++++++++++++++++------ 2 files changed, 62 insertions(+), 27 deletions(-) diff --git a/performance-release-report/renv.lock b/performance-release-report/renv.lock index df9011d8297..a8aba3b978f 100644 --- a/performance-release-report/renv.lock +++ b/performance-release-report/renv.lock @@ -832,13 +832,8 @@ }, "renv": { "Package": "renv", - "Version": "1.0.3", - "Source": "Repository", - "Repository": "CRAN", - "Requirements": [ - "utils" - ], - "Hash": "41b847654f567341725473431dd0d5ab" + "Version": "1.0.7", + "Source": "Repository" }, "rlang": { "Package": "rlang", diff --git a/performance-release-report/renv/activate.R b/performance-release-report/renv/activate.R index cb5401f93cb..d13f9932a16 100644 --- a/performance-release-report/renv/activate.R +++ b/performance-release-report/renv/activate.R @@ -2,11 +2,13 @@ local({ # the requested version of renv - version <- "1.0.3" + version <- "1.0.7" attr(version, "sha") <- NULL # the project directory - project <- getwd() + project <- Sys.getenv("RENV_PROJECT") + if (!nzchar(project)) + project <- getwd() # use start-up diagnostics if enabled diagnostics <- Sys.getenv("RENV_STARTUP_DIAGNOSTICS", unset = "FALSE") @@ -31,6 +33,14 @@ local({ if (!is.null(override)) return(override) + # if we're being run in a context where R_LIBS is already set, + # don't load -- presumably we're being run as a sub-process and + # the parent process has already set up library paths for us + rcmd <- Sys.getenv("R_CMD", unset = NA) + rlibs <- Sys.getenv("R_LIBS", unset = NA) + if (!is.na(rlibs) && !is.na(rcmd)) + return(FALSE) + # next, check environment variables # TODO: prefer using the configuration one in the future envvars <- c( @@ -50,9 +60,22 @@ local({ }) - if (!enabled) + # bail if we're not enabled + if (!enabled) { + + # if we're not enabled, we might still need to manually load + # the user profile here + profile <- Sys.getenv("R_PROFILE_USER", unset = "~/.Rprofile") + if (file.exists(profile)) { + cfg <- Sys.getenv("RENV_CONFIG_USER_PROFILE", unset = "TRUE") + if (tolower(cfg) %in% c("true", "t", "1")) + sys.source(profile, envir = globalenv()) + } + return(FALSE) + } + # avoid recursion if (identical(getOption("renv.autoloader.running"), TRUE)) { warning("ignoring recursive attempt to run renv autoloader") @@ -108,6 +131,21 @@ local({ } + heredoc <- function(text, leave = 0) { + + # remove leading, trailing whitespace + trimmed <- gsub("^\\s*\\n|\\n\\s*$", "", text) + + # split into lines + lines <- strsplit(trimmed, "\n", fixed = TRUE)[[1L]] + + # compute common indent + indent <- regexpr("[^[:space:]]", lines) + common <- min(setdiff(indent, -1L)) - leave + paste(substring(lines, common), collapse = "\n") + + } + startswith <- function(string, prefix) { substring(string, 1, nchar(prefix)) == prefix } @@ -610,6 +648,9 @@ local({ # if the user has requested an automatic prefix, generate it auto <- Sys.getenv("RENV_PATHS_PREFIX_AUTO", unset = NA) + if (is.na(auto) && getRversion() >= "4.4.0") + auto <- "TRUE" + if (auto %in% c("TRUE", "True", "true", "1")) return(renv_bootstrap_platform_prefix_auto()) @@ -801,24 +842,23 @@ local({ # the loaded version of renv doesn't match the requested version; # give the user instructions on how to proceed - remote <- if (!is.null(description[["RemoteSha"]])) { + dev <- identical(description[["RemoteType"]], "github") + remote <- if (dev) paste("rstudio/renv", description[["RemoteSha"]], sep = "@") - } else { + else paste("renv", description[["Version"]], sep = "@") - } # display both loaded version + sha if available friendly <- renv_bootstrap_version_friendly( version = description[["Version"]], - sha = description[["RemoteSha"]] + sha = if (dev) description[["RemoteSha"]] ) - fmt <- paste( - "renv %1$s was loaded from project library, but this project is configured to use renv %2$s.", - "- Use `renv::record(\"%3$s\")` to record renv %1$s in the lockfile.", - "- Use `renv::restore(packages = \"renv\")` to install renv %2$s into the project library.", - sep = "\n" - ) + fmt <- heredoc(" + renv %1$s was loaded from project library, but this project is configured to use renv %2$s. + - Use `renv::record(\"%3$s\")` to record renv %1$s in the lockfile. + - Use `renv::restore(packages = \"renv\")` to install renv %2$s into the project library. + ") catf(fmt, friendly, renv_bootstrap_version_friendly(version), remote) FALSE @@ -1041,7 +1081,7 @@ local({ # if jsonlite is loaded, use that instead if ("jsonlite" %in% loadedNamespaces()) { - json <- catch(renv_json_read_jsonlite(file, text)) + json <- tryCatch(renv_json_read_jsonlite(file, text), error = identity) if (!inherits(json, "error")) return(json) @@ -1050,7 +1090,7 @@ local({ } # otherwise, fall back to the default JSON reader - json <- catch(renv_json_read_default(file, text)) + json <- tryCatch(renv_json_read_default(file, text), error = identity) if (!inherits(json, "error")) return(json) @@ -1063,14 +1103,14 @@ local({ } renv_json_read_jsonlite <- function(file = NULL, text = NULL) { - text <- paste(text %||% read(file), collapse = "\n") + text <- paste(text %||% readLines(file, warn = FALSE), collapse = "\n") jsonlite::fromJSON(txt = text, simplifyVector = FALSE) } renv_json_read_default <- function(file = NULL, text = NULL) { # find strings in the JSON - text <- paste(text %||% read(file), collapse = "\n") + text <- paste(text %||% readLines(file, warn = FALSE), collapse = "\n") pattern <- '["](?:(?:\\\\.)|(?:[^"\\\\]))*?["]' locs <- gregexpr(pattern, text, perl = TRUE)[[1]] @@ -1118,14 +1158,14 @@ local({ map <- as.list(map) # remap strings in object - remapped <- renv_json_remap(json, map) + remapped <- renv_json_read_remap(json, map) # evaluate eval(remapped, envir = baseenv()) } - renv_json_remap <- function(json, map) { + renv_json_read_remap <- function(json, map) { # fix names if (!is.null(names(json))) { @@ -1152,7 +1192,7 @@ local({ # recurse if (is.recursive(json)) { for (i in seq_along(json)) { - json[i] <- list(renv_json_remap(json[[i]], map)) + json[i] <- list(renv_json_read_remap(json[[i]], map)) } } From 7abf5690fb0eb0bf8e84fadce6518c4bdd0f32e0 Mon Sep 17 00:00:00 2001 From: Sam Albers Date: Fri, 28 Jun 2024 13:35:15 -0700 Subject: [PATCH 7/7] bump quarto --- .github/workflows/performance-release-report.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/performance-release-report.yml b/.github/workflows/performance-release-report.yml index dd48dda749c..060efaa599c 100644 --- a/.github/workflows/performance-release-report.yml +++ b/.github/workflows/performance-release-report.yml @@ -47,7 +47,7 @@ jobs: - name: Setup Quarto uses: quarto-dev/quarto-actions/setup@v2 with: - version: '1.4.549' + version: '1.4.557' - name: Install libcurl on ubuntu shell: bash