From 9c88e9e5373ef555db2f43d3ec346d9b102448eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rasmus=20Skytte=20Randl=C3=B8v?= Date: Thu, 18 Apr 2024 09:37:52 +0200 Subject: [PATCH] feat(benchmark): Swap to bench::mark() to capture memory also --- .github/workflows/benchmark.yaml | 10 +++++----- DESCRIPTION | 2 +- data-raw/benchmark.R | 4 ++-- vignettes/benchmarks.Rmd | 19 ++++++++++--------- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 86b256c1..387b51d5 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -62,7 +62,7 @@ jobs: - name: Delete previous benchmark files if: always() - run: rm -rf inst/extdata/benchmark-*.rds + run: rm -rf inst/extdata/benchmark*.rds @@ -156,8 +156,7 @@ jobs: benchmarks <- benchmark_files |> purrr::map(readRDS) |> - purrr::map(tibble::as_tibble) |> - purrr::reduce(rbind) + purrr::list_rbind() benchmarks <- benchmarks |> dplyr::mutate( @@ -180,7 +179,8 @@ jobs: dplyr::pull("database") benchmarks <- benchmarks |> - dplyr::mutate("database" = paste0(database, ifelse(database %in% slow_backends, "*", ""))) + dplyr::mutate("database" = paste0(database, ifelse(database %in% slow_backends, "*", ""))) |> + tidyr::unnest(c(time, gc)) # Mean and standard deviation (see ggplot2::mean_se()) @@ -192,7 +192,7 @@ jobs: g <- ggplot2::ggplot( benchmarks, - ggplot2::aes(x = version, y = time / 1e9) + ggplot2::aes(x = version, y = time) ) + ggplot2::stat_summary(fun.data = mean_sd, geom = "pointrange", size = 0.5, linewidth = 1) + ggplot2::facet_grid(rows = ggplot2::vars(benchmark_function), cols = ggplot2::vars(database)) + diff --git a/DESCRIPTION b/DESCRIPTION index 2c8d13eb..e3e10bda 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -39,6 +39,7 @@ Imports: tidyselect, utils Suggests: + bench, callr, conflicted, duckdb, @@ -46,7 +47,6 @@ Suggests: jsonlite, knitr, lintr, - microbenchmark, odbc, rmarkdown, roxygen2, diff --git a/data-raw/benchmark.R b/data-raw/benchmark.R index cc456ab7..5350eccf 100644 --- a/data-raw/benchmark.R +++ b/data-raw/benchmark.R @@ -118,7 +118,7 @@ if (identical(Sys.getenv("CI"), "true") && identical(Sys.getenv("BACKEND"), "")) } # Construct the list of benchmarks - update_snapshot_benchmark <- microbenchmark::microbenchmark(scdb_updates(conn, data_on_conn), times = 25) |> + update_snapshot_benchmark <- bench::mark(scdb_updates(conn, data_on_conn), iterations = 25) |> dplyr::mutate( "benchmark_function" = "update_snapshot()", "database" = names(conns)[[1]], @@ -145,7 +145,7 @@ if (identical(Sys.getenv("CI"), "true") && identical(Sys.getenv("BACKEND"), "")) } # Construct the list of benchmarks - update_snapshot_benchmark <- microbenchmark::microbenchmark(scdb_updates(conn, data), times = 5) |> + update_snapshot_benchmark <- bench::mark(scdb_updates(conn, data), iterations = 5) |> dplyr::mutate( "benchmark_function" = "update_snapshot() - complexity", "database" = names(conns)[[1]], diff --git a/vignettes/benchmarks.Rmd b/vignettes/benchmarks.Rmd index 5e437754..37026d8c 100644 --- a/vignettes/benchmarks.Rmd +++ b/vignettes/benchmarks.Rmd @@ -31,7 +31,7 @@ This data forms the basis for three "snapshots" used in the benchmarks: The benchmark function uses three consecutive calls to `update_snapshot()` to create the table with first snapshot and then update it to the second and third snapshot. Finally, the table is deleted. -The performance of this benchmark function is timed with the `microbenchmark` package using 10 replicates. +The performance of this benchmark function is timed with the `bench` package using 10 replicates. All benchmarks are run on the same machine. The results of the benchmark are shown graphically below (mean and standard deviation), where we compare the current @@ -45,7 +45,8 @@ benchmark_location <- c( purrr::discard(~ identical(., "")) |> purrr::pluck(1) -benchmarks <- readRDS(benchmark_location) +benchmarks <- readRDS(benchmark_location) |> + tidyr::unnest(c(time, gc)) # Determine if the SHA is on main sha <- benchmarks |> @@ -82,7 +83,7 @@ mean_sd <- function(x) { ```{r benchmark_1, echo = FALSE, eval = requireNamespace("here")} # Use data for benchmark 1 benchmark_1 <- benchmarks |> - dplyr::filter(!stringr::str_ends(.data$benchmark_function, "complexity")) + dplyr::filter(!stringr::str_ends(.data$benchmark_function, stringr::fixed("complexity"))) # Add note slow backends slow_backends <- benchmark_1 |> @@ -96,7 +97,7 @@ benchmark_1 <- benchmark_1 |> g <- ggplot2::ggplot( benchmark_1, - ggplot2::aes(x = version, y = time / 1e9) + ggplot2::aes(x = version, y = time) ) + ggplot2::stat_summary(fun.data = mean_sd, geom = "pointrange", size = 0.5, linewidth = 1) + ggplot2::facet_grid(rows = ggplot2::vars(benchmark_function), cols = ggplot2::vars(database)) + @@ -111,27 +112,27 @@ g We include another benchmark to highlight the complexity scaling of the `update_snapshot() ` with the size of the input -data. The datasets are similar to the first benchmark is used, but the number of repeats is varied to see the impact of +data. The data sets are similar to the first benchmark is used, but the number of repeats is varied to see the impact of increasing data size. The benchmarks are run from a "clean" state, where the target_table does not exists. The benchmark measures both the time to create the table and to remove it again afterwards (to restore the clean state). -The performance of this benchmark function is timed with the `microbenchmark` package using 5 replicates. +The performance of this benchmark function is timed with the `bench` package using 5 replicates. All benchmarks are run on the same machine. The results of the benchmark are shown graphically below (mean and standard deviation), where we compare the current development version of `SCDB` with the current CRAN version. -NOTE: There are reports of a superlinear complexity for very large data sets. If you experience such problems, consider +NOTE: There are reports of a super-linear complexity for very large data sets. If you experience such problems, consider batching the updates via the `filters` argument. ```{r benchmark_2, echo = FALSE, eval = requireNamespace("here")} # Use data for benchmark 2 benchmark_2 <- benchmarks |> - dplyr::filter(stringr::str_ends(.data$benchmark_function, "complexity")) + dplyr::filter(stringr::str_ends(.data$benchmark_function, stringr::fixed("complexity"))) ggplot2::ggplot( benchmark_2, - ggplot2::aes(x = n * nrow(iris) / 1e3, y = time / 1e9, color = version) + ggplot2::aes(x = n * nrow(iris) / 1e3, y = time, color = version) ) + ggplot2::stat_summary(fun.data = mean_sd, geom = "pointrange", size = 0.5, linewidth = 1) + ggplot2::facet_grid(rows = ggplot2::vars(benchmark_function), cols = ggplot2::vars(database)) +