Bug 1807324 - Add macrobenchmarks for startup metrics

fenix/.buildconfig.yml

@@ -135,6 +135,17 @@ variants:
     fileName: app-fenix-x86_64-beta-unsigned.apk
   build_type: beta
   name: fenixBeta
+- apks:
+  - abi: arm64-v8a
+    fileName: app-fenix-arm64-v8a-benchmark-unsigned.apk
+  - abi: armeabi-v7a
+    fileName: app-fenix-armeabi-v7a-benchmark-unsigned.apk
+  - abi: x86
+    fileName: app-fenix-x86-benchmark-unsigned.apk
+  - abi: x86_64
+    fileName: app-fenix-x86_64-benchmark-unsigned.apk
+  build_type: benchmark
+  name: fenixBenchmark
 - apks:
   - abi: noarch
     fileName: app-debug-androidTest.apk

fenix/app/build.gradle

@@ -11,8 +11,6 @@ apply plugin: 'kotlin-parcelize'
 apply plugin: 'jacoco'
 apply plugin: 'androidx.navigation.safeargs.kotlin'
 apply plugin: 'com.google.android.gms.oss-licenses-plugin'
-apply plugin: 'androidx.benchmark'
-
 
 import com.android.build.OutputFile
 import groovy.json.JsonOutput
@@ -148,6 +146,11 @@ android {
                     "deepLinkScheme": deepLinkSchemeValue
             ]
         }
+        benchmark releaseTemplate >> {
+            initWith buildTypes.nightly
+            applicationIdSuffix ".fenix"
+            debuggable false
+        }
     }
 
     buildFeatures {
@@ -447,6 +450,7 @@ nimbus {
             fenixNightly: "nightly",
             fenixBeta: "beta",
             fenixRelease: "release",
+            fenixBenchmark: "developer",
     ]
     // This is generated by the FML and should be checked into git.
     // It will be fetched by Experimenter (the Nimbus experiment website)
@@ -620,6 +624,8 @@ dependencies {
     implementation FenixDependencies.google_play_review
     implementation FenixDependencies.google_play_review_ktx
 
+    implementation FenixDependencies.androidx_profileinstaller
+
     androidTestImplementation FenixDependencies.uiautomator
     androidTestImplementation "tools.fastlane:screengrab:2.0.0"
     // This Falcon version is added to maven central now required for Screengrab

fenix/app/src/main/AndroidManifest.xml

@@ -48,6 +48,10 @@
         android:usesCleartextTraffic="true"
         tools:ignore="UnusedAttribute">
 
+        <profileable
+            android:shell="true"
+            tools:targetApi="29" />
+
         <!--
           We inherited this entry (${applicationId}.App) from Fennec. We need to keep this as our
           main launcher to avoid launcher icons on the home screen disappearing for all our users.

fenix/app/src/main/java/org/mozilla/fenix/Config.kt

@@ -53,7 +53,7 @@ enum class ReleaseChannel {
 object Config {
     val channel = when (BuildConfig.BUILD_TYPE) {
         "debug" -> ReleaseChannel.Debug
-        "nightly" -> ReleaseChannel.Nightly
+        "nightly", "benchmark" -> ReleaseChannel.Nightly
         "beta" -> ReleaseChannel.Beta
         "release" -> ReleaseChannel.Release
         else -> {

fenix/benchmark/build.gradle

@@ -0,0 +1,70 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+plugins {
+    id 'com.android.test'
+    id 'org.jetbrains.kotlin.android'
+}
+
+android {
+    namespace 'org.mozilla.fenix.benchmark'
+    compileSdk Config.compileSdkVersion
+
+    compileOptions {
+        sourceCompatibility = JavaVersion.VERSION_11
+        targetCompatibility = JavaVersion.VERSION_11
+    }
+
+    kotlinOptions {
+        jvmTarget = "11"
+    }
+
+    defaultConfig {
+        minSdk 23
+        targetSdk Config.targetSdkVersion
+        testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
+    }
+
+    buildTypes {
+        // This benchmark buildType is used for benchmarking, and should function like your
+        // release build (for example, with minification on). It's signed with a debug key
+        // for easy local testing.
+        benchmark {
+            debuggable = true
+            signingConfig signingConfigs.debug
+            matchingFallbacks = ["release"]
+        }
+    }
+
+    targetProjectPath = ":app"
+    experimentalProperties["android.experimental.self-instrumenting"] = true
+}
+
+/**
+ * This fixes the dependency resolution issue with Glean Native. The glean gradle plugin does this
+ * and that's applied to the app module. Since there are no other uses of the glean plugin in the
+ * benchmark module, we do this manually here.
+ */
+configurations.all {
+    resolutionStrategy.capabilitiesResolution.withCapability("org.mozilla.telemetry:glean-native") {
+        def toBeSelected = candidates.find { it.id instanceof ModuleComponentIdentifier && it.id.module.contains('geckoview') }
+        if (toBeSelected != null) {
+            select(toBeSelected)
+        }
+        because 'use GeckoView Glean instead of standalone Glean'
+    }
+}
+
+dependencies {
+    implementation FenixDependencies.androidx_junit
+    implementation FenixDependencies.espresso_core
+    implementation FenixDependencies.uiautomator
+    implementation FenixDependencies.androidx_benchmark_macro_junit4
+}
+
+androidComponents {
+    beforeVariants(selector().all()) {
+        enabled = buildType == "benchmark"
+    }
+}

fenix/benchmark/src/main/AndroidManifest.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+
+    <queries>
+        <package android:name="org.mozilla.fenix" />
+    </queries>
+</manifest>

fenix/benchmark/src/main/java/org/mozilla/fenix/benchmark/StartupBenchmark.kt

@@ -0,0 +1,49 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.fenix.benchmark
+
+import androidx.benchmark.macro.StartupMode
+import androidx.benchmark.macro.StartupTimingMetric
+import androidx.benchmark.macro.junit4.MacrobenchmarkRule
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import org.junit.Rule
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.mozilla.fenix.benchmark.utils.measureRepeatedDefault
+
+/**
+ * This is a startup benchmark.
+ * It navigates to the device's home screen, and launches the default activity.
+ *
+ * Before running this benchmark,
+ * switch your app's active build variant in the Studio (affects Studio runs only)
+ *
+ * Run this benchmark from Studio to see startup measurements, and captured system traces
+ * for investigating your app's performance.
+ */
+@RunWith(AndroidJUnit4::class)
+class StartupBenchmark {
+    @get:Rule
+    val benchmarkRule = MacrobenchmarkRule()
+
+    @Test
+    fun startupCold() = startupBenchmark(StartupMode.COLD)
+
+    @Test
+    fun startupWarm() = startupBenchmark(StartupMode.WARM)
+
+    @Test
+    fun startupHot() = startupBenchmark(StartupMode.HOT)
+
+    private fun startupBenchmark(startupMode: StartupMode) = benchmarkRule.measureRepeatedDefault(
+        metrics = listOf(StartupTimingMetric()),
+        startupMode = startupMode,
+        setupBlock = {
+            pressHome()
+        }
+    ) {
+        startActivityAndWait()
+    }
+}

fenix/benchmark/src/main/java/org/mozilla/fenix/benchmark/utils/Constants.kt

@@ -0,0 +1,8 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.fenix.benchmark.utils
+
+const val TARGET_PACKAGE = "org.mozilla.fenix"
+const val DEFAULT_ITERATIONS = 5

fenix/benchmark/src/main/java/org/mozilla/fenix/benchmark/utils/MacroBenchmarkRule.kt

@@ -0,0 +1,37 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.fenix.benchmark.utils
+
+import androidx.annotation.IntRange
+import androidx.benchmark.macro.CompilationMode
+import androidx.benchmark.macro.MacrobenchmarkScope
+import androidx.benchmark.macro.Metric
+import androidx.benchmark.macro.StartupMode
+import androidx.benchmark.macro.junit4.MacrobenchmarkRule
+
+/**
+ * Extension function that calls [MacrobenchmarkRule.measureRepeated] with
+ * defaults parameters set for [packageName] and [iterations].
+ */
+fun MacrobenchmarkRule.measureRepeatedDefault(
+    packageName: String = TARGET_PACKAGE,
+    metrics: List<Metric>,
+    compilationMode: CompilationMode = CompilationMode.DEFAULT,
+    startupMode: StartupMode? = null,
+    @IntRange(from = 1)
+    iterations: Int = DEFAULT_ITERATIONS,
+    setupBlock: MacrobenchmarkScope.() -> Unit = {},
+    measureBlock: MacrobenchmarkScope.() -> Unit,
+) {
+    measureRepeated(
+        packageName = packageName,
+        metrics = metrics,
+        compilationMode = compilationMode,
+        startupMode = startupMode,
+        iterations = iterations,
+        setupBlock = setupBlock,
+        measureBlock = measureBlock,
+    )
+}

fenix/buildSrc/src/main/java/FenixDependencies.kt

@@ -23,11 +23,12 @@ object FenixVersions {
     const val androidx_compose = "1.3.1"
     const val androidx_compose_compiler = "1.4.3"
     const val androidx_appcompat = "1.3.0"
-    const val androidx_benchmark = "1.0.0"
+    const val androidx_benchmark = "1.1.1"
     const val androidx_biometric = "1.1.0"
     const val androidx_coordinator_layout = "1.1.0"
     const val androidx_constraint_layout = "2.0.4"
     const val androidx_preference = "1.1.1"
+    const val androidx_profileinstaller = "1.2.2"
     const val androidx_legacy = "1.0.0"
     const val androidx_annotation = "1.5.0"
     const val androidx_lifecycle = "2.5.1"
@@ -84,6 +85,8 @@ object FenixDependencies {
     const val androidx_compose_material = "androidx.compose.material:material:${FenixVersions.androidx_compose}"
     const val androidx_annotation = "androidx.annotation:annotation:${FenixVersions.androidx_annotation}"
     const val androidx_benchmark_junit4 = "androidx.benchmark:benchmark-junit4:${FenixVersions.androidx_benchmark}"
+    const val androidx_benchmark_macro_junit4 = "androidx.benchmark:benchmark-macro-junit4:${FenixVersions.androidx_benchmark}"
+    const val androidx_profileinstaller = "androidx.profileinstaller:profileinstaller:${FenixVersions.androidx_profileinstaller}"
     const val androidx_biometric = "androidx.biometric:biometric:${FenixVersions.androidx_biometric}"
     const val androidx_fragment = "androidx.fragment:fragment-ktx:${FenixVersions.androidx_fragment}"
     const val androidx_appcompat = "androidx.appcompat:appcompat:${FenixVersions.androidx_appcompat}"

fenix/settings.gradle

@@ -27,6 +27,7 @@ plugins {
 include ':app'
 include ':mozilla-detekt-rules'
 include ':mozilla-lint-rules'
+include ':benchmark'
 
 // Synchronized library configuration for all modules
 // This "componentsVersion" number is defined in "version.txt" and should follow