refactor: use custom setting in Core ML backend to detect NCHW input. (…

…#924) * Add GetConfigValue() * Add custom setting data-format for Core ML * Use GetConfigValue() to get stable_diffusion_seed and stable_diffusion_num_steps
mlcommons · Nov 12, 2024 · 1bddf37 · 1bddf37
1 parent fad36c1
commit 1bddf37
Show file tree

Hide file tree

Showing 8 changed files with 94 additions and 17 deletions.
diff --git a/flutter/cpp/utils.cc b/flutter/cpp/utils.cc
@@ -186,5 +186,45 @@ SettingList CreateSettingList(const BackendSetting &backend_setting,
   return setting_list;
 }
 
+template <typename T>
+T GetConfigValue(mlperf_backend_configuration_t *configs, const char *key,
+                 T defaultValue);
+
+template <>
+int GetConfigValue<int>(mlperf_backend_configuration_t *configs,
+                        const char *key, int defaultValue) {
+  for (int i = 0; i < configs->count; ++i) {
+    if (strcmp(configs->keys[i], key) == 0) {
+      const char *valueStr = configs->values[i];
+      char *endptr = nullptr;
+      errno = 0;
+      long value =
+          strtol(valueStr, &endptr, 10);  // Base 10 for decimal conversion
+      if (errno == ERANGE || value < INT_MIN || value > INT_MAX) {
+        LOG(ERROR) << "Value out of range for int: " << valueStr;
+        return defaultValue;
+      }
+      if (endptr == valueStr || *endptr != '\0') {
+        LOG(ERROR) << "Invalid value for int: " << valueStr;
+        return defaultValue;
+      }
+      return static_cast<int>(value);
+    }
+  }
+  return defaultValue;
+}
+
+template <>
+std::string GetConfigValue<std::string>(mlperf_backend_configuration_t *configs,
+                                        const char *key,
+                                        std::string defaultValue) {
+  for (int i = 0; i < configs->count; ++i) {
+    if (strcmp(configs->keys[i], key) == 0) {
+      return std::string(configs->values[i]);
+    }
+  }
+  return defaultValue;
+}
+
 }  // namespace mobile
 }  // namespace mlperf
diff --git a/flutter/cpp/utils.h b/flutter/cpp/utils.h
@@ -69,6 +69,10 @@ SettingList CreateSettingList(const BackendSetting &backend_setting,
                               const std::string &custom_config,
                               const std::string &benchmark_id);
 
+template <typename T>
+T GetConfigValue(mlperf_backend_configuration_t *configs, const char *key,
+                 T defaultValue);
+
 }  // namespace mobile
 }  // namespace mlperf
 

diff --git a/mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt b/mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt
@@ -22,7 +22,7 @@ benchmark_setting {
       model_checksum: "39483b20b878d46144ab4cfe9a3e5600"
     }
   }
-   delegate_choice: {
+  delegate_choice: {
     delegate_name: "CPU & ANE"
     accelerator_name: "cpu&ane"
     accelerator_desc: "CPU and Neural Engine"
@@ -57,7 +57,7 @@ benchmark_setting {
       model_checksum: "39483b20b878d46144ab4cfe9a3e5600"
     }
   }
-   delegate_choice: {
+  delegate_choice: {
     delegate_name: "CPU & ANE"
     accelerator_name: "cpu&ane"
     accelerator_desc: "CPU and Neural Engine"
@@ -81,6 +81,10 @@ benchmark_setting {
       model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.1-coreml/mobilenetv4_fp32_NCHW.mlpackage.zip"
       model_checksum: "164c504eb3e9af6c730c1765b8b81b32"
     }
+    custom_setting {
+      id: "data-format"
+      value: "NCHW"
+    }
   }
   delegate_choice: {
     delegate_name: "CPU & GPU"
@@ -90,6 +94,10 @@ benchmark_setting {
       model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.1-coreml/mobilenetv4_fp32_NCHW.mlpackage.zip"
       model_checksum: "164c504eb3e9af6c730c1765b8b81b32"
     }
+    custom_setting {
+      id: "data-format"
+      value: "NCHW"
+    }
   }
   delegate_choice: {
     delegate_name: "CPU & ANE"
@@ -99,6 +107,10 @@ benchmark_setting {
       model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.1-coreml/mobilenetv4_fp32_NCHW.mlpackage.zip"
       model_checksum: "164c504eb3e9af6c730c1765b8b81b32"
     }
+    custom_setting {
+      id: "data-format"
+      value: "NCHW"
+    }
   }
   delegate_selected: "CPU & GPU & ANE"
 }
@@ -115,6 +127,10 @@ benchmark_setting {
       model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.1-coreml/mobilenetv4_fp32_NCHW.mlpackage.zip"
       model_checksum: "164c504eb3e9af6c730c1765b8b81b32"
     }
+    custom_setting {
+      id: "data-format"
+      value: "NCHW"
+    }
   }
   delegate_choice: {
     delegate_name: "CPU & GPU"
@@ -125,6 +141,10 @@ benchmark_setting {
       model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.1-coreml/mobilenetv4_fp32_NCHW.mlpackage.zip"
       model_checksum: "164c504eb3e9af6c730c1765b8b81b32"
     }
+    custom_setting {
+      id: "data-format"
+      value: "NCHW"
+    }
   }
   delegate_choice: {
     delegate_name: "CPU & ANE"
@@ -135,6 +155,10 @@ benchmark_setting {
       model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.1-coreml/mobilenetv4_fp32_NCHW.mlpackage.zip"
       model_checksum: "164c504eb3e9af6c730c1765b8b81b32"
     }
+    custom_setting {
+      id: "data-format"
+      value: "NCHW"
+    }
   }
   delegate_selected: "CPU & GPU & ANE"
 }
@@ -160,7 +184,7 @@ benchmark_setting {
       model_checksum: "ef849fbf2132e205158f05ca42db25f4"
     }
   }
-   delegate_choice: {
+  delegate_choice: {
     delegate_name: "CPU & ANE"
     accelerator_name: "cpu&ane"
     accelerator_desc: "CPU and Neural Engine"
@@ -217,7 +241,7 @@ benchmark_setting {
       model_checksum: "362d6b5bb1b8e10ae5b4e223f60d4d10"
     }
   }
-   delegate_choice: {
+  delegate_choice: {
     delegate_name: "CPU & ANE"
     accelerator_name: "cpu&ane"
     accelerator_desc: "CPU and Neural Engine"
@@ -250,7 +274,7 @@ benchmark_setting {
       model_checksum: "62489706f20b0c2ae561fb2204eefb61"
     }
   }
-   delegate_choice: {
+  delegate_choice: {
     delegate_name: "CPU & ANE"
     accelerator_name: "cpu&ane"
     accelerator_desc: "CPU and Neural Engine"

diff --git a/mobile_back_apple/cpp/backend_coreml/main.cc b/mobile_back_apple/cpp/backend_coreml/main.cc
@@ -101,8 +101,9 @@ mlperf_backend_ptr_t mlperf_backend_create(
 
   CoreMLBackendData *backend_data = new CoreMLBackendData();
   backendExists = true;
-  // quick hack for checking if model expects NCHW input.
-  if (strcasestr(model_path, "NCHW") != nullptr) {
+  std::string dataFormat =
+      mlperf::mobile::GetConfigValue(configs, "data-format", std::string(""));
+  if (dataFormat == "NCHW") {
     backend_data->expectNCHW = true;
     LOG(INFO) << "Will convert inputs from NHWC to NCHW!";
   }

diff --git a/mobile_back_apple/dev-utils/Makefile b/mobile_back_apple/dev-utils/Makefile
@@ -24,7 +24,7 @@ app:
 
 tflite-build:
 	cd ${REPO_ROOT_DIR} && \
-	bazel build -c opt --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 --macos_minimum_os=12.1 \
+	bazel build -c opt --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 --macos_minimum_os=13.1 \
 	//flutter/cpp/binary:main //mobile_back_tflite:tflitebackend
 
 tflite-run-ic:
@@ -141,7 +141,7 @@ tflite-run-sd:
 
 coreml-build:
 	cd ${REPO_ROOT_DIR} && \
-	bazel build -c opt --cxxopt=-fobjc-arc --cxxopt=-xobjective-c++ --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 --macos_minimum_os=12.1 \
+	bazel build -c opt --cxxopt=-fobjc-arc --cxxopt=-xobjective-c++ --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 --macos_minimum_os=13.1 \
 	//flutter/cpp/binary:main //mobile_back_apple:coremlbackend
 
 coreml-run-ic:

diff --git a/mobile_back_tflite/cpp/backend_tflite/BUILD b/mobile_back_tflite/cpp/backend_tflite/BUILD
@@ -68,6 +68,7 @@ cc_library(
     }),
     deps = [
         ":tflite_settings",
+        "//flutter/cpp:utils",
         "//flutter/cpp/c:headers",
         "@org_tensorflow//tensorflow/core:tflite_portable_logging",
         "@org_tensorflow//tensorflow/lite/c:c_api",

diff --git a/mobile_back_tflite/cpp/backend_tflite/neuron/BUILD b/mobile_back_tflite/cpp/backend_tflite/neuron/BUILD
@@ -70,6 +70,7 @@ cc_library(
     local_defines = ["MTK_TFLITE_NEURON_BACKEND"],
     deps = [
         ":tflite_settings",
+        "//flutter/cpp:utils",
         "//flutter/cpp/c:headers",
         "//mobile_back_tflite/cpp/backend_tflite:tflite_settings",
         "@org_tensorflow//tensorflow/core:tflite_portable_logging",

diff --git a/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc b/mobile_back_tflite/cpp/backend_tflite/stable_diffusion_pipeline.cc
@@ -7,6 +7,7 @@
 #include <valarray>
 
 #include "flutter/cpp/c/backend_c.h"
+#include "flutter/cpp/utils.h"
 #include "stable_diffusion_invoker.h"
 #include "tensorflow/lite/c/c_api.h"
 #include "tensorflow/lite/c/common.h"
@@ -58,21 +59,26 @@ mlperf_backend_ptr_t StableDiffusionPipeline::backend_create(
 
   // Verify only one instance of the backend exists at any time
   if (backendExists) {
+    LOG(ERROR) << "Backend already exists";
     return nullptr;
   }
 
   SDBackendData* backend_data = new SDBackendData();
   backendExists = true;
 
-  for (int i = 0; i < configs->count; ++i) {
-    if (strcmp(configs->keys[i], "stable_diffusion_seed") == 0) {
-      backend_data->seed = atoi(configs->values[i]);
-    }
-    if (strcmp(configs->keys[i], "stable_diffusion_num_steps") == 0) {
-      backend_data->num_steps = atoi(configs->values[i]);
-    }
+  // Read seed and num_steps value from SD task settings
+  backend_data->seed =
+      mlperf::mobile::GetConfigValue(configs, "stable_diffusion_seed", 0);
+  if (backend_data->seed == 0) {
+    LOG(ERROR) << "Cannot get stable_diffusion_seed";
+    return nullptr;
+  }
+  backend_data->num_steps =
+      mlperf::mobile::GetConfigValue(configs, "stable_diffusion_num_steps", 0);
+  if (backend_data->num_steps == 0) {
+    LOG(ERROR) << "Cannot get stable_diffusion_num_steps";
+    return nullptr;
   }
-
   // Load models from the provided directory path
   std::string text_encoder_path =
       std::string(model_path) + "/sd_text_encoder_dynamic.tflite";