Qualcomm AI Engine Direct - Use AIHub's context binary file for Stabl…

…e Diffusion (#4836)

Summary:
- Add script for the export and runtime of AIHUB Stable Diffusion.
- Add AIHUB Stable Diffusion runner
- Add README tutorial

backends/qualcomm/tests/test_qnn_delegate.py

@@ -1998,6 +1998,55 @@ def test_llama3_8b(self):
                 model_out = msg["result"]
                 self.assertTrue(model_out.startswith(prompt))
 
+    def test_stable_diffusion(self):
+        if not self.required_envs():
+            self.skipTest("missing required envs")
+
+        prompt = "a photo of an astronaut riding a horse on mars"
+        cmds = [
+            "python",
+            f"{self.executorch_root}/examples/qualcomm/qaihub_scripts/stable_diffusion/qaihub_stable_diffusion.py",
+            "--artifact",
+            self.artifact_dir,
+            "--build_folder",
+            self.build_folder,
+            "--device",
+            self.device,
+            "--model",
+            self.model,
+            "--text_encoder_bin",
+            f"{self.artifact_dir}/text_encoder.serialized.bin",
+            "--unet_bin",
+            f"{self.artifact_dir}/unet.serialized.bin",
+            "--vae_bin",
+            f"{self.artifact_dir}/vae.serialized.bin",
+            "--vocab_json",
+            f"{self.artifact_dir}/vocab.json",
+            "--num_time_steps",
+            "20",
+            "--ip",
+            self.ip,
+            "--port",
+            str(self.port),
+            "--prompt",
+            f"{prompt}",
+            "--fix_latents",
+        ]
+        if self.host:
+            cmds.extend(["--host", self.host])
+
+        p = subprocess.Popen(cmds, stdout=subprocess.DEVNULL)
+        with Listener((self.ip, self.port)) as listener:
+            conn = listener.accept()
+            p.communicate()
+            msg = json.loads(conn.recv())
+            if "Error" in msg:
+                self.fail(msg["Error"])
+            else:
+                # For the default settings and prompt, the expected results will be {PSNR: 23.258, SSIM: 0.852}
+                self.assertGreaterEqual(msg["PSNR"], 20)
+                self.assertGreaterEqual(msg["SSIM"], 0.8)
+
 
 class TestExampleScript(TestQNN):
     def required_envs(self, conditions=None) -> bool:

examples/qualcomm/CMakeLists.txt

@@ -81,3 +81,8 @@ add_subdirectory(
 add_subdirectory(
   ${CMAKE_CURRENT_SOURCE_DIR}/qaihub_scripts/llama
 )
+
+# build qaihub_stable_diffusion_runner
+add_subdirectory(
+  ${CMAKE_CURRENT_SOURCE_DIR}/qaihub_scripts/stable_diffusion
+)

examples/qualcomm/qaihub_scripts/stable_diffusion/CMakeLists.txt

@@ -0,0 +1,26 @@
+# Copyright (c) Qualcomm Innovation Center, Inc.
+# All rights reserved
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# preprocess qaihub_stable_diffusion_runner_src files
+set(_qaihub_stable_diffusion_runner__srcs
+  ${CMAKE_CURRENT_LIST_DIR}/qaihub_stable_diffusion_runner.cpp
+  ${CMAKE_CURRENT_LIST_DIR}/runner/runner.cpp
+  ${CMAKE_CURRENT_LIST_DIR}/runner/runner.h
+)
+
+# build qaihub_stable_diffusion_runner
+add_executable(qaihub_stable_diffusion_runner ${_qaihub_stable_diffusion_runner__srcs})
+target_include_directories(qaihub_stable_diffusion_runner
+  PUBLIC ${_common_include_directories}
+)
+target_link_libraries(qaihub_stable_diffusion_runner
+  qnn_executorch_backend
+  executorch_no_prim_ops
+  extension_data_loader
+  extension_module
+  gflags
+)
+target_compile_options(qaihub_stable_diffusion_runner PUBLIC ${_common_compile_options})

examples/qualcomm/qaihub_scripts/stable_diffusion/README.md

@@ -0,0 +1,35 @@
+# Summary
+
+## Overview
+This file provides you the instructions to run Stable-Diffusion-v2.1 with different parameters via Qualcomm HTP backend. We will demonstrate how to run Stable Diffusion v2.1 on mobile devices using context binaries from Qualcomm AI Hub’s Stable Diffusion v2.1
+
+Please check corresponding section for more information.
+
+## Stable-Diffusion-v2.1
+The model architecture, scheduler, and time embedding are from the [stabilityai/stable-diffusion-2-1-base](https://huggingface.co/stabilityai/stable-diffusion-2-1-base).
+
+### Instructions
+#### Step 1: Setup
+1. Follow the [tutorial](https://pytorch.org/executorch/main/getting-started-setup) to set up ExecuTorch.
+2. Follow the [tutorial](https://pytorch.org/executorch/stable/build-run-qualcomm-ai-engine-direct-backend.html) to build Qualcomm AI Engine Direct Backend.
+
+#### Step2: Prepare Model
+1. Download the context binaries for TextEncoder, UNet, and VAEDecoder under https://huggingface.co/qualcomm/Stable-Diffusion-v2.1/tree/main
+2. Download vocab.json under https://huggingface.co/openai/clip-vit-base-patch32/tree/main
+
+
+#### Step3: Install Requirements
+Before running the code, you need to install the necessary Python packages.
+
+We have verified the code with `diffusers`==0.29.0 and `piq`==0.8.0. Please follow the instructions here to install the required items:
+```bash
+sh examples/qualcomm/qaihub_scripts/stable_diffusion/install_requirements.sh
+```
+
+#### Step4: Run default example
+In this example, we execute the script for 20 time steps with the `prompt` 'a photo of an astronaut riding a horse on mars':
+```bash
+python examples/qualcomm/qaihub_scripts/stable_diffusion/qaihub_stable_diffusion.py -a ${ARTIFACTS} -b build_android -m ${SOC_MODEL} --s ${SERIAL_NUM} --text_encoder_bin ${PATH_TO_TEXT_ENCODER_CONTEXT_BINARY} --unet_bin ${PATH_TO_UNET_CONTEXT_BINARY} --vae_bin ${PATH_TO_VAE_CONTEXT_BINARY} --vocab_json  ${PATH_TO_VOCAB_JSON_FILE} --num_time_steps 20 --prompt "a photo of an astronaut riding a horse on mars"
+```
+- Please replace `${PATH_TO_TEXT_ENCODER_CONTEXT_BINARY}`, `${PATH_TO_UNET_CONTEXT_BINARY}`, and `${PATH_TO_VAE_CONTEXT_BINARY}` with the actual paths to your AI Hub context binary files.
+- Please replace `${PATH_TO_VOCAB_JSON_FILE}` with the actual path to your vocab.json file.

examples/qualcomm/qaihub_scripts/stable_diffusion/install_requirements.sh

@@ -0,0 +1,3 @@
+# For Stable Diffusion V2.1
+pip install diffusers==0.29.0
+pip install piq==0.8.0