Skip to content

Commit

Permalink
update umd test with three config Test_P
Browse files Browse the repository at this point in the history
  • Loading branch information
@DanLiu2Intel
DanLiu2Intel committed Nov 11, 2024
1 parent b1797aa commit 1eda1e8
Show file tree
Hide file tree
Showing 2 changed files with 74 additions and 104 deletions.
176 changes: 73 additions & 103 deletions src/plugins/intel_npu/tests/functional/internal/overload/driver_caching.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,13 @@
#include <common_test_utils/test_assertions.hpp>
#include <sstream>

// #include "shared_test_classes/base/ov_subgraph.hpp"s
#include "base/ov_behavior_test_utils.hpp"

#include "intel_npu/npu_private_properties.hpp"
#include "openvino/core/except.hpp"
#include "openvino/opsets/opset8.hpp"

#include "stdio.h" //
#include <stdlib.h>// env setting
#include <stdlib.h>

#include "intel_npu/utils/zero/zero_init.hpp"
#include "intel_npu/utils/zero/zero_utils.hpp"
Expand All @@ -20,13 +18,11 @@
#include "intel_npu/config/runtime.hpp"
#include "intel_npu/config/config.hpp"

// #include "openvino/runtime/properties.hpp" //include by config files
#include "openvino/runtime/intel_npu/properties.hpp"

#include <filesystem>
#include <chrono> // cal time
#include <chrono>

//mkdir folder
#ifdef WIN32
#include "Shlobj.h"
#include "shlobj_core.h"
Expand Down Expand Up @@ -59,50 +55,42 @@ inline std::shared_ptr<ov::Model> getConstantGraph() {
return std::make_shared<Model>(results, params);
}

std::string generateCacheDirName(const std::string& test_name) {
// Generate unique file names based on test name, thread id and timestamp
// This allows execution of tests in parallel (stress mode)
auto hash = std::to_string(std::hash<std::string>()(test_name));
std::stringstream ss;
auto ts = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::high_resolution_clock::now().time_since_epoch());
ss << hash << "_"
<< "_" << ts.count();
return ss.str();
}
void checkSystemCacheDirectory() {
std::filesystem::path path{};
#ifdef WIN32
wchar_t* local = nullptr;
auto result = SHGetKnownFolderPath( FOLDERID_LocalAppData, 0, NULL, &local );

std::map<std::string, std::string> any_copy(const ov::AnyMap& params) {
std::map<std::string, std::string> result;
for (auto&& value : params) {
// The value of cache_encryption_callbacks cannot be converted to std::string
if (value.first == ov::cache_encryption_callbacks.name()) {
continue;
if(SUCCEEDED(result)) {
// prepend to enable long path name support
path = std::filesystem::path( L"\\\\?\\" + std::wstring( local ) + +L"\\Intel\\NPU" );

CoTaskMemFree( local );
}
#else
const char *env = getenv("ZE_INTEL_NPU_CACHE_DIR");
if (env) {
path = std::filesystem::path(env);
} else {
env = getenv("HOME");
if (env) {
path = std::filesystem::path(env) / ".cache/ze_intel_npu_cache";
} else {
path = std::filesystem::current_path() / ".cache/ze_intel_npu_cache";
}
result.emplace(value.first, value.second.as<std::string>());
}
return result;
}

bool containsCacheStatus(const std::string& str) {
return (str.find("cache_status_t::stored") != std::string::npos || str.find("cache_status_t::found") != std::string::npos);
}
#endif

inline std::vector<std::string> listFilesWithExt(const std::string& path) {
struct dirent* ent;
DIR* dir = opendir(path.c_str());
std::vector<std::string> res;
if (dir != nullptr) {
while ((ent = readdir(dir)) != NULL) {
auto file = ov::test::utils::makePath(path, std::string(ent->d_name));
struct stat stat_path;
stat(file.c_str(), &stat_path);
//cache not contian file extension.
if (!S_ISDIR(stat_path.st_mode) && ov::test::utils::endsWith(file, "")) {
res.push_back(std::move(file));
}
if (std::filesystem::exists(path) && std::filesystem::is_directory(path)) {
for (const auto& entry : std::filesystem::directory_iterator(path)) {
std::filesystem::remove_all(entry.path());
}
closedir(dir);
}
return res;

}

bool containsCacheStatus(const std::string& str, const std::string cmpstr) {
return str.find(cmpstr) != std::string::npos;
}

class CompileAndDriverCaching : public testing::WithParamInterface<CompileAndModelCachingParams>,
Expand Down Expand Up @@ -142,18 +130,22 @@ class CompileAndDriverCaching : public testing::WithParamInterface<CompileAndMod
GTEST_SKIP() << "Skipping test for Driver version less than 1.5, current driver version: " << graphDdiExtVersion;
}

checkSystemCacheDirectory();
APIBaseTest::SetUp();
}

void TearDown() override {
if (!m_cache_dir.empty() && !std::filesystem::exists(m_cache_dir)) {
std::filesystem::remove_all(m_cache_dir);
}

if (!configuration.empty()) {
utils::PluginCache::get().reset();
if (!m_cachedir.empty()) {
std::printf(" printf m_cachedir:%s \n", m_cachedir.c_str());
core->set_property({ov::cache_dir()});
core.reset();
ov::test::utils::removeFilesWithExt(m_cachedir, "blob");
ov::test::utils::removeDir(m_cachedir);
}
ov::test::utils::PluginCache::get().reset();
}

checkSystemCacheDirectory();
APIBaseTest::TearDown();
}

Expand All @@ -162,41 +154,14 @@ class CompileAndDriverCaching : public testing::WithParamInterface<CompileAndMod
ov::AnyMap configuration;
std::shared_ptr<ov::Model> function;
std::shared_ptr<::intel_npu::ZeroInitStructsHolder> initStruct;
std::string m_cache_dir;
std::string m_cachedir;
};

TEST_P(CompileAndDriverCaching, CompilationCacheFlag) {
TEST_P(CompileAndDriverCaching, UMDCacheWithEmptyConfig) {
ze_graph_dditable_ext_decorator& graph_ddi_table_ext = initStruct->getGraphDdiTable();

ov::CompiledModel execNet;
//first run time will long and will generate the model cache.
auto startFirst = std::chrono::high_resolution_clock::now();
OV_ASSERT_NO_THROW(execNet = core->compile_model(function, target_device, configuration));
auto endFirst = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> durationFirst = endFirst - startFirst;

std::string driverLogContent = ::intel_npu::zeroUtils::getLatestBuildError(graph_ddi_table_ext);
EXPECT_TRUE(!containsCacheStatus(driverLogContent));

//second time compilation
auto startSecond = std::chrono::high_resolution_clock::now();
OV_ASSERT_NO_THROW(execNet = core->compile_model(function, target_device, configuration));
auto endSecond = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> durationSecond = endSecond - startSecond;

std::string driverLogContent = ::intel_npu::zeroUtils::getLatestBuildError(graph_ddi_table_ext);
if ((configuration.find("CACHE_DIR") != configuration.end()) || configuration.find("NPU_BYPASS_UMD_CACHING") != configuration.end()) {
EXPECT_TRUE(!containsCacheStatus(driverLogContent));
} else {
EXPECT_TRUE(containsCacheStatus(driverLogContent));
}

//With or without enable UMD caching, the compilation time for the second time should be shorter than the first.
EXPECT_GT(durationFirst.count(), durationSecond.count());
}

TEST_P(CompileAndDriverCaching, CompilationCacheWithEmptyConfig) {
ze_graph_dditable_ext_decorator& graph_ddi_table_ext = initStruct->getGraphDdiTable();
EXPECT_TRUE(containsCacheStatus(driverLogContent, ""));

ov::CompiledModel execNet;
//first run time will long and will generate the model cache.
Expand All @@ -205,26 +170,36 @@ TEST_P(CompileAndDriverCaching, CompilationCacheWithEmptyConfig) {
auto endFirst = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> durationFirst = endFirst - startFirst;

std::string driverLogContent = ::intel_npu::zeroUtils::getLatestBuildError(graph_ddi_table_ext);
EXPECT_TRUE(!containsCacheStatus(driverLogContent));
std::string driverLogContent2 = ::intel_npu::zeroUtils::getLatestBuildError(graph_ddi_table_ext);
#ifdef WIN32
EXPECT_TRUE(containsCacheStatus(driverLogContent2, "cache_status_t::found"));
#else
EXPECT_TRUE(containsCacheStatus(driverLogContent2, "cache_status_t::stored"));
#endif

//second time compilation
auto startSecond = std::chrono::high_resolution_clock::now();
OV_ASSERT_NO_THROW(execNet = core->compile_model(function, target_device, configuration));
auto endSecond = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> durationSecond = endSecond - startSecond;

std::string driverLogContent = ::intel_npu::zeroUtils::getLatestBuildError(graph_ddi_table_ext);
std::string driverLogContent3 = ::intel_npu::zeroUtils::getLatestBuildError(graph_ddi_table_ext);
if ((configuration.find("CACHE_DIR") != configuration.end()) || configuration.find("NPU_BYPASS_UMD_CACHING") != configuration.end()) {
EXPECT_TRUE(!containsCacheStatus(driverLogContent));
if (configuration.find("CACHE_DIR") != configuration.end()) {
m_cachedir = configuration.at(ov::cache_dir.name()).as<std::string>();
}
EXPECT_TRUE(containsCacheStatus(driverLogContent3, ""));
} else {
EXPECT_TRUE(containsCacheStatus(driverLogContent));
EXPECT_TRUE(containsCacheStatus(driverLogContent3, "cache_status_t::found"));
}
}

TEST_P(CompileAndDriverCaching, CompilationCacheWithOVCacheConfig) {
TEST_P(CompileAndDriverCaching, UMDCacheWithOVCacheConfig) {
ze_graph_dditable_ext_decorator& graph_ddi_table_ext = initStruct->getGraphDdiTable();

std::string driverLogContent = ::intel_npu::zeroUtils::getLatestBuildError(graph_ddi_table_ext);
EXPECT_TRUE(containsCacheStatus(driverLogContent, ""));

configuration[ov::cache_dir.name()] = "testCacheDir";
ov::CompiledModel execNet;
//first run time will long and will generate the model cache.
Expand All @@ -233,27 +208,24 @@ TEST_P(CompileAndDriverCaching, CompilationCacheWithOVCacheConfig) {
auto endFirst = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> durationFirst = endFirst - startFirst;

std::string driverLogContent = ::intel_npu::zeroUtils::getLatestBuildError(graph_ddi_table_ext);
EXPECT_TRUE(!containsCacheStatus(driverLogContent));
std::string driverLogContent2 = ::intel_npu::zeroUtils::getLatestBuildError(graph_ddi_table_ext);
EXPECT_TRUE(containsCacheStatus(driverLogContent2, ""));

//second time compilation
auto startSecond = std::chrono::high_resolution_clock::now();
OV_ASSERT_NO_THROW(execNet = core->compile_model(function, target_device, configuration));
auto endSecond = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> durationSecond = endSecond - startSecond;

std::string driverLogContent = ::intel_npu::zeroUtils::getLatestBuildError(graph_ddi_table_ext);
EXPECT_TRUE(!containsCacheStatus(driverLogContent));
std::string driverLogContent3 = ::intel_npu::zeroUtils::getLatestBuildError(graph_ddi_table_ext);
EXPECT_TRUE(containsCacheStatus(driverLogContent3, ""));
}

TEST_P(CompileAndDriverCaching, CompilationCacheWithBypassConfig) {
TEST_P(CompileAndDriverCaching, UMDCacheWithBypassConfig) {
ze_graph_dditable_ext_decorator& graph_ddi_table_ext = initStruct->getGraphDdiTable();
uint32_t graphDdiExtVersion = graph_ddi_table_ext.version();

// Check driver version. If driver version is less than 1.5, driver will not support UMD cache feature.
if (graphDdiExtVersion < ZE_GRAPH_EXT_VERSION_1_5) {
GTEST_SKIP() << "Skipping test for Driver version less than 1.5, current driver version: " << graphDdiExtVersion;
}

std::string driverLogContent = ::intel_npu::zeroUtils::getLatestBuildError(graph_ddi_table_ext);
EXPECT_TRUE(containsCacheStatus(driverLogContent, ""));

configuration[ov::intel_npu::bypass_umd_caching.name()] = true;
ov::CompiledModel execNet;
Expand All @@ -263,21 +235,19 @@ TEST_P(CompileAndDriverCaching, CompilationCacheWithBypassConfig) {
auto endFirst = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> durationFirst = endFirst - startFirst;

std::string driverLogContent = ::intel_npu::zeroUtils::getLatestBuildError(graph_ddi_table_ext);
EXPECT_TRUE(!containsCacheStatus(driverLogContent));
std::string driverLogContent2 = ::intel_npu::zeroUtils::getLatestBuildError(graph_ddi_table_ext);
EXPECT_TRUE(containsCacheStatus(driverLogContent2, ""));

//second time compilation
auto startSecond = std::chrono::high_resolution_clock::now();
OV_ASSERT_NO_THROW(execNet = core->compile_model(function, target_device, configuration));
auto endSecond = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> durationSecond = endSecond - startSecond;

std::string driverLogContent = ::intel_npu::zeroUtils::getLatestBuildError(graph_ddi_table_ext);
EXPECT_TRUE(!containsCacheStatus(driverLogContent));
std::string driverLogContent3 = ::intel_npu::zeroUtils::getLatestBuildError(graph_ddi_table_ext);
EXPECT_TRUE(containsCacheStatus(driverLogContent3, ""));
}



} // namespace behavior
} // namespace test
} // namespace ov
2 changes: 1 addition & 1 deletion ...el_npu/tests/functional/shared_tests_instances/behavior/compiled_model/driver_caching.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ namespace {

using namespace ov::test::behavior;

const std::vector<ov::AnyMap> Config = {{}, {ov::cache_dir("testCacheDir")}, {ov::intel_npu::bypass_umd_caching(true)}};
const std::vector<ov::AnyMap> Config = {{}};

INSTANTIATE_TEST_SUITE_P(smoke_CompilationUMDCacheg,
CompileAndDriverCaching,
Expand Down

0 comments on commit 1eda1e8

Please sign in to comment.