Skip to content

Commit

Permalink
[TEST] Compare whole Tensor in Template Plugin tests (openvinotoolkit…
Browse files Browse the repository at this point in the history 
…#23408)

### Details:
- `ov::test::utils::compare(...)` used for output vs reference values
comparison (where possible).
- Tests with a need of complex changes were marked with `legacy_compare`
flag. These tests still use legacy raw data comparison and theirs
transition will be addressed in further PR's.

### Tickets:
 - CVS-135274, CVS-135275

---------

Co-authored-by: Michal Lukaszewski <[email protected]>
  • Loading branch information
@t-jankowski @mlukasze
t-jankowski and mlukasze authored Apr 3, 2024
1 parent 3b2a7ef commit 0799163
Show file tree
Hide file tree
Showing 93 changed files with 1,302 additions and 1,166 deletions.
4 changes: 2 additions & 2 deletions src/plugins/template/tests/functional/op_reference/adaptive_avg_pool.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ struct AdaptiveAvgPoolParams {
m_output_shape(output_shape),
m_input_type(input_type),
m_output_type(ouput_type),
m_input_data(CreateTensor(input_type, input_values)),
m_expected_data(CreateTensor(ouput_type, output_values)),
m_input_data(CreateTensor(m_input_shape, input_type, input_values)),
m_expected_data(CreateTensor(m_output_shape, ouput_type, output_values)),
m_adaptive_shape(adaptive_shape),
m_adaptive_values(adaptive_values) {}
Shape m_input_shape;
Expand Down
6 changes: 3 additions & 3 deletions src/plugins/template/tests/functional/op_reference/adaptive_max_pool.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@ struct AdaptiveMaxPoolParams {
m_output_shape(output_shape),
m_input_type(input_type),
m_output_type(output_type),
m_input_data(CreateTensor(input_type, input_values)),
m_expected_data(CreateTensor(output_type, output_values)),
m_expected_indices(CreateTensor(element::Type_t::i64, output_indices)),
m_input_data(CreateTensor(input_shape, input_type, input_values)),
m_expected_data(CreateTensor(output_shape, output_type, output_values)),
m_expected_indices(CreateTensor(output_shape, element::Type_t::i64, output_indices)),
m_adaptive_shape(adaptive_shape),
m_adaptive_values(adaptive_values) {}
Shape m_input_shape;
Expand Down
48 changes: 27 additions & 21 deletions src/plugins/template/tests/functional/op_reference/add.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,9 @@ namespace {

struct AddParams {
template <class IT>
AddParams(const PartialShape& shape1,
const PartialShape& shape2,
AddParams(const Shape& shape1,
const Shape& shape2,
const Shape& outshape,
const element::Type& iType,
const std::vector<IT>& iValues1,
const std::vector<IT>& iValues2,
Expand All @@ -25,12 +26,12 @@ struct AddParams {
pshape2(shape2),
inType(iType),
outType(iType),
inputData1(CreateTensor(iType, iValues1)),
inputData2(CreateTensor(iType, iValues2)),
refData(CreateTensor(iType, oValues)) {}
inputData1(CreateTensor(shape1, iType, iValues1)),
inputData2(CreateTensor(shape2, iType, iValues2)),
refData(CreateTensor(outshape, iType, oValues)) {}

PartialShape pshape1;
PartialShape pshape2;
Shape pshape1;
Shape pshape2;
element::Type inType;
element::Type outType;
ov::Tensor inputData1;
Expand Down Expand Up @@ -58,8 +59,8 @@ class ReferenceAddLayerTest : public testing::TestWithParam<AddParams>, public C
}

private:
static std::shared_ptr<Model> CreateFunction(const PartialShape& input_shape1,
const PartialShape& input_shape2,
static std::shared_ptr<Model> CreateFunction(const Shape& input_shape1,
const Shape& input_shape2,
const element::Type& input_type,
const element::Type& expected_output_type) {
const auto in1 = std::make_shared<op::v0::Parameter>(input_type, input_shape1);
Expand Down Expand Up @@ -89,8 +90,8 @@ class ReferenceAddInPlaceLayerTest : public testing::TestWithParam<AddParams>, p
}

private:
static std::shared_ptr<Model> CreateFunction(const PartialShape& input_shape1,
const PartialShape& input_shape2,
static std::shared_ptr<Model> CreateFunction(const Shape& input_shape1,
const Shape& input_shape2,
const element::Type& input_type,
const element::Type& expected_output_type) {
const auto in1 = std::make_shared<op::v0::Parameter>(input_type, input_shape1);
Expand All @@ -116,26 +117,30 @@ std::vector<AddParams> generateParamsForAdd() {
using T = typename element_type_traits<IN_ET>::value_type;

std::vector<AddParams> params{
AddParams(ov::PartialShape{2, 2},
ov::PartialShape{2, 2},
AddParams(ov::Shape{2, 2},
ov::Shape{2, 2},
ov::Shape{2, 2},
IN_ET,
std::vector<T>{1, 2, 3, 4},
std::vector<T>{5, 6, 7, 8},
std::vector<T>{6, 8, 10, 12}),
AddParams(ov::PartialShape{1, 2},
ov::PartialShape{3, 2, 2},
AddParams(ov::Shape{1, 2},
ov::Shape{3, 2, 2},
ov::Shape{3, 2, 2},
IN_ET,
std::vector<T>{1, 2},
std::vector<T>{5, 6, 7, 8, 2, 3, 1, 5, 6, 7, 1, 3},
std::vector<T>{6, 8, 8, 10, 3, 5, 2, 7, 7, 9, 2, 5}),
AddParams(ov::PartialShape{1},
ov::PartialShape{1},
AddParams(ov::Shape{1},
ov::Shape{1},
ov::Shape{1},
IN_ET,
std::vector<T>{2},
std::vector<T>{8},
std::vector<T>{10}),
AddParams(ov::PartialShape{2, 2},
ov::PartialShape{1},
AddParams(ov::Shape{2, 2},
ov::Shape{1},
ov::Shape{2, 2},
IN_ET,
std::vector<T>{2, 4, 7, 8},
std::vector<T>{8},
Expand All @@ -148,8 +153,9 @@ template <element::Type_t IN_ET>
std::vector<AddParams> generateParamsForAddInPlace() {
using T = typename element_type_traits<IN_ET>::value_type;

std::vector<AddParams> params{AddParams(ov::PartialShape{2, 2},
ov::PartialShape{2, 2},
std::vector<AddParams> params{AddParams(ov::Shape{2, 2},
ov::Shape{2, 2},
ov::Shape{2, 2},
IN_ET,
std::vector<T>{1, 2, 3, 4},
std::vector<T>{5, 6, 7, 8},
Expand Down
1 change: 1 addition & 0 deletions src/plugins/template/tests/functional/op_reference/atanh.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ struct Builder : ParamsBuilder<AtanhParams> {
class ReferenceAtanhLayerTest : public testing::TestWithParam<AtanhParams>, public CommonReferenceTest {
public:
void SetUp() override {
legacy_compare = true;
auto params = GetParam();
function = CreateFunction(params.input.shape, params.input.type);
inputData = {params.input.data};
Expand Down
2 changes: 1 addition & 1 deletion src/plugins/template/tests/functional/op_reference/augru_cell.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ std::vector<AUGRUCellParams> generateParams() {
0.07849785f,
0.76568159f}),
reference_tests::Tensor(ET, {1, 1}, std::vector<T>{0.3333f}),
reference_tests::Tensor(ET, {2, 2}, std::vector<T>{0.666063f, 0.451451f, 0.792762f, 0.453281f}),
reference_tests::Tensor(ET, {1, 4}, std::vector<T>{0.666063f, 0.451451f, 0.792762f, 0.453281f}),
"augru_different_input_and_hidden_size"),
};
return params;
Expand Down
4 changes: 2 additions & 2 deletions src/plugins/template/tests/functional/op_reference/avg_pool.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ struct AvgPoolParams {
m_output_shape(output_shape),
m_input_type(input_type),
m_output_type(ouput_type),
m_input_data(CreateTensor(input_type, input_values)),
m_expected_data(CreateTensor(ouput_type, output_values)),
m_input_data(CreateTensor(input_shape, input_type, input_values)),
m_expected_data(CreateTensor(output_shape, ouput_type, output_values)),
m_strides(strides),
m_pads_begin(pads_begin),
m_pads_end(pads_end),
Expand Down
36 changes: 35 additions & 1 deletion src/plugins/template/tests/functional/op_reference/base_reference_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,13 @@ void CommonReferenceTest::Validate() {

ASSERT_EQ(refOutData.size(), actualOutData.size());
for (size_t i = 0; i < refOutData.size(); i++) {
ValidateBlobs(refOutData[i], actualOutData[i], i, threshold, abs_threshold, actual_comparision_size);
ValidateBlobs(refOutData[i],
actualOutData[i],
i,
threshold,
abs_threshold,
legacy_compare,
actual_comparision_size);
}
}

Expand All @@ -85,6 +91,7 @@ void CommonReferenceTest::ValidateBlobs(const ov::Tensor& refBlob,
const size_t blob_idx,
float threshold,
float abs_threshold,
bool legacy_compare,
size_t actual_comparision_size) {
ASSERT_EQ(refBlob.get_element_type(), outBlob.get_element_type())
<< "Incompatible element type for blob with index " << blob_idx;
Expand All @@ -95,6 +102,33 @@ void CommonReferenceTest::ValidateBlobs(const ov::Tensor& refBlob,
actual_comparision_size = refBlob.get_size();

const auto& element_type = refBlob.get_element_type();
if (!legacy_compare) {
double abs_threshold_{abs_threshold};
if (abs_threshold_ < 0.)
abs_threshold_ = std::numeric_limits<double>::max();

switch (element_type) {
case ov::element::boolean:
case ov::element::bf16:
case ov::element::f16:
case ov::element::f32:
case ov::element::f64:
case ov::element::i8:
case ov::element::i16:
case ov::element::i32:
case ov::element::i64:
case ov::element::u8:
case ov::element::u16:
case ov::element::u32:
case ov::element::u64:
ov::test::utils::compare(refBlob,
outBlob,
// actual_comparision_size,
abs_threshold_,
threshold);
return;
}
}
switch (element_type) {
case ov::element::bf16:
ov::test::utils::compare_raw_data<ov::bfloat16, ov::bfloat16>(refBlob.data<const ov::bfloat16>(),
Expand Down
6 changes: 4 additions & 2 deletions src/plugins/template/tests/functional/op_reference/base_reference_test.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,11 @@ class CommonReferenceTest {
const size_t blob_idx,
float threshold,
float abs_threshold,
size_t actual_comparision_size = 0);
bool legacy_compare,
size_t actual_comparision_size);

protected:
bool legacy_compare = false;
const std::string targetDevice;
std::shared_ptr<ov::Core> core;
std::shared_ptr<ov::Model> function;
Expand Down Expand Up @@ -82,7 +84,7 @@ struct Tensor {

template <typename T>
Tensor(const ov::Shape& shape, ov::element::Type type, const std::vector<T>& data_elements)
: Tensor{shape, type, CreateTensor(type, data_elements)} {}
: Tensor{shape, type, CreateTensor(shape, type, data_elements)} {}

// Temporary constructor to create blob with passed input shape (not 1-dimensional)
template <typename T>
Expand Down
4 changes: 2 additions & 2 deletions src/plugins/template/tests/functional/op_reference/batch_norm.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ struct BatchNormParams {
m_expected_shape(expected_shape),
m_input_type(input_type),
m_expected_type(expected_type),
m_input_value(CreateTensor(input_type, input_value)),
m_expected_value(CreateTensor(expected_type, expected_value)),
m_input_value(CreateTensor(input_shape, input_type, input_value)),
m_expected_value(CreateTensor(expected_shape, expected_type, expected_value)),
m_gamma(CreateTensor(Shape{input_shape.at(1)}, input_type, gamma)),
m_beta(CreateTensor(Shape{input_shape.at(1)}, input_type, beta)),
m_mean(CreateTensor(Shape{input_shape.at(1)}, input_type, mean)),
Expand Down
2 changes: 1 addition & 1 deletion src/plugins/template/tests/functional/op_reference/batch_to_space.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ std::vector<BatchToSpaceParams> generateBatchToSpaceParams() {
reference_tests::Tensor({4}, element::i64, std::vector<int64_t>{1, 1, 2, 2}),
reference_tests::Tensor({4}, element::i64, std::vector<int64_t>{0, 0, 0, 0}),
reference_tests::Tensor({4}, element::i64, std::vector<int64_t>{0, 0, 0, 0}),
reference_tests::Tensor({2, 1, 1, 6}, IN_ET, std::vector<T>{1, 4, 2, 5, 3, 6, 7, 10, 8, 11, 9, 12}),
reference_tests::Tensor({1, 1, 2, 6}, IN_ET, std::vector<T>{1, 4, 2, 5, 3, 6, 7, 10, 8, 11, 9, 12}),
"input_with_shape_4x1x1x3_2"),

// input_with_shape_4x1x2x3
Expand Down
48 changes: 24 additions & 24 deletions src/plugins/template/tests/functional/op_reference/binary_convolution.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@ using namespace ov;
namespace {
struct BinaryConvolutionParams {
template <class T>
BinaryConvolutionParams(const PartialShape& inputShape,
BinaryConvolutionParams(const Shape& inputShape,
const Shape& filterShape,
const PartialShape& outputShape,
const Shape& outputShape,
const element::Type& iType,
const std::vector<T>& iValues,
const std::vector<uint8_t>& filterValues,
Expand All @@ -33,18 +33,18 @@ struct BinaryConvolutionParams {
outputShape(outputShape),
inType(iType),
outType(iType),
inputData(CreateTensor(iType, iValues)),
inputData(CreateTensor(inputShape, iType, iValues)),
filterData(filterValues),
refData(CreateTensor(iType, oValues)),
refData(CreateTensor(outputShape, iType, oValues)),
strides(strides),
padBegin(padBegin),
padEnd(padEnd),
dialations(dialations),
padValue(padValue) {}

PartialShape inputShape;
Shape inputShape;
Shape filterShape;
PartialShape outputShape;
Shape outputShape;
ov::element::Type inType;
ov::element::Type outType;
ov::Tensor inputData;
Expand Down Expand Up @@ -119,9 +119,9 @@ std::vector<BinaryConvolutionParams> generateBinaryConvolutionParams() {

std::vector<BinaryConvolutionParams> binaryConvolutionParams{
// --------------------- 2D BinaryConvolution ------------------------------------------
BinaryConvolutionParams(PartialShape{1, 1, 4, 4},
BinaryConvolutionParams(Shape{1, 1, 4, 4},
Shape{1, 1, 3, 3},
PartialShape{1, 1, 2, 2},
Shape{1, 1, 2, 2},
IN_ET,
std::vector<T>{1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1},
std::vector<uint8_t>{0xAA, 0x80}, // 10101010 10000000
Expand All @@ -130,9 +130,9 @@ std::vector<BinaryConvolutionParams> generateBinaryConvolutionParams() {
{0, 0},
{0, 0},
{1, 1}),
BinaryConvolutionParams(PartialShape{1, 1, 4, 4},
BinaryConvolutionParams(Shape{1, 1, 4, 4},
Shape{1, 1, 3, 3},
PartialShape{1, 1, 4, 4},
Shape{1, 1, 4, 4},
IN_ET,
std::vector<T>{1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1},
std::vector<uint8_t>{0xAA, 0x80}, // 10101010 10000000
Expand All @@ -141,9 +141,9 @@ std::vector<BinaryConvolutionParams> generateBinaryConvolutionParams() {
{1, 1},
{1, 1},
{1, 1}),
BinaryConvolutionParams(PartialShape{1, 1, 4, 4},
BinaryConvolutionParams(Shape{1, 1, 4, 4},
Shape{1, 1, 3, 3},
PartialShape{1, 1, 4, 4},
Shape{1, 1, 4, 4},
IN_ET,
std::vector<T>{1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1},
std::vector<uint8_t>{0xAA, 0x80}, // 10101010 10000000
Expand All @@ -154,9 +154,9 @@ std::vector<BinaryConvolutionParams> generateBinaryConvolutionParams() {
{1, 1},
1.0f),
BinaryConvolutionParams(
PartialShape{1, 1, 5, 5},
Shape{1, 1, 5, 5},
Shape{1, 1, 3, 3},
PartialShape{1, 1, 2, 2},
Shape{1, 1, 2, 2},
IN_ET,
std::vector<T>{0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 1},
std::vector<uint8_t>{0x2E, 0x00}, // 10101010 10000000
Expand All @@ -166,9 +166,9 @@ std::vector<BinaryConvolutionParams> generateBinaryConvolutionParams() {
{0, 0},
{1, 1}),
BinaryConvolutionParams(
PartialShape{1, 1, 7, 7},
Shape{1, 1, 7, 7},
Shape{1, 1, 3, 3},
Shape{1, 1, 3, 3},
PartialShape{1, 1, 3, 3},
IN_ET,
std::vector<T>{1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1,
1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0},
Expand All @@ -179,9 +179,9 @@ std::vector<BinaryConvolutionParams> generateBinaryConvolutionParams() {
{0, 0},
{2, 2}),
BinaryConvolutionParams(
PartialShape{1, 1, 7, 7},
Shape{1, 1, 7, 7},
Shape{1, 1, 3, 3},
PartialShape{1, 1, 4, 4},
Shape{1, 1, 4, 4},
IN_ET,
std::vector<T>{1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1,
1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0},
Expand All @@ -192,9 +192,9 @@ std::vector<BinaryConvolutionParams> generateBinaryConvolutionParams() {
{2, 2},
{2, 2}),
BinaryConvolutionParams(
PartialShape{1, 1, 7, 7},
Shape{1, 1, 7, 7},
Shape{1, 1, 3, 3},
PartialShape{1, 1, 4, 4},
Shape{1, 1, 4, 4},
IN_ET,
std::vector<T>{1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1,
1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0},
Expand All @@ -205,9 +205,9 @@ std::vector<BinaryConvolutionParams> generateBinaryConvolutionParams() {
{2, 2},
{2, 2},
1.0f),
BinaryConvolutionParams(PartialShape{1, 2, 4, 4},
BinaryConvolutionParams(Shape{1, 2, 4, 4},
Shape{1, 2, 3, 3},
PartialShape{1, 1, 2, 2},
Shape{1, 1, 2, 2},
IN_ET,
std::vector<T>{// channel 1
1,
Expand Down Expand Up @@ -249,9 +249,9 @@ std::vector<BinaryConvolutionParams> generateBinaryConvolutionParams() {
{0, 0},
{0, 0},
{1, 1}),
BinaryConvolutionParams(PartialShape{2, 1, 4, 4},
BinaryConvolutionParams(Shape{2, 1, 4, 4},
Shape{1, 1, 3, 3},
PartialShape{2, 1, 2, 2},
Shape{2, 1, 2, 2},
IN_ET,
std::vector<T>{// batch 1
1,
Expand Down
Loading

0 comments on commit 0799163

Please sign in to comment.