Skip to content

Commit

Permalink
Run C++ formatter
Browse files Browse the repository at this point in the history
  • Loading branch information
@markkohdev
markkohdev committed Mar 29, 2024
1 parent 49165f7 commit cd6f9a1
Show file tree
Hide file tree
Showing 11 changed files with 123 additions and 227 deletions.
2 changes: 1 addition & 1 deletion cpp/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ add_custom_target(check-formatting
)

# Run formatter
add_custom_target(check-formatting
add_custom_target(format
COMMAND ${FORMAT_COMMAND} `${FIND_COMMAND}`
COMMENT "Running C++ formatter"
)
2 changes: 1 addition & 1 deletion cpp/src/Index.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ class Index {
virtual void setEF(size_t ef) = 0;
virtual int getEF() const = 0;

virtual SpaceType getSpace( ) const = 0;
virtual SpaceType getSpace() const = 0;
virtual std::string getSpaceName() const = 0;

virtual StorageDataType getStorageDataType() const = 0;
Expand Down
52 changes: 17 additions & 35 deletions cpp/src/Spaces/Euclidean.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,8 @@ namespace hnswlib {
* should automatically do the loop unrolling for us here and vectorize as
* appropriate.
*/
template <typename dist_t, typename data_t = dist_t, int K = 1,
typename scalefactor = std::ratio<1, 1>>
static dist_t L2Sqr(const data_t *__restrict pVect1,
const data_t *__restrict pVect2, const size_t qty) {
template <typename dist_t, typename data_t = dist_t, int K = 1, typename scalefactor = std::ratio<1, 1>>
static dist_t L2Sqr(const data_t *__restrict pVect1, const data_t *__restrict pVect2, const size_t qty) {
dist_t res = 0;

for (size_t i = 0; i < qty / K; i++) {
Expand All @@ -51,22 +49,18 @@ static dist_t L2Sqr(const data_t *__restrict pVect1,
return (res * scale * scale);
}

template <typename dist_t, typename data_t = dist_t, int K,
typename scalefactor = std::ratio<1, 1>>
static dist_t L2SqrAtLeast(const data_t *__restrict pVect1,
const data_t *__restrict pVect2, const size_t qty) {
template <typename dist_t, typename data_t = dist_t, int K, typename scalefactor = std::ratio<1, 1>>
static dist_t L2SqrAtLeast(const data_t *__restrict pVect1, const data_t *__restrict pVect2, const size_t qty) {
size_t remainder = qty - K;

return L2Sqr<dist_t, data_t, K, scalefactor>(pVect1, pVect2, K) +
L2Sqr<dist_t, data_t, 1, scalefactor>(pVect1 + K, pVect2 + K,
remainder);
L2Sqr<dist_t, data_t, 1, scalefactor>(pVect1 + K, pVect2 + K, remainder);
}

#if defined(USE_AVX512)

// Favor using AVX512 if available.
static float L2SqrSIMD16Ext(const float *pVect1, const float *pVect2,
const size_t qty) {
static float L2SqrSIMD16Ext(const float *pVect1, const float *pVect2, const size_t qty) {
float PORTABLE_ALIGN64 TmpRes[16];
size_t qty16 = qty >> 4;

Expand All @@ -86,19 +80,16 @@ static float L2SqrSIMD16Ext(const float *pVect1, const float *pVect2,
}

_mm512_store_ps(TmpRes, sum);
float res = TmpRes[0] + TmpRes[1] + TmpRes[2] + TmpRes[3] + TmpRes[4] +
TmpRes[5] + TmpRes[6] + TmpRes[7] + TmpRes[8] + TmpRes[9] +
TmpRes[10] + TmpRes[11] + TmpRes[12] + TmpRes[13] + TmpRes[14] +
TmpRes[15];
float res = TmpRes[0] + TmpRes[1] + TmpRes[2] + TmpRes[3] + TmpRes[4] + TmpRes[5] + TmpRes[6] + TmpRes[7] +
TmpRes[8] + TmpRes[9] + TmpRes[10] + TmpRes[11] + TmpRes[12] + TmpRes[13] + TmpRes[14] + TmpRes[15];

return (res);
}

#elif defined(USE_AVX)

// Favor using AVX if available.
static float L2SqrSIMD16Ext(const float *pVect1, const float *pVect2,
const size_t qty) {
static float L2SqrSIMD16Ext(const float *pVect1, const float *pVect2, const size_t qty) {
float PORTABLE_ALIGN32 TmpRes[8];
size_t qty16 = qty >> 4;

Expand All @@ -124,14 +115,12 @@ static float L2SqrSIMD16Ext(const float *pVect1, const float *pVect2,
}

_mm256_store_ps(TmpRes, sum);
return TmpRes[0] + TmpRes[1] + TmpRes[2] + TmpRes[3] + TmpRes[4] + TmpRes[5] +
TmpRes[6] + TmpRes[7];
return TmpRes[0] + TmpRes[1] + TmpRes[2] + TmpRes[3] + TmpRes[4] + TmpRes[5] + TmpRes[6] + TmpRes[7];
}

#elif defined(USE_SSE)

static float L2SqrSIMD16Ext(const float *pVect1, const float *pVect2,
const size_t qty) {
static float L2SqrSIMD16Ext(const float *pVect1, const float *pVect2, const size_t qty) {
float PORTABLE_ALIGN32 TmpRes[8];
size_t qty16 = qty >> 4;

Expand Down Expand Up @@ -177,21 +166,18 @@ static float L2SqrSIMD16Ext(const float *pVect1, const float *pVect2,
#endif

#if defined(USE_SSE) || defined(USE_AVX) || defined(USE_AVX512)
static float L2SqrSIMD16ExtResiduals(const float *pVect1, const float *pVect2,
const size_t qty) {
static float L2SqrSIMD16ExtResiduals(const float *pVect1, const float *pVect2, const size_t qty) {
size_t qty16 = qty >> 4 << 4;
float res = L2SqrSIMD16Ext(pVect1, pVect2, qty16);

size_t qty_left = qty - qty16;
float res_tail =
L2Sqr<float, float>(pVect1 + qty16, pVect2 + qty16, qty_left);
float res_tail = L2Sqr<float, float>(pVect1 + qty16, pVect2 + qty16, qty_left);
return (res + res_tail);
}
#endif

#ifdef USE_SSE
static float L2SqrSIMD4Ext(const float *pVect1, const float *pVect2,
const size_t qty) {
static float L2SqrSIMD4Ext(const float *pVect1, const float *pVect2, const size_t qty) {
float PORTABLE_ALIGN32 TmpRes[8];
size_t qty4 = qty >> 2;

Expand All @@ -212,8 +198,7 @@ static float L2SqrSIMD4Ext(const float *pVect1, const float *pVect2,
return TmpRes[0] + TmpRes[1] + TmpRes[2] + TmpRes[3];
}

static float L2SqrSIMD4ExtResiduals(const float *pVect1, const float *pVect2,
const size_t qty) {
static float L2SqrSIMD4ExtResiduals(const float *pVect1, const float *pVect2, const size_t qty) {
size_t qty4 = qty >> 2 << 2;

float res = L2SqrSIMD4Ext(pVect1, pVect2, qty4);
Expand All @@ -225,8 +210,7 @@ static float L2SqrSIMD4ExtResiduals(const float *pVect1, const float *pVect2,
}
#endif

template <typename dist_t, typename data_t = dist_t,
typename scalefactor = std::ratio<1, 1>>
template <typename dist_t, typename data_t = dist_t, typename scalefactor = std::ratio<1, 1>>
class EuclideanSpace : public Space<dist_t, data_t> {
DISTFUNC<dist_t, data_t> fstdistfunc_;
size_t data_size_;
Expand Down Expand Up @@ -272,9 +256,7 @@ class EuclideanSpace : public Space<dist_t, data_t> {
~EuclideanSpace() {}
};

template <>
EuclideanSpace<float, float>::EuclideanSpace(size_t dim)
: data_size_(dim * sizeof(float)), dim_(dim) {
template <> EuclideanSpace<float, float>::EuclideanSpace(size_t dim) : data_size_(dim * sizeof(float)), dim_(dim) {
fstdistfunc_ = L2Sqr<float, float>;
#if defined(USE_SSE) || defined(USE_AVX) || defined(USE_AVX512)
if (dim % 16 == 0)
Expand Down
75 changes: 24 additions & 51 deletions cpp/src/Spaces/InnerProduct.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,8 @@ namespace hnswlib {
* compiler should automatically do the loop unrolling for us here and vectorize
* as appropriate.
*/
template <typename dist_t, typename data_t = dist_t, int K = 1,
typename scalefactor = std::ratio<1, 1>>
static dist_t InnerProductWithoutScale(const data_t *pVect1,
const data_t *pVect2, size_t qty) {
template <typename dist_t, typename data_t = dist_t, int K = 1, typename scalefactor = std::ratio<1, 1>>
static dist_t InnerProductWithoutScale(const data_t *pVect1, const data_t *pVect2, size_t qty) {
dist_t res = 0;

qty = qty / K;
Expand All @@ -51,28 +49,20 @@ static dist_t InnerProductWithoutScale(const data_t *pVect1,
return res;
}

template <typename dist_t, typename data_t = dist_t, int K = 1,
typename scalefactor = std::ratio<1, 1>>
static dist_t InnerProduct(const data_t *pVect1, const data_t *pVect2,
size_t qty) {
dist_t res = InnerProductWithoutScale<dist_t, data_t, K, scalefactor>(
pVect1, pVect2, qty);
template <typename dist_t, typename data_t = dist_t, int K = 1, typename scalefactor = std::ratio<1, 1>>
static dist_t InnerProduct(const data_t *pVect1, const data_t *pVect2, size_t qty) {
dist_t res = InnerProductWithoutScale<dist_t, data_t, K, scalefactor>(pVect1, pVect2, qty);
constexpr dist_t scale = (dist_t)scalefactor::num / (dist_t)scalefactor::den;
res *= scale * scale;
res = (static_cast<dist_t>(1.0f) - res);
return res;
}

template <typename dist_t, typename data_t = dist_t, int K,
typename scalefactor = std::ratio<1, 1>>
static dist_t InnerProductAtLeast(const data_t *__restrict pVect1,
const data_t *__restrict pVect2,
const size_t qty) {
template <typename dist_t, typename data_t = dist_t, int K, typename scalefactor = std::ratio<1, 1>>
static dist_t InnerProductAtLeast(const data_t *__restrict pVect1, const data_t *__restrict pVect2, const size_t qty) {
size_t remainder = qty - K;
dist_t res = InnerProductWithoutScale<dist_t, data_t, K, scalefactor>(
pVect1, pVect2, K) +
InnerProductWithoutScale<dist_t, data_t, 1, scalefactor>(
pVect1 + K, pVect2 + K, remainder);
dist_t res = InnerProductWithoutScale<dist_t, data_t, K, scalefactor>(pVect1, pVect2, K) +
InnerProductWithoutScale<dist_t, data_t, 1, scalefactor>(pVect1 + K, pVect2 + K, remainder);
constexpr dist_t scale = (dist_t)scalefactor::num / (dist_t)scalefactor::den;
res *= scale * scale;
res = (static_cast<dist_t>(1.0f) - res);
Expand All @@ -82,8 +72,7 @@ static dist_t InnerProductAtLeast(const data_t *__restrict pVect1,
#if defined(USE_AVX)

// Favor using AVX if available.
static float InnerProductSIMD4Ext(const float *pVect1, const float *pVect2,
const size_t qty) {
static float InnerProductSIMD4Ext(const float *pVect1, const float *pVect2, const size_t qty) {
float PORTABLE_ALIGN32 TmpRes[8];

size_t qty16 = qty / 16;
Expand Down Expand Up @@ -111,8 +100,7 @@ static float InnerProductSIMD4Ext(const float *pVect1, const float *pVect2,
}

__m128 v1, v2;
__m128 sum_prod = _mm_add_ps(_mm256_extractf128_ps(sum256, 0),
_mm256_extractf128_ps(sum256, 1));
__m128 sum_prod = _mm_add_ps(_mm256_extractf128_ps(sum256, 0), _mm256_extractf128_ps(sum256, 1));

while (pVect1 < pEnd2) {
v1 = _mm_loadu_ps(pVect1);
Expand All @@ -130,8 +118,7 @@ static float InnerProductSIMD4Ext(const float *pVect1, const float *pVect2,

#elif defined(USE_SSE)

static float InnerProductSIMD4Ext(const float *pVect1, const float *pVect2,
const size_t qty) {
static float InnerProductSIMD4Ext(const float *pVect1, const float *pVect2, const size_t qty) {
float PORTABLE_ALIGN32 TmpRes[8];

size_t qty16 = qty / 16;
Expand Down Expand Up @@ -187,8 +174,7 @@ static float InnerProductSIMD4Ext(const float *pVect1, const float *pVect2,

#if defined(USE_AVX512)

static float InnerProductSIMD16Ext(const float *pVect1, const float *pVect2,
const size_t qty) {
static float InnerProductSIMD16Ext(const float *pVect1, const float *pVect2, const size_t qty) {
float PORTABLE_ALIGN64 TmpRes[16];

size_t qty16 = qty / 16;
Expand All @@ -208,18 +194,15 @@ static float InnerProductSIMD16Ext(const float *pVect1, const float *pVect2,
}

_mm512_store_ps(TmpRes, sum512);
float sum = TmpRes[0] + TmpRes[1] + TmpRes[2] + TmpRes[3] + TmpRes[4] +
TmpRes[5] + TmpRes[6] + TmpRes[7] + TmpRes[8] + TmpRes[9] +
TmpRes[10] + TmpRes[11] + TmpRes[12] + TmpRes[13] + TmpRes[14] +
TmpRes[15];
float sum = TmpRes[0] + TmpRes[1] + TmpRes[2] + TmpRes[3] + TmpRes[4] + TmpRes[5] + TmpRes[6] + TmpRes[7] +
TmpRes[8] + TmpRes[9] + TmpRes[10] + TmpRes[11] + TmpRes[12] + TmpRes[13] + TmpRes[14] + TmpRes[15];

return 1.0f - sum;
}

#elif defined(USE_AVX)

static float InnerProductSIMD16Ext(const float *pVect1, const float *pVect2,
const size_t qty) {
static float InnerProductSIMD16Ext(const float *pVect1, const float *pVect2, const size_t qty) {
float PORTABLE_ALIGN32 TmpRes[8];

size_t qty16 = qty / 16;
Expand All @@ -245,16 +228,14 @@ static float InnerProductSIMD16Ext(const float *pVect1, const float *pVect2,
}

_mm256_store_ps(TmpRes, sum256);
float sum = TmpRes[0] + TmpRes[1] + TmpRes[2] + TmpRes[3] + TmpRes[4] +
TmpRes[5] + TmpRes[6] + TmpRes[7];
float sum = TmpRes[0] + TmpRes[1] + TmpRes[2] + TmpRes[3] + TmpRes[4] + TmpRes[5] + TmpRes[6] + TmpRes[7];

return 1.0f - sum;
}

#elif defined(USE_SSE)

static float InnerProductSIMD16Ext(const float *pVect1, const float *pVect2,
const size_t qty) {
static float InnerProductSIMD16Ext(const float *pVect1, const float *pVect2, const size_t qty) {
float PORTABLE_ALIGN32 TmpRes[8];
size_t qty16 = qty / 16;

Expand Down Expand Up @@ -297,35 +278,28 @@ static float InnerProductSIMD16Ext(const float *pVect1, const float *pVect2,
#endif

#if defined(USE_SSE) || defined(USE_AVX) || defined(USE_AVX512)
static float InnerProductSIMD16ExtResiduals(const float *pVect1,
const float *pVect2,
const size_t qty) {
static float InnerProductSIMD16ExtResiduals(const float *pVect1, const float *pVect2, const size_t qty) {
size_t qty16 = qty >> 4 << 4;
float res = InnerProductSIMD16Ext(pVect1, pVect2, qty16);

size_t qty_left = qty - qty16;
float res_tail =
InnerProduct<float, float>(pVect1 + qty16, pVect2 + qty16, qty_left);
float res_tail = InnerProduct<float, float>(pVect1 + qty16, pVect2 + qty16, qty_left);
return res + res_tail - 1.0f;
}

static float InnerProductSIMD4ExtResiduals(const float *pVect1,
const float *pVect2,
const size_t qty) {
static float InnerProductSIMD4ExtResiduals(const float *pVect1, const float *pVect2, const size_t qty) {
size_t qty4 = qty >> 2 << 2;

float res = InnerProductSIMD4Ext(pVect1, pVect2, qty4);
size_t qty_left = qty - qty4;

float res_tail =
InnerProduct<float, float>(pVect1 + qty4, pVect2 + qty4, qty_left);
float res_tail = InnerProduct<float, float>(pVect1 + qty4, pVect2 + qty4, qty_left);

return res + res_tail - 1.0f;
}
#endif

template <typename dist_t, typename data_t = dist_t,
typename scalefactor = std::ratio<1, 1>>
template <typename dist_t, typename data_t = dist_t, typename scalefactor = std::ratio<1, 1>>
class InnerProductSpace : public Space<dist_t, data_t> {
DISTFUNC<dist_t, data_t> fstdistfunc_;
size_t data_size_;
Expand Down Expand Up @@ -371,8 +345,7 @@ class InnerProductSpace : public Space<dist_t, data_t> {
};

template <>
InnerProductSpace<float, float>::InnerProductSpace(size_t dim)
: data_size_(dim * sizeof(float)), dim_(dim) {
InnerProductSpace<float, float>::InnerProductSpace(size_t dim) : data_size_(dim * sizeof(float)), dim_(dim) {
fstdistfunc_ = InnerProduct<float, float>;
#if defined(USE_SSE) || defined(USE_AVX) || defined(USE_AVX512)
if (dim % 16 == 0)
Expand Down
3 changes: 1 addition & 2 deletions cpp/src/Spaces/Space.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,7 @@

namespace hnswlib {
template <typename MTYPE, typename data_t = MTYPE>
using DISTFUNC =
std::function<MTYPE(const data_t *, const data_t *, const size_t)>;
using DISTFUNC = std::function<MTYPE(const data_t *, const data_t *, const size_t)>;

/**
* An abstract class representing a type of space to search through,
Expand Down
Loading

0 comments on commit cd6f9a1

Please sign in to comment.