Skip to content

Commit

Permalink
fix: cross platform float compatibility
Browse files Browse the repository at this point in the history
  • Loading branch information
josephcopenhaver committed Aug 8, 2024
1 parent 3092302 commit 277a286
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 3 deletions.
6 changes: 5 additions & 1 deletion loadtester/variance.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,13 +47,17 @@ func (wv *welfordVariance) Variance(count int) float64 {
//
// Note that if you copy this function it really should panic if the input is negative. Variance cannot be negative!
func varianceFloatString(f float64) string {
// round first thing because it can cause an overflow on various CPU architectures
f = math.Round(f)

if math.IsNaN(f) {
return ""
}

v := int64(math.MaxInt64)
if !math.IsInf(f, 1) {
v = int64(math.Round(f))
// converting to int64 requires a "& math.MaxInt64" because it can cause an overflow on various CPU architectures
v = int64(f) & math.MaxInt64
}

return strconv.FormatInt(v, 10)
Expand Down
4 changes: 2 additions & 2 deletions loadtester/variance_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ func Test_varianceFloatString(t *testing.T) {
}
{
// garbage in - garbage out case
exp := "-9223372036854775808"
exp := "0"
if v := varianceFloatString(math.Inf(-1)); v != exp {
t.Fatalf("expected %s string, got %s", exp, v)
}
Expand All @@ -33,7 +33,7 @@ func Test_varianceFloatString(t *testing.T) {
}
{
// garbage in - garbage out case
exp := "-1"
exp := "9223372036854775807"
if v := varianceFloatString(-1.0); v != exp {
t.Fatalf("expected %s string, got %s", exp, v)
}
Expand Down

0 comments on commit 277a286

Please sign in to comment.