Skip to content

Commit

Permalink
fix: cross platform float compatibility
Browse files Browse the repository at this point in the history
  • Loading branch information
josephcopenhaver committed Aug 8, 2024
1 parent 3092302 commit 7f4c3c0
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 3 deletions.
10 changes: 9 additions & 1 deletion loadtester/variance.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,14 +47,22 @@ func (wv *welfordVariance) Variance(count int) float64 {
//
// Note that if you copy this function it really should panic if the input is negative. Variance cannot be negative!
func varianceFloatString(f float64) string {
println("call")
// round first thing because it can cause an overflow on various CPU architectures
f = math.Round(f)

if math.IsNaN(f) {
println("NaN")
return ""
}

v := int64(math.MaxInt64)
if !math.IsInf(f, 1) {
v = int64(math.Round(f))
println("not +Inf")
// converting to int64 requires a "& math.MaxInt64" because it can cause an overflow on various CPU architectures
v = int64(f) & math.MaxInt64
}
println("return: " + strconv.FormatInt(v, 10))

return strconv.FormatInt(v, 10)
}
4 changes: 2 additions & 2 deletions loadtester/variance_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ func Test_varianceFloatString(t *testing.T) {
}
{
// garbage in - garbage out case
exp := "-9223372036854775808"
exp := "0"
if v := varianceFloatString(math.Inf(-1)); v != exp {
t.Fatalf("expected %s string, got %s", exp, v)
}
Expand All @@ -33,7 +33,7 @@ func Test_varianceFloatString(t *testing.T) {
}
{
// garbage in - garbage out case
exp := "-1"
exp := "9223372036854775807"
if v := varianceFloatString(-1.0); v != exp {
t.Fatalf("expected %s string, got %s", exp, v)
}
Expand Down

0 comments on commit 7f4c3c0

Please sign in to comment.