From 6d1cb12aafe19a0c5574e1356aac18f1d8c6f786 Mon Sep 17 00:00:00 2001
From: jakobeha <jakobeha@gmail.com>
Date: Thu, 6 Jun 2024 18:39:29 -0400
Subject: [PATCH] *maybe* fix LatticeTest rare failure

---
 .../org/prlprg/ir/type/OverloadRType.java     | 37 +++++++++++++++++++
 .../org/prlprg/ir/type/RFunctionType.java     |  3 +-
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/prlprg/ir/type/OverloadRType.java b/src/main/java/org/prlprg/ir/type/OverloadRType.java
index 491f0bc87..0a05f9243 100644
--- a/src/main/java/org/prlprg/ir/type/OverloadRType.java
+++ b/src/main/java/org/prlprg/ir/type/OverloadRType.java
@@ -454,6 +454,43 @@ public int hashCode() {
         parameters, fallbackEffects, genericEffects, fallbackReturnType, genericReturnType);
   }
 
+  /**
+   * If the types are equal, returns 0. Otherwise, returns a positive or negative integer, so that
+   * this function is commutative and associative.
+   */
+  int rawCompareTo(OverloadRTypeImpl other) {
+    // Compare hashCode, but in the rare case they are equal, we must do a deep comparison.
+    // Specifically, we do `toString` and compare generic effects because that finds any
+    // differences.
+    var hashCodeCmp = Integer.compare(hashCode(), other.hashCode());
+    if (hashCodeCmp != 0) {
+      return hashCodeCmp;
+    }
+    var toStringCmp = toString().compareTo(other.toString());
+    if (toStringCmp != 0) {
+      return toStringCmp;
+    }
+    if (genericEffects != null && other.genericEffects != null) {
+      var genericEffectsCmp =
+          Integer.compare(genericEffects.hashCode(), other.genericEffects.hashCode());
+      if (genericEffectsCmp != 0) {
+        return genericEffectsCmp;
+      }
+    } else if (genericEffects != null) {
+      return 1;
+    } else if (other.genericEffects != null) {
+      return -1;
+    }
+    if (genericReturnType != null && other.genericReturnType != null) {
+      return Integer.compare(genericReturnType.hashCode(), other.genericReturnType.hashCode());
+    } else if (genericReturnType != null) {
+      return 1;
+    } else if (other.genericReturnType != null) {
+      return -1;
+    }
+    return 0;
+  }
+
   @Override
   public String toString() {
     return "("
diff --git a/src/main/java/org/prlprg/ir/type/RFunctionType.java b/src/main/java/org/prlprg/ir/type/RFunctionType.java
index 518c8fb17..0ceb8e101 100644
--- a/src/main/java/org/prlprg/ir/type/RFunctionType.java
+++ b/src/main/java/org/prlprg/ir/type/RFunctionType.java
@@ -118,7 +118,8 @@ static ImmutableList<OverloadRType> normalizeOverloadsWrtSubsuming(
           if (o2SupersetsO1 && !o1SupersetsO2) {
             return -1;
           }
-          return Integer.compare(o1.hashCode(), o2.hashCode());
+
+          return ((OverloadRTypeImpl) o1).rawCompareTo((OverloadRTypeImpl) o2);
         });
     return ImmutableList.copyOf(list);
   }