Omit circular bounds in upbound to avoid false Union{}

This is not ideal. As some times circular bound seems meaningful. But at least better than `Union{}` ?
JuliaLang · Jan 7, 2023 · 68e51c6 · 68e51c6
1 parent 8652f26
commit 68e51c6
Show file tree

Hide file tree

Showing 2 changed files with 40 additions and 7 deletions.
diff --git a/src/subtype.c b/src/subtype.c
@@ -2546,6 +2546,31 @@ static int var_occurs_inside(jl_value_t *v, jl_tvar_t *var, int inside, int want
     return 0;
 }
 
+static jl_value_t *omit_bad_union(jl_value_t *u, jl_tvar_t *t) JL_NOTSAFEPOINT
+{
+    if (jl_is_unionall(u) && ((jl_unionall_t *)u)->var != t) {
+        if (jl_has_typevar(((jl_unionall_t *)u)->var->lb, t))
+            return NULL;
+        jl_value_t *ub = omit_bad_union(((jl_unionall_t *)u)->var->ub, t);
+        jl_value_t *body = omit_bad_union(((jl_unionall_t *)u)->body, t);
+        if (body == NULL || ub == NULL)
+            return NULL;
+        ((jl_unionall_t *)u)->var->ub = ub;
+        ((jl_unionall_t *)u)->body = body;
+    }
+    else if (jl_is_uniontype(u)) {
+        jl_value_t *a = omit_bad_union(((jl_uniontype_t *)u)->a, t);
+        jl_value_t *b = omit_bad_union(((jl_uniontype_t *)u)->b, t);
+        if (a == NULL) return b;
+        if (b == NULL) return a;
+        ((jl_uniontype_t *)u)->a = a;
+        ((jl_uniontype_t *)u)->b = b;
+    }
+    else if (jl_has_typevar(u, t))
+        return NULL;
+    return u;
+}
+
 // Caller might not have rooted `res`
 static jl_value_t *finish_unionall(jl_value_t *res JL_MAYBE_UNROOTED, jl_varbinding_t *vb, jl_unionall_t *u, jl_stenv_t *e)
 {
@@ -2629,8 +2654,11 @@ static jl_value_t *finish_unionall(jl_value_t *res JL_MAYBE_UNROOTED, jl_varbind
         }
         if (jl_has_typevar(btemp->ub, vb->var)) {
             if (vb->ub == (jl_value_t*)btemp->var) {
-                JL_GC_POP();
-                return jl_bottom_type;
+                btemp->ub = omit_bad_union(btemp->ub, vb->var);
+                if (btemp->ub == NULL) {
+                    JL_GC_POP();
+                    return jl_bottom_type;
+                }
             }
             if (varval) {
                 JL_TRY {
@@ -2750,10 +2778,15 @@ static jl_value_t *intersect_unionall_(jl_value_t *t, jl_unionall_t *u, jl_stenv
             // T=Bottom in covariant position
             res = jl_bottom_type;
         }
-        else if (jl_has_typevar(vb->lb, u->var) || jl_has_typevar(vb->ub, u->var)) {
+        else if (jl_has_typevar(vb->lb, u->var)) {
             // fail on circular constraints
             res = jl_bottom_type;
         }
+        else {
+            vb->ub = omit_bad_union(vb->ub, u->var);
+            if (vb->ub == NULL)
+                res = jl_bottom_type;
+        }
     }
     if (res != jl_bottom_type)
         // res is rooted by callee

diff --git a/test/subtype.jl b/test/subtype.jl
@@ -1893,10 +1893,10 @@ end
 
 let
     # issue #22787
-    # for now check that these don't stack overflow
-    t = typeintersect(Tuple{Type{Q}, Q, Ref{Q}} where Q<:Ref,
-                      Tuple{Type{S}, Union{Ref{S}, Ref{R}}, R} where R where S)
-    @test_broken t != Union{}
+    @testintersect(Tuple{Type{Q}, Q, Ref{Q}} where Q<:Ref,
+                   Tuple{Type{S}, Union{Ref{S}, Ref{R}}, R} where R where S,
+                   !Union{})
+
     t = typeintersect(Tuple{Type{T}, T, Ref{T}} where T,
                       Tuple{Type{S}, Ref{S}, S} where S)
     @test_broken t != Union{}