Mutually recursive field types, the easy version

This is an alternative to #32581, that is easier to implement, but
more restrictivive. Here, the forward declaration must contain
everything except for the field types:
```
incomplete type Foo{Bar} <: Baz; end
```
with it being an error to deviate from this specification when
completing the type
```
struct Foo <: Baz; end # Error: Missing type parameter
struct Foo{A, B} <: Baz; end # Error: Too many type parameters
struct Foo{Qux} <: Baz; end # Error: Name of type parameter doesn't match
struct Foo{Bar<:Int64} <: Baz; end # Error: Bounds of type parameter don't match
struct Foo{Bar}; end; # Error supertype dosesn't match
```
This is easier because this way we have enough information for subtyping
and type application and therefor do not need to delay this until
the entire dependency graph is complete as we did in #32581.
Of course this also means that we don't get the union feature that
was requested in #269:
```
inomplete type U; end
struct A; x::U; end
struct B; x::U; end
U = Union{A, B}; #ERROR
```
However, it could of course be emulated by wrapping the union type:
```
struct U
   data::Union{A, B}
end
```

However, given the simplicity of this change and the difficulty of #32581,
this seems like the way to go for the moment. We may want to revisit all
this if we ever want to do computed field types, which will encounter similar
challenges as #32581.

Fixes #269.

base/compiler/tfuncs.jl

@@ -32,6 +32,7 @@ const DATATYPE_TYPES_FIELDINDEX = fieldindex(DataType, :types)
 const DATATYPE_SUPER_FIELDINDEX = fieldindex(DataType, :super)
 const DATATYPE_MUTABLE_FIELDINDEX = fieldindex(DataType, :mutable)
 const DATATYPE_INSTANCE_FIELDINDEX = fieldindex(DataType, :instance)
+const DATATYPE_INCOMPLETE_FIELDINDEX = fieldindex(DataType, :incomplete)
 
 const TYPENAME_NAME_FIELDINDEX = fieldindex(Core.TypeName, :name)
 const TYPENAME_MODULE_FIELDINDEX = fieldindex(Core.TypeName, :module)
@@ -547,19 +548,26 @@ function subtype_tfunc(@nospecialize(a), @nospecialize(b))
 end
 add_tfunc(<:, 2, 2, subtype_tfunc, 0)
 
+is_dt_complete_const_field(fld::Int) = (
+     fld == DATATYPE_TYPES_FIELDINDEX ||
+     fld == DATATYPE_MUTABLE_FIELDINDEX ||
+     fld == DATATYPE_INSTANCE_FIELDINDEX ||
+     fld == DATATYPE_INCOMPLETE_FIELDINDEX
+    )
+
 is_dt_const_field(fld::Int) = (
      fld == DATATYPE_NAME_FIELDINDEX ||
      fld == DATATYPE_PARAMETERS_FIELDINDEX ||
-     fld == DATATYPE_TYPES_FIELDINDEX ||
-     fld == DATATYPE_SUPER_FIELDINDEX ||
-     fld == DATATYPE_MUTABLE_FIELDINDEX ||
-     fld == DATATYPE_INSTANCE_FIELDINDEX
+     fld == DATATYPE_SUPER_FIELDINDEX
     )
+
 function const_datatype_getfield_tfunc(@nospecialize(sv), fld::Int)
     if fld == DATATYPE_INSTANCE_FIELDINDEX
         return isdefined(sv, fld) ? Const(getfield(sv, fld)) : Union{}
     elseif is_dt_const_field(fld)
         return Const(getfield(sv, fld))
+    elseif !getfield(sv, :incomplete) && is_dt_complete_const_field(fld)
+        return Const(getfield(sv, fld))
     end
     return nothing
 end

base/reflection.jl

@@ -428,6 +428,11 @@ function isprimitivetype(@nospecialize(t::Type))
     return !hasfield && t.size != 0 && !t.abstract
 end
 
+struct IncompleteTypeException
+    dt::DataType
+end
+must_be_complete(t::DataType) = (t.incomplete && throw(IncompleteTypeException(t)); t)
+
 """
     isbitstype(T)
 
@@ -449,14 +454,14 @@ julia> isbitstype(Complex)
 false
 ```
 """
-isbitstype(@nospecialize(t::Type)) = (@_pure_meta; isa(t, DataType) && t.isbitstype)
+isbitstype(@nospecialize(t::Type)) = (@_pure_meta; isa(t, DataType) && must_be_complete(t).isbitstype)
 
 """
     isbits(x)
 
 Return `true` if `x` is an instance of an `isbitstype` type.
 """
-isbits(@nospecialize x) = (@_pure_meta; typeof(x).isbitstype)
+isbits(@nospecialize x) = typeof(x).isbitstype
 
 """
     isdispatchtuple(T)
@@ -465,7 +470,7 @@ Determine whether type `T` is a tuple "leaf type",
 meaning it could appear as a type signature in dispatch
 and has no subtypes (or supertypes) which could appear in a call.
 """
-isdispatchtuple(@nospecialize(t)) = (@_pure_meta; isa(t, DataType) && t.isdispatchtuple)
+isdispatchtuple(@nospecialize(t)) = (@_pure_meta; isa(t, DataType) && must_be_complete(t).isdispatchtuple)
 
 iskindtype(@nospecialize t) = (t === DataType || t === UnionAll || t === Union || t === typeof(Bottom))
 isconcretedispatch(@nospecialize t) = isconcretetype(t) && !iskindtype(t)

src/ast.c

@@ -45,6 +45,7 @@ jl_sym_t *splatnew_sym;
 jl_sym_t *const_sym;   jl_sym_t *thunk_sym;
 jl_sym_t *abstracttype_sym; jl_sym_t *primtype_sym;
 jl_sym_t *structtype_sym;   jl_sym_t *foreigncall_sym;
+jl_sym_t *incompletetype_sym;
 jl_sym_t *global_sym; jl_sym_t *list_sym;
 jl_sym_t *dot_sym;    jl_sym_t *newvar_sym;
 jl_sym_t *boundscheck_sym; jl_sym_t *inbounds_sym;
@@ -334,6 +335,7 @@ void jl_init_frontend(void)
     abstracttype_sym = jl_symbol("abstract_type");
     primtype_sym = jl_symbol("primitive_type");
     structtype_sym = jl_symbol("struct_type");
+    incompletetype_sym = jl_symbol("incomplete_type");
     toplevel_sym = jl_symbol("toplevel");
     dot_sym = jl_symbol(".");
     colon_sym = jl_symbol(":");

src/builtins.c

@@ -794,6 +794,8 @@ static jl_value_t *get_fieldtype(jl_value_t *t, jl_value_t *f, int dothrow)
     if (!jl_is_datatype(t))
         jl_type_error("fieldtype", (jl_value_t*)jl_datatype_type, t);
     jl_datatype_t *st = (jl_datatype_t*)t;
+    if (st->incomplete)
+        jl_error("Type is incomplete");
     int field_index;
     if (jl_is_long(f)) {
         field_index = jl_unbox_long(f) - 1;

src/codegen.cpp

@@ -4248,7 +4248,7 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaval)
             return ghostValue(jl_void_type);
         }
         if (head == abstracttype_sym || head == structtype_sym ||
-            head == primtype_sym) {
+            head == primtype_sym || head == incompletetype_sym) {
             jl_errorf("type definition not allowed inside a local scope");
         }
         else {

src/datatype.c

@@ -88,7 +88,7 @@ jl_datatype_t *jl_new_uninitialized_datatype(void)
     t->has_concrete_subtype = 1;
     t->layout = NULL;
     t->names = NULL;
-    t->incomplete = 0;
+    t->incomplete = 1;
     return t;
 }
 
@@ -540,6 +540,7 @@ JL_DLLEXPORT jl_datatype_t *jl_new_primitivetype(jl_value_t *name, jl_module_t *
     bt->size = nbytes;
     bt->layout = jl_get_layout(0, alignm, 0, NULL);
     bt->instance = NULL;
+    bt->incomplete = 0;
     return bt;
 }
 
@@ -568,6 +569,7 @@ JL_DLLEXPORT jl_datatype_t * jl_new_foreign_type(jl_sym_t *name,
     desc->sweepfunc = sweepfunc;
     bt->layout = layout;
     bt->instance = NULL;
+    bt->incomplete = 0;
     return bt;
 }
 

src/dump.c

@@ -363,7 +363,8 @@ static void jl_serialize_datatype(jl_serializer_state *s, jl_datatype_t *dt) JL_
     write_int32(s->s, dt->size);
     int has_instance = (dt->instance != NULL);
     int has_layout = (dt->layout != NULL);
-    write_uint8(s->s, dt->abstract | (dt->mutabl << 1) | (has_layout << 2) | (has_instance << 3));
+    write_uint8(s->s, dt->abstract | (dt->mutabl << 1) | (has_layout << 2) |
+        (has_instance << 3) | (dt->incomplete << 4));
     write_uint8(s->s, dt->hasfreetypevars
             | (dt->isconcretetype << 1)
             | (dt->isdispatchtuple << 2)
@@ -1398,6 +1399,7 @@ static jl_value_t *jl_deserialize_datatype(jl_serializer_state *s, int pos, jl_v
     dt->mutabl = (flags >> 1) & 1;
     int has_layout = (flags >> 2) & 1;
     int has_instance = (flags >> 3) & 1;
+    dt->incomplete = (flags >> 4) & 1;
     dt->hasfreetypevars = memflags & 1;
     dt->isconcretetype = (memflags >> 1) & 1;
     dt->isdispatchtuple = (memflags >> 2) & 1;

src/gf.c

@@ -2367,6 +2367,7 @@ jl_function_t *jl_new_generic_function_with_supertype(jl_sym_t *name, jl_module_
     jl_set_const(module, tname, (jl_value_t*)ftype);
     jl_value_t *f = jl_new_struct(ftype);
     ftype->instance = f; jl_gc_wb(ftype, f);
+    ftype->incomplete = 0;
     JL_GC_POP();
     return (jl_function_t*)f;
 }

src/interpreter.c

@@ -166,6 +166,52 @@ static void eval_abstracttype(jl_expr_t *ex, interpreter_state *s)
     if (temp == NULL || !equiv_type(dt, (jl_datatype_t*)jl_unwrap_unionall(temp))) {
         jl_checked_assignment(b, w);
     }
+    dt->incomplete = 0;
+    JL_GC_POP();
+}
+
+static void eval_incompletetype(jl_expr_t *ex, interpreter_state *s)
+{
+    jl_value_t **args = jl_array_ptr_data(ex->args);
+    if (inside_typedef)
+        jl_error("cannot eval a new incomplete type definition while defining another type");
+    jl_value_t *name = args[0];
+    jl_value_t *para = eval_value(args[1], s);
+    jl_value_t *super = NULL;
+    jl_value_t *temp = NULL;
+    jl_datatype_t *dt = NULL;
+    jl_value_t *w = NULL;
+    jl_module_t *modu = s->module;
+    JL_GC_PUSH5(&para, &super, &temp, &w, &dt);
+    assert(jl_is_svec(para));
+    if (jl_is_globalref(name)) {
+        modu = jl_globalref_mod(name);
+        name = (jl_value_t*)jl_globalref_name(name);
+    }
+    assert(jl_is_symbol(name));
+    dt = jl_new_datatype((jl_sym_t*)name, modu, (jl_datatype_t*)super,
+                         (jl_svec_t*)para, NULL, NULL, 0, 0, 0);
+    w = dt->name->wrapper;
+    jl_binding_t *b = jl_get_binding_wr(modu, (jl_sym_t*)name, 1);
+    temp = b->value;
+    check_can_assign_type(b, w);
+    b->value = w;
+    jl_gc_wb_binding(b, w);
+    JL_TRY {
+        inside_typedef = 1;
+        super = eval_value(args[2], s);
+        jl_set_datatype_super(dt, super);
+        jl_reinstantiate_inner_types(dt);
+    }
+    JL_CATCH {
+        jl_reset_instantiate_inner_types(dt);
+        b->value = temp;
+        jl_rethrow();
+    }
+    b->value = temp;
+    if (temp == NULL || !equiv_type(dt, (jl_datatype_t*)jl_unwrap_unionall(temp))) {
+        jl_checked_assignment(b, w);
+    }
     JL_GC_POP();
 }
 
@@ -217,6 +263,7 @@ static void eval_primitivetype(jl_expr_t *ex, interpreter_state *s)
     if (temp == NULL || !equiv_type(dt, (jl_datatype_t*)jl_unwrap_unionall(temp))) {
         jl_checked_assignment(b, w);
     }
+    dt->incomplete = 0;
     JL_GC_POP();
 }
 
@@ -240,12 +287,44 @@ static void eval_structtype(jl_expr_t *ex, interpreter_state *s)
     assert(jl_is_symbol(name));
     assert(jl_is_svec(para));
     temp = eval_value(args[2], s);  // field names
-    dt = jl_new_datatype((jl_sym_t*)name, modu, NULL, (jl_svec_t*)para,
-                         (jl_svec_t*)temp, NULL,
-                         0, args[5]==jl_true ? 1 : 0, jl_unbox_long(args[6]));
+    jl_binding_t *b = jl_get_binding_wr(modu, (jl_sym_t*)name, 1);
+    jl_datatype_t *bdt = NULL;
+    if (b->value)
+        bdt = (jl_datatype_t *)jl_unwrap_unionall(b->value);
+    if (bdt &&
+        jl_is_datatype(bdt) &&
+        bdt->name->name == (jl_sym_t*)name &&
+        bdt->name->module == modu &&
+        bdt->incomplete) {
+        super = eval_value(args[3], s);
+        // Check that supertype matches
+        if (!jl_types_equal((jl_value_t*)bdt->super, super))
+            jl_error("Super type does not match forward declaration");
+        // Check that parameters match
+        if (jl_svec_len(para) != jl_svec_len(bdt->parameters)) {
+            jl_error("number of type parameters does not match forward declared type");
+        }
+        for (int i = 0; i < jl_svec_len(para); ++i) {
+            jl_tvar_t *v = (jl_tvar_t*)jl_svecref(para, i);
+            jl_tvar_t *w = (jl_tvar_t*)jl_svecref(bdt->parameters, i);
+            assert(jl_is_typevar(v) && jl_is_typevar(w));
+            if (v->name != w->name)
+                jl_errorf("Name `%s` of type parameter does not match name `%s` from forward declaration.",
+                    jl_symbol_name(v->name), jl_symbol_name(w->name));
+            else if (!jl_types_equal(v->lb, w->lb) || !jl_types_equal(v->ub, w->ub)) {
+                jl_errorf("Bounds of type parameter `%s` do not match forward declaration",
+                    jl_symbol_name(v->name));
+            }
+        }
+        dt = bdt;
+        dt->mutabl = args[5]==jl_true ? 1 : 0;
+    } else {
+        dt = jl_new_datatype((jl_sym_t*)name, modu, NULL, (jl_svec_t*)para,
+                            (jl_svec_t*)temp, NULL,
+                            0, args[5]==jl_true ? 1 : 0, jl_unbox_long(args[6]));
+    }
     w = dt->name->wrapper;
 
-    jl_binding_t *b = jl_get_binding_wr(modu, (jl_sym_t*)name, 1);
     temp = b->value;  // save old value
     // temporarily assign so binding is available for field types
     check_can_assign_type(b, w);
@@ -255,8 +334,10 @@ static void eval_structtype(jl_expr_t *ex, interpreter_state *s)
     JL_TRY {
         inside_typedef = 1;
         // operations that can fail
-        super = eval_value(args[3], s);
-        jl_set_datatype_super(dt, super);
+        if (!super) {
+            super = eval_value(args[3], s);
+            jl_set_datatype_super(dt, super);
+        }
         dt->types = (jl_svec_t*)eval_value(args[4], s);
         jl_gc_wb(dt, dt->types);
         for (size_t i = 0; i < jl_svec_len(dt->types); i++) {
@@ -280,6 +361,7 @@ static void eval_structtype(jl_expr_t *ex, interpreter_state *s)
     if (temp == NULL || !equiv_type(dt, (jl_datatype_t*)jl_unwrap_unionall(temp))) {
         jl_checked_assignment(b, w);
     }
+    dt->incomplete = 0;
 
     JL_GC_POP();
 }
@@ -745,6 +827,9 @@ SECT_INTERP static jl_value_t *eval_body(jl_array_t *stmts, interpreter_state *s
                 else if (head == structtype_sym) {
                     eval_structtype((jl_expr_t*)stmt, s);
                 }
+                else if (head == incompletetype_sym) {
+                    eval_incompletetype((jl_expr_t*)stmt, s);
+                }
                 else if (jl_is_toplevel_only_expr(stmt)) {
                     jl_toplevel_eval(s->module, stmt);
                 }