Merge ability to retry compilation if forwarded package is no longer available from Tristan Knoth.

Author: CuppoJava

compiler/dl-ir.stanza

@@ -6,6 +6,7 @@ defpackage stz/dl-ir :
   import stz/type-fargs
   import stz/absolute-info
   import stz/base-dir
+  import stz/printing-utils
 
 ;============================================================
 ;======================== PackageIO =========================
@@ -31,16 +32,29 @@ defmethod hash (p : ImportedPackage) :
 defmethod equal? (p1 : ImportedPackage, p2 : ImportedPackage) :
   equal?(package-name(p1), package-name(p2))
 
+;- forwarded-imports: If package A imports B, and B forwards C,
+;  then we say that C is imported into A via forwarding (assuming
+;  that C is not already directly imported into A).
+;  This field stores all packages that are imported via forwarding.
+;- direct-imports: These are the packages that are explicitly imported
+;  by the user via an import statement. This is a subset of the
+;  'imported-packages' field in that it does not include the packages
+;  that are implicitly imported due to type signatures.
 public defstruct PackageIO :
   package: Symbol
   imported-packages: Tuple<ImportedPackage> with: (updater => sub-imported-packages)
+  forwarded-imports: Tuple<Symbol>
+  direct-imports: Tuple<Symbol>
   imports: Tuple<Import> with: (updater => sub-imports)
   exports: Tuple<Export> with: (updater => sub-exports)
   documentation?:False|String with: (updater => sub-documentation)
 
+;Subset of PackageIO which does not include 'Import' field.
 public defstruct PackageExports :
   package: Symbol
   imported-packages: Tuple<ImportedPackage>
+  forwarded-imports: Tuple<Symbol>
+  direct-imports: Tuple<Symbol>
   exports: Tuple<Export> with: (updater => sub-exports)
 
 ;============================================================
@@ -442,7 +456,7 @@ public defn match? (x:Rec, y:Rec, types-in-io:HashSet<TypeId>|False) -> True|Fal
 ;============================================================
 
 public defn to-package-exports (io:PackageIO) :
-  PackageExports(package(io), imported-packages(io), exports(io))
+  PackageExports(package(io), imported-packages(io), forwarded-imports(io), direct-imports(io), exports(io))
 
 public defn to-package-exports? (io:PackageIO|False) :
   match(io:PackageIO) :
@@ -567,10 +581,26 @@ defmethod print (o:OutputStream, x:Export) :
     lnprint(o2, "doc: %_" % [documentation?(x)])
 
 defmethod print (o:OutputStream, x:PackageIO) :
-  print(o, "package %~ :" % [package(x)])
-  val o2 = IndentedStream(o)
-  lnprint(o2, "imported-packages = (%@)" % [imported-packages(x)])
-  lnprints(o2, cat(imports(x), exports(x)))
+  val items = [
+    falseable-field("documentation", documentation?(x))
+    named-list-fields("imported-packages", imported-packages(x))
+    line-wrapped-field("forwarded-imports", forwarded-imports(x))
+    line-wrapped-field("direct-imports", direct-imports(x))
+    inline-fields(imports(x))
+    inline-fields(exports(x))]
+  print(o, "package %~ %_" % [package(x), colon-field-list(items)])
+
+defmethod print (o:OutputStream, x:ImportedPackage) :
+  val items = [
+    simple-field("forward", forward(x))
+    falseable-field("only", only(x))
+    inline-fields(prefix(x))]
+  print(o, "import package %~ %_" % [package-name(x), colon-field-list(items)])
+
+defmethod print (o:OutputStream, x:ImportPrefix) :
+  match(names(x)) :
+    (names:Tuple<Symbol>) : print(o, "prefix(%,) => %~" % [names, prefix(x)])
+    (f:False) : print(o, "prefix => %~" % [prefix(x)])
 
 defmethod print (o:OutputStream, x:PackageExports) :
   print(o, "package %~ :" % [package(x)])
@@ -603,7 +633,8 @@ defsyntax dl-ir :
                          ?imports:#import ...
                          ?exports:#export ...) :
     ;[TODO] Add doc string reader macro. Defaulted to false for now.
-    PackageIO(name, to-tuple(map({ImportedPackage(_)}, ips)), to-tuple(imports), to-tuple(exports), false)
+    ;[TODO]: is the empty direct-import list OK?
+    PackageIO(name, to-tuple(map({ImportedPackage(_)}, ips)), [], [], to-tuple(imports), to-tuple(exports), false)
 
   defproduction dtype : DType
   defrule dtype = (byte) : DByte()

compiler/el-ir.stanza

@@ -1065,7 +1065,7 @@ defsyntax el-ir :
   defrule epackage = (package ?name:#symbol : (?ss:#tstmt ... #E)) :
     val ins = to-tuple(filter-by<Import>(ss))
     val exs = to-tuple(filter-by<Export>(ss))
-    val io = PackageIO(name, [], ins, exs, false)
+    val io = PackageIO(name, [], [], [], ins, exs, false)
     val es = to-tuple(filter-by<ETExp>(ss))
     EPackage(io, to-tuple(es))
 

compiler/el.stanza

@@ -262,7 +262,7 @@ defn collapse (epackages:Tuple<EPackage>) -> EPackage :
         for ex in exports(packageio(e)) seq :
           val n* = global-rec-ids[id(rec(ex))]
           sub-n(ex, n*)
-    PackageIO(`prog, [], [], exports*, false)
+    PackageIO(`prog, [], [], [], [], exports*, false)
 
   ;Rename one specific package
   defn renamed-exps (epackage:EPackage) :

compiler/front-end.stanza

@@ -540,11 +540,12 @@ defn FrontEnd (sys:FrontEndInputs) -> FrontEnd :
     val resolved = resolve-packages(package-names, errorlist)
     val new-inputs = HashTable<Symbol,PackageInFile>()
     for e in errorlist do :
-      match(e:MissingType) :
+      match(e:MissingType|MissingForwardedPackage) :
         match(source-file?(src-package(e))) :
           (file:String) : new-inputs[src-package(e)] = PackageInFile(src-package(e), file)
           (file:False) : throw(FrontEndErrors(errorlist))
-      else : throw(FrontEndErrors(errorlist))
+      else :
+        throw(FrontEndErrors(errorlist))
     [resolved, to-tuple(values(new-inputs))]
 
   defn resolve-packages! (package-names:Tuple<Symbol>) -> ResolverResult :

compiler/il-ir.stanza

@@ -10,10 +10,17 @@ defpackage stz/il-ir :
 ;General Multis for IExp
 public defmulti info (e:IExp) -> False|AbsoluteFileInfo
 
+;- imports: Corresponds one-to-one with the user's 'import' statements
+;  nested under 'defpackage'. 
+;- forwarded-imports: If package A imports B, and B forwards C,
+;  then we say that C is imported into A via forwarding (assuming
+;  that C is not already directly imported into A).
+;  This field stores all packages that are imported via forwarding.
 public defstruct IPackage :
   name: Symbol
   documentation?: IDoc|False
   imports: Tuple<IImport> with: (updater => sub-imports)
+  forwarded-imports: Tuple<Symbol> with: (updater => sub-forwarded-imports)
   exps: List<IExp> with: (updater => sub-exps)
   info: AbsoluteFileInfo|False
   namemap: NameMap with: (default => void, updater => sub-namemap)

compiler/il-to-tl.stanza

@@ -169,11 +169,12 @@ defn to-imported-package (i:IImport) -> ImportedPackage :
 ;The imported/exported entries will be populated
 ;after type inference.
 defn to-packageio (ipackage:IPackage) -> PackageIO :
+  val direct-imports = to-tuple(seq(package, imports(ipackage)))
   val imports = to-tuple(seq(to-imported-package, imports(ipackage)))
   val doc? = match(documentation?(ipackage)) :
     (d:IDoc) : value(string(d) as ILiteral) as String
     (f:False) : false
-  PackageIO(name(ipackage), imports, [], [], doc?)
+  PackageIO(name(ipackage), imports, forwarded-imports(ipackage), direct-imports, [], [], doc?)
 
 ;============================================================
 ;=============== Conversion Functions =======================

compiler/input.stanza

@@ -468,23 +468,23 @@ defn split-packages (e:IExp,
           val e = all-exps[i + 1] as IDefPackage
           val [exps, end] = package-exps(i + 2)
           val imports = to-tuple $ seq(to-iimport, imports(e) as List<IImportExp>)
-          add(packages, IPackage(name!(e), doc, imports, exps, info(e)))
+          add(packages, IPackage(name!(e), doc, imports, [], exps, info(e)))
           loop(end)
 
         ;Package without preceding IDoc form.
         DefPackageExp:
           val e = all-exps[i] as IDefPackage
           val [exps, end] = package-exps(i + 1)
           val imports = to-tuple $ seq(to-iimport, imports(e) as List<IImportExp>)
-          add(packages, IPackage(name!(e), false, imports, exps, info(e)))
+          add(packages, IPackage(name!(e), false, imports, [], exps, info(e)))
           loop(end)
 
         ;Default package.
         WithinPackageExp:
           val [exps, end] = package-exps(i)
           val info = info(head(exps)) when not empty?(exps)
           val name = gensym(`default)
-          add(packages, IPackage(name, false, default-imports, exps, info))
+          add(packages, IPackage(name, false, default-imports, [], exps, info))
           loop(end)
 
   ;Return packages

compiler/pkg-serializer.stanza

@@ -154,6 +154,8 @@ defserializer PkgSerializer (include-asm?:True|False) :
   deftype packageio (PackageIO) :
     package:symbol
     imported-packages:tuple(importedpackage)
+    forwarded-imports:tuple(symbol)
+    direct-imports:tuple(symbol)
     imports:tuple(dimport)
     exports:tuple(dexport)
     documentation?:opt(string)

compiler/repl.stanza

@@ -797,6 +797,7 @@ defn REPLEnv () :
     IPackage(gensym(`repl),
              false,
              to-tuple(values(exp-imports)),
+             []
              List(wrap-display-result(exp)),
              info(exp))
 

compiler/resolver.stanza

@@ -167,7 +167,7 @@ public defn resolve-il (packages:Tuple<IPackage|PackageExports>, env:Env) -> Res
 
     ;Resolve type dependencies from pkgs
     check-exported-types(symtables, ipackages*, package-exports(symtables), error-accum)
-
+    check-forwarded-imports(symtables, package-exports(symtables), error-accum)
 
     ;Compute initialization order
     defn init-order () :
@@ -228,7 +228,9 @@ defn resolve-package (ipackage:IPackage,
     map(resolve-exp{_, eng}, exps(ipackage))
   val ipackage* = sub-exps(ipackage, exps*)
   if length(error-accum) == num-errors :
-    add-qualified-imports(ipackage*, qualified-imports(eng))
+    ipackage*
+      $> add-qualified-imports{_, qualified-imports(eng)}
+      $> compute-forwarded-imports{_, symtables}
 
 ;<doc>=======================================================
 ;=================== Check Type Dependencies ================
@@ -260,7 +262,7 @@ issue an error if it doesn't.
 defn check-exported-types (symtables:SymbolTables, pkgs:Collection<IPackage>, pexs:Tuple<PackageExports>, error-accum:Vector<ResolveError>) :
   ;Ensure that all types referenced in exports are defined
   for pex in pexs do :
-    ;Check for existence of typeid
+   ;Check for existence of typeid
     defn check-for-type (e:TypeId) :
       val symtable = symtables[package(e)]
       match(symtable:SymbolTable) :
@@ -276,6 +278,41 @@ defn check-exported-types (symtables:SymbolTables, pkgs:Collection<IPackage>, pe
     ;Scan all exports
     do(scan-for-type, exports(pex))
 
+;<doc>=======================================================
+;================ Check Forwarded Imports ===================
+;============================================================
+
+At the time of creation, each .pkg file stores the packages
+that were imported via forwarding.
+
+This check ensures that for each of the .pkg files, any
+package that was previously imported via forwarding is still
+imported via forwarding.
+
+;============================================================
+;=======================================================<doc>
+
+defn check-forwarded-imports (symtables:SymbolTables,
+                              pexs:Tuple<PackageExports>,
+                              error-accum:Vector<ResolveError>) ->
+                              False :
+
+  ;For each PackageExports.
+  for pex in pexs do :
+
+    ;Compute the set of packages that are currently forwarded to the .pkg file.
+    val currently-forwarded = forwarded-imports(symtables, direct-imports(pex))
+
+    ;Retrieve the set of packages that were previously forwarded to the .pkg file
+    ;at the time the .pkg file was originally compiled.
+    val previously-forwarded = forwarded-imports(pex)
+
+    ;Ensure that any package that was previously imported via forwarding,
+    ;is still imported via forwarding.
+    for forwarded in previously-forwarded do :
+      if not currently-forwarded[forwarded] :
+        add(error-accum, MissingForwardedPackage(package(pex), forwarded))
+
 ;============================================================
 ;===================== Defresolver ==========================
 ;============================================================
@@ -686,6 +723,39 @@ defn add-qualified-imports (ipackage:IPackage, imps:Tuple<Symbol>) :
   ;Add imports
   sub-imports(ipackage, to-tuple(imports))
 
+;Compute the 'forwarded-inputs' field for the given IPackage.
+;Finds all packages that are imported via forwarding.
+defn compute-forwarded-imports (ipackage:IPackage,
+                                symtables:SymbolTables) ->
+                                IPackage :
+  val directly-imported = to-tuple(seq(package-name, imports(ipackage)))
+  val forwarded = forwarded-imports(symtables, directly-imported)
+  sub-forwarded-imports(ipackage, to-tuple(forwarded))
+
+;Given the list of direct imports, compute the set of
+;all additional packages that are imported via forwarding.
+defn forwarded-imports (symtables:SymbolTables,
+                        direct-imports:Seqable<Symbol>) ->
+                        HashSet<Symbol> :
+  val forwarded-set = within package = transitive-set(direct-imports) :
+    seq(package-name, filter(forward,imports(symtables,package)))
+  do(remove{forwarded-set, _}, direct-imports)
+  forwarded-set
+
+;Helper: Return the name of the package being imported.
+defn package-name (i:IImport) : package(i)
+
+;Returns all symbols reachable from the initial set 'xs'
+;via the 'next' relation.
+defn transitive-set (next:Symbol -> Seqable<Symbol>,
+                     xs:Seqable<Symbol>) ->
+                     HashSet<Symbol> :
+  val hashset = HashSet<Symbol>()
+  defn scan (x:Symbol) :
+    do(scan,next(x)) when add(hashset,x)
+  do(scan, xs)
+  hashset
+
 ;============================================================
 ;=================== Symbol Tables ==========================
 ;============================================================
@@ -696,6 +766,10 @@ defmulti package-exports (st:SymbolTables) -> Tuple<PackageExports>
 defmulti new-ipackages (st:SymbolTables) -> Tuple<IPackage>
 defmulti load-conditional-dependencies (st:SymbolTables) -> Vector<Symbol>
 
+;Return the explicit import statements (ImportedPackage) listed
+;in the given package. Returns empty if the package could not be loaded.
+defmulti imports (st:SymbolTables, package:Symbol) -> Tuple<IImport|ImportedPackage>
+
 ; Three different import modes:
 ; - "import" mode: copy all imported definitions into symbol table.
 ;   Corresponds to "import" clause
@@ -907,6 +981,11 @@ defn SymbolTables (packages:Tuple<IPackage|PackageExports>, env:Env, errors:Vect
   new SymbolTables :
     defmethod get (this, package:Symbol) :
       symtables[package]
+    defmethod imports (this, package:Symbol) :
+      match(load-package(package)) :
+        (p:IPackage) : imports(p)
+        (p:PackageExports) : imported-packages(p)
+        (f:False) : []
     defmethod package-exports (this) :
       to-tuple $ filter-by<PackageExports>(values(pkg-table))
     defmethod new-ipackages (this) :
@@ -955,7 +1034,7 @@ defn SymbolTable (package-name:Symbol, base:NameMap|Tuple<Export>) :
   val basetable = HashTable<Symbol, List<BaseEntry>>(List())
 
 
-    
+
   ; Update an entry in the basetable
   ; Assumes the resulting entry should be forwarded
   defn forward-base-entry (e:VEntry, f, xs:List<BaseEntry>) -> List<BaseEntry> :
@@ -964,8 +1043,8 @@ defn SymbolTable (package-name:Symbol, base:NameMap|Tuple<Export>) :
     else : cons(BaseEntry(f(e), true), xs)
 
   ; Forward symbol table entry to new-package under some prefix by updating its package field
-  ; Concretely: 
-  ;   Search for existing entry e, update package field if found. 
+  ; Concretely:
+  ;   Search for existing entry e, update package field if found.
   ;   If not found, simply update package field in e and add
   defn forward-entry (e:VEntry, new-package:Symbol, prefix:String|False) :
     defn fwd (e) : sub-package(e, new-package) ; Update package field
@@ -986,7 +1065,7 @@ defn SymbolTable (package-name:Symbol, base:NameMap|Tuple<Export>) :
   ;   already been added
   defn add-unique<?E> (e:?E&Equalable, entries:List<E&Equalable>) -> List<E> :
     if contains?(entries, e) : entries
-    else : cons(e, entries)  
+    else : cons(e, entries)
 
   ; Add a VEntry to the symbol table under its prefix
   defn add-entry (e:VEntry, prefix:String|False) :
@@ -1203,6 +1282,15 @@ public defstruct NotCapVar <: ResolveError :
 public defstruct MissingType <: ResolveError :
   src-package:Symbol
   typeid:TypeId
+
+;Indicates that a package expected to be imported via forwarding
+;is missing.
+;- src-package: The package that is being analyzed by the resolver.
+;- imported-package: The package that is being imported (via
+;  forwarding) that is missing.
+public defstruct MissingForwardedPackage <: ResolveError :
+  src-package:Symbol
+  imported-package:Symbol
 public defstruct ForwardingCycle <: ResolveError :
   src-packages:Seq<Symbol>
 
@@ -1214,6 +1302,10 @@ defmethod print (o:OutputStream, e:MissingType) :
   val FMT = "Missing type: The %~ package requires the %_/%_ type to be defined."
   print(o, FMT % [src-package(e), package(typeid(e)), name(typeid(e))])
 
+defmethod print (o:OutputStream, e:MissingForwardedPackage) :
+  val FMT = "Missing forwarded package: The %~ package needs the %~ package."
+  print(o, FMT % [src-package(e), imported-package(e)])
+
 defmethod print (o:OutputStream, e:NoResolve) :
   val FMT = "%_Could not resolve %~."
   print(o, FMT % [infostr(e), name(e)])

compiler/tl-to-el.stanza

@@ -1309,6 +1309,8 @@ defn analyze-imports (p:EPackage, export-table:ExportTable, transient-package?:T
     add-all(pset, seq(ImportedPackage{package(id(rec(_)))}, new-imports))
   val io* = PackageIO(name(p),
                       new-imported-packages,
+                      forwarded-imports(io)
+                      direct-imports(io),
                       to-tuple(new-imports),
                       exports(io),
                       documentation?(io))