[ESI][BSP] Emit engine records (#8204)

Instead of trying to shoehorn all of the engine records into one service
record (for the ChannelEngineService), add the capability to emit
individual engine records.

frontends/PyCDE/src/pycde/bsp/common.py

@@ -733,7 +733,12 @@ def generate(ports, bundles: esi._ServiceGeneratorBundles):
 def DummyToHostEngine(client_type: Type) -> type['DummyToHostEngineImpl']:
   """Create a fake DMA engine which just throws everything away."""
 
-  class DummyToHostEngineImpl(Module):
+  class DummyToHostEngineImpl(esi.EngineModule):
+
+    @property
+    def TypeName(self):
+      return "DummyToHostEngine"
+
     clk = Clock()
     rst = Reset()
     input_channel = InputChannel(client_type)
@@ -749,7 +754,12 @@ def build(ports):
 def DummyFromHostEngine(client_type: Type) -> type['DummyFromHostEngineImpl']:
   """Create a fake DMA engine which just never produces messages."""
 
-  class DummyFromHostEngineImpl(Module):
+  class DummyFromHostEngineImpl(esi.EngineModule):
+
+    @property
+    def TypeName(self):
+      return "DummyFromHostEngine"
+
     clk = Clock()
     rst = Reset()
     output_channel = OutputChannel(client_type)
@@ -781,23 +791,38 @@ class ChannelEngineService(esi.ServiceImplementation):
 
     @generator
     def build(ports, bundles: esi._ServiceGeneratorBundles):
+
+      def build_engine_appid(client_appid: List[esi.AppID],
+                             channel_name: str) -> esi.AppID:
+        appid_strings = [str(appid) for appid in client_appid]
+        return esi.AppID(f"{'_'.join(appid_strings)}.{channel_name}")
+
       for bundle in bundles.to_client_reqs:
         bundle_type = bundle.type
         to_channels = {}
         # Create a DMA engine for each channel headed TO the client (from the host).
         for bc in bundle_type.channels:
           if bc.direction == ChannelDirection.TO:
-            engine = from_host_engine_gen(bc.channel.inner_type)(clk=ports.clk,
-                                                                 rst=ports.rst)
+            eng_appid = build_engine_appid(bundle.client_name, bc.name)
+            engine_mod = from_host_engine_gen(bc.channel.inner_type)
+            engine = engine_mod(clk=ports.clk, rst=ports.rst, appid=eng_appid)
             to_channels[bc.name] = engine.output_channel
+            engine_rec = bundles.emit_engine(engine)
+            engine_rec.add_record(bundle, {bc.name: {"engine_inst": eng_appid}})
 
         client_bundle_sig, froms = bundle_type.pack(**to_channels)
         bundle.assign(client_bundle_sig)
 
         # Create a DMA engine for each channel headed FROM the client (to the host).
         for bc in bundle_type.channels:
           if bc.direction == ChannelDirection.FROM:
-            engine = to_host_engine_gen(bc.channel.inner_type)
-            engine(clk=ports.clk, rst=ports.rst, input_channel=froms[bc.name])
+            eng_appid = build_engine_appid(bundle.client_name, bc.name)
+            engine_mod = to_host_engine_gen(bc.channel.inner_type)
+            engine = engine_mod(appid=eng_appid,
+                                clk=ports.clk,
+                                rst=ports.rst,
+                                input_channel=froms[bc.name])
+            engine_rec = bundles.emit_engine(engine)
+            engine_rec.add_record(bundle, {bc.name: {"engine_inst": eng_appid}})
 
   return ChannelEngineService

frontends/PyCDE/src/pycde/esi.py

@@ -129,6 +129,22 @@ def Cosim(decl: ServiceDecl, clk, rst):
   decl.instantiate_builtin(AppID("cosim", 0), "cosim", [], [clk, rst])
 
 
+class EngineModule(Module):
+  """A module which implements an ESI engines. Engines have the responsibility
+  of transporting messages between two different devices."""
+
+  @property
+  def TypeName(self):
+    assert False, "Engine modules must have a TypeName property."
+
+  def __init__(self, appid: AppID, **inputs):
+    super(EngineModule, self).__init__(appid=appid, **inputs)
+
+  @property
+  def appid(self) -> AppID:
+    return AppID(self.inst.attributes["esi.appid"])
+
+
 class NamedChannelValue(ChannelSignal):
   """A ChannelValue with the name of the client request."""
 
@@ -189,14 +205,48 @@ def assign(self, new_value: ChannelSignal):
           f"Channel type mismatch. Expected {self.type}, got {new_value.type}.")
     msft.replaceAllUsesWith(self._bundle_to_replace, new_value.value)
     self._bundle_to_replace = None
-    self.req = None
 
   def cleanup(self):
     """Null out all the references to all the ops to allow them to be GC'd."""
     self.req = None
     self.rec = None
 
 
+class EngineServiceRecord:
+  """Represents a record in the engine section of the manifest."""
+
+  def __init__(self,
+               engine: EngineModule,
+               details: Optional[Dict[str, object]] = None):
+    rec_appid = AppID(f"{engine.appid.name}_record", engine.appid.index)
+    self._rec = raw_esi.ServiceImplRecordOp(appID=rec_appid._appid,
+                                            serviceImplName=engine.TypeName,
+                                            implDetails=details,
+                                            isEngine=True)
+    self._rec.regions[0].blocks.append()
+
+  def add_record(self,
+                 client: _OutputBundleSetter,
+                 channel_assignments: Optional[Dict] = None,
+                 details: Optional[Dict[str, object]] = None):
+    """Add a record to the manifest for this client request. Generally used to
+    give the runtime necessary information about how to connect to the client
+    through the generated service. For instance, offsets into an MMIO space."""
+
+    channel_assignments = optional_dict_to_dict_attr(channel_assignments)
+    details = optional_dict_to_dict_attr(details)
+
+    with get_user_loc(), ir.InsertionPoint.at_block_begin(
+        self._rec.reqDetails.blocks[0]):
+      raw_esi.ServiceImplClientRecordOp(
+          client.req.relativeAppIDPath,
+          client.req.servicePort,
+          ir.TypeAttr.get(client.req.toClient.type),
+          channelAssignments=channel_assignments,
+          implDetails=details,
+      )
+
+
 class _ServiceGeneratorBundles:
   """Provide access to the bundles which the service generator is responsible
   for connecting up."""
@@ -220,6 +270,14 @@ def __init__(self, mod: ModuleLikeBuilderBase,
     ]
     assert len(self._output_reqs) == len(req.results) - num_output_ports
 
+  def emit_engine(self,
+                  engine: EngineModule,
+                  details: Dict[str, object] = None):
+    """Emit and return an engine record."""
+    details = optional_dict_to_dict_attr(details)
+    with get_user_loc(), ir.InsertionPoint(self._rec):
+      return EngineServiceRecord(engine, details)
+
   @property
   def to_client_reqs(self) -> List[_OutputBundleSetter]:
     return self._output_reqs

lib/Dialect/ESI/runtime/cpp/lib/Services.cpp

@@ -232,10 +232,7 @@ BundlePort *CallService::getPort(AppIDPath id, const BundleType *type) const {
 
 CallService::Callback::Callback(AcceleratorConnection &acc, AppID id,
                                 const BundleType *type, PortMap channels)
-    : ServicePort(id, type, channels), acc(acc) {
-  if (channels.size() != 2)
-    throw std::runtime_error("CallService must have exactly two channels");
-}
+    : ServicePort(id, type, channels), acc(acc) {}
 
 CallService::Callback *CallService::Callback::get(AcceleratorConnection &acc,
                                                   AppID id, BundleType *type,