GG: emit a fileset-grouped manifest

davidbiancolin · davidbiancolin · commit 7aa9235b521c · 2022-07-02T07:13:52.000Z
diff --git a/sim/firesim-lib/src/main/scala/passes/EC2F1Artefacts.scala b/sim/firesim-lib/src/main/scala/passes/EC2F1Artefacts.scala
@@ -5,7 +5,7 @@ package firesim.passes
 import firesim.util.{BuildStrategy}
 import midas.{DesiredHostFrequency}
 import midas.stage.phases.ConfigParametersAnnotation
-import midas.stage.GoldenGateOutputFileAnnotation
+import midas.stage.{GoldenGateOutputFileAnnotation, DownstreamFlows}
 
 import freechips.rocketchip.config.Parameters
 
@@ -31,7 +31,7 @@ object EC2F1Artefacts extends Transform {
 set desired_host_frequency ${requestedFrequency}
 ${buildStrategy.emitTcl}
 """
-    GoldenGateOutputFileAnnotation(constraints, fileSuffix = ".env.tcl")
+    GoldenGateOutputFileAnnotation(constraints, fileSuffix = ".env.tcl", downstreamDependencies = Set(DownstreamFlows.BitstreamCompile))
   }
 
   def execute(state: CircuitState): CircuitState = {
diff --git a/sim/midas/src/main/scala/midas/passes/AutoILATransform.scala b/sim/midas/src/main/scala/midas/passes/AutoILATransform.scala
@@ -4,7 +4,7 @@ package midas.passes
 
 import midas.{EnableAutoILA, ILADepthKey, ILAProbeTriggersKey}
 import midas.targetutils.FirrtlFpgaDebugAnnotation
-import midas.stage.GoldenGateOutputFileAnnotation
+import midas.stage.{GoldenGateOutputFileAnnotation, DownstreamFlows}
 import midas.stage.phases.ConfigParametersAnnotation
 
 import firrtl._
@@ -181,6 +181,7 @@ object AutoILATransform extends Transform with DependencyAPIMigration {
           |  CONFIG.ALL_PROBE_SAME_MU_CNT {$probeTriggers}] [get_ips ${ilaBlackBoxName}]
           |""".stripMargin,
       s".${ilaBlackBoxName}.ipgen.tcl",
+      downstreamDependencies = Set(DownstreamFlows.BitstreamCompile)
     )
 
     val ilaWrapperModuleName = circuitNamespace.newName("ila_wrapper")
diff --git a/sim/midas/src/main/scala/midas/passes/SimulationMapping.scala b/sim/midas/src/main/scala/midas/passes/SimulationMapping.scala
@@ -24,7 +24,7 @@ import freechips.rocketchip.diplomacy.LazyModule
 
 import midas.core._
 import midas.platform.PlatformShim
-import midas.stage.{OutputFileBuilder, GoldenGateOutputFileAnnotation}
+import midas.stage.{OutputFileBuilder, GoldenGateOutputFileAnnotation, DownstreamFlows}
 
 private[passes] class SimulationMapping(targetName: String) extends firrtl.Transform {
   def inputForm = LowForm
@@ -39,7 +39,9 @@ private[passes] class SimulationMapping(targetName: String) extends firrtl.Trans
         |// This contains target-specific preprocessor macro definitions,
         |// and encodes all required bridge metadata to instantiate bridge drivers.
         |""".stripMargin,
-      fileSuffix = ".const.h")
+      fileSuffix = ".const.h",
+      downstreamDependencies = Set(DownstreamFlows.MetasimulatorCompile, DownstreamFlows.DriverCompile)
+      )
     csb append "#ifndef __%s_H\n".format(targetName.toUpperCase)
     csb append "#define __%s_H\n".format(targetName.toUpperCase)
     c.genHeader(csb.getBuilder, targetName)
@@ -50,7 +52,9 @@ private[passes] class SimulationMapping(targetName: String) extends firrtl.Trans
         |// This file encodes variable width fields used in MIDAS-level simulation
         |// and is not used in FPGA compilation flows.
         |""".stripMargin,
-      fileSuffix = ".const.vh")
+      fileSuffix = ".const.vh",
+      downstreamDependencies = Set(DownstreamFlows.MetasimulatorCompile),
+    )
 
     vsb append "`ifndef __%s_H\n".format(targetName.toUpperCase)
     vsb append "`define __%s_H\n".format(targetName.toUpperCase)
diff --git a/sim/midas/src/main/scala/midas/passes/WriteXDCFile.scala b/sim/midas/src/main/scala/midas/passes/WriteXDCFile.scala
@@ -7,8 +7,9 @@ import firrtl.stage.Forms
 import firrtl.annotations._
 import firrtl.analyses.InstanceKeyGraph
 
-import midas.stage.{GoldenGateFileEmission}
+import midas.stage.{GoldenGateFileEmission, DownstreamFlows}
 import midas.targetutils.xdc._
+import midas.stage.DownstreamFlows
 
 /**
   * We could reuse [[GoldenGateOutputFileAnnotation]] here, but this makes it
@@ -17,6 +18,7 @@ import midas.targetutils.xdc._
 private[midas] case class XDCOutputAnnotation(fileBody: String, suffix: Option[String])
     extends NoTargetAnnotation with GoldenGateFileEmission {
   def getBytes = fileBody.getBytes
+  def downstreamDependencies = Set(DownstreamFlows.BitstreamCompile)
 }
 
 private[midas] object WriteXDCFile extends Transform with DependencyAPIMigration with XDCAnnotationConstants {
diff --git a/sim/midas/src/main/scala/midas/platform/F1Shim.scala b/sim/midas/src/main/scala/midas/platform/F1Shim.scala
@@ -10,7 +10,7 @@ import freechips.rocketchip.util.HeterogeneousBag
 
 import midas.core.{DMANastiKey}
 import midas.widgets.{AXI4Printf, CtrlNastiKey}
-import midas.stage.GoldenGateOutputFileAnnotation
+import midas.stage.{GoldenGateOutputFileAnnotation, DownstreamFlows}
 import midas.targetutils.xdc._
 
 case object AXIDebugPrint extends Field[Boolean]
@@ -52,7 +52,8 @@ class F1Shim(implicit p: Parameters) extends PlatformShim {
           |`define USE_DDR_CHANNEL_B ${channelInUse(2)}
           |`define USE_DDR_CHANNEL_D ${channelInUse(3)}
           |""".stripMargin,
-      fileSuffix = ".defines.vh")
+      fileSuffix = ".defines.vh",
+      downstreamDependencies = Set(DownstreamFlows.BitstreamCompile))
 
     SpecifyXDCCircuitPaths(Some("firesim_top"), Some("WRAPPER_INST/CL/firesim_top"))
   }
diff --git a/sim/midas/src/main/scala/midas/platform/VitisShim.scala b/sim/midas/src/main/scala/midas/platform/VitisShim.scala
@@ -11,7 +11,7 @@ import freechips.rocketchip.util.HeterogeneousBag
 
 import midas.core.{DMANastiKey, HostMemChannelKey}
 import midas.widgets.{AXI4Printf, CtrlNastiKey}
-import midas.stage.GoldenGateOutputFileAnnotation
+import midas.stage.{GoldenGateOutputFileAnnotation, DownstreamFlows}
 import midas.platform.xilinx._
 import midas.targetutils.xdc._
 
@@ -95,8 +95,13 @@ class VitisShim(implicit p: Parameters) extends PlatformShim {
     GoldenGateOutputFileAnnotation.annotateFromChisel(
       s"// Vitis Shim requires no dynamically generated macros \n",
       fileSuffix = ".defines.vh",
+      downstreamDependencies = Set(DownstreamFlows.BitstreamCompile)
+    )
+    GoldenGateOutputFileAnnotation.annotateFromChisel(
+      s"# Currently unused",
+      ".env.tcl",
+      downstreamDependencies = Set(DownstreamFlows.BitstreamCompile)
     )
-    GoldenGateOutputFileAnnotation.annotateFromChisel(s"# Currenty unused", ".env.tcl")
     // We don't need to provide paths because
     // 1) The Shim module is the top-level of the kernel
     // 2) Implementation constraints are scoped to the kernel level in our vitis flow
diff --git a/sim/midas/src/main/scala/midas/platform/xilinx/AXI4Crossing.scala b/sim/midas/src/main/scala/midas/platform/xilinx/AXI4Crossing.scala
@@ -4,6 +4,7 @@ import chisel3._
 import chisel3.util._
 import freechips.rocketchip.amba.axi4._
 import midas.stage.GoldenGateOutputFileAnnotation
+import midas.stage.DownstreamFlows
 
 /** An AXI4 bundle definition whose names should match the interfaces exposed on Xilinx IP blocks. aresetn and clock are
   * omitted, and no user fields are provided.
@@ -209,5 +210,6 @@ class AXI4ClockConverter(
         |             [get_ips ${desiredName}]
         |""".stripMargin,
     s".${desiredName}.ipgen.tcl",
+    Set(DownstreamFlows.BitstreamCompile)
   )
 }
diff --git a/sim/midas/src/main/scala/midas/platform/xilinx/Clocking.scala b/sim/midas/src/main/scala/midas/platform/xilinx/Clocking.scala
@@ -1,7 +1,7 @@
 package midas.platform.xilinx
 
 import chisel3._
-import midas.stage.GoldenGateOutputFileAnnotation
+import midas.stage.{DownstreamFlows, GoldenGateOutputFileAnnotation}
 
 class MMCM(inputFreqMHz: Double, outputFreqMHz: Double, override val desiredName: String) extends BlackBox {
   val io = IO(new Bundle {
@@ -27,5 +27,6 @@ class MMCM(inputFreqMHz: Double, outputFreqMHz: Double, override val desiredName
         |             [get_ips ${desiredName}]
         |""".stripMargin,
     s".${desiredName}.ipgen.tcl",
+    Set(DownstreamFlows.BitstreamCompile),
   )
 }
diff --git a/sim/midas/src/main/scala/midas/stage/GoldenGateFileEmission.scala b/sim/midas/src/main/scala/midas/stage/GoldenGateFileEmission.scala
@@ -2,15 +2,31 @@
 
 package midas.stage
 
+import firrtl.AnnotationSeq
 import firrtl.annotations.{NoTargetAnnotation, Annotation}
 import firrtl.options.{CustomFileEmission}
 
 import chisel3.experimental.{annotate, ChiselAnnotation}
 
+object DownstreamFlows {
+  sealed trait FlowType 
+  case object BitstreamCompile extends FlowType
+  case object MetasimulatorCompile extends FlowType
+  case object DriverCompile extends FlowType
+  case object RuntimeDeployment extends FlowType
+  val allFlows = Seq(BitstreamCompile, MetasimulatorCompile, RuntimeDeployment, DriverCompile)
+
+  val fileManifestSuffix = ".file-manifest.json"
+  val emittedVerilogKey = "EmittedVerilogFiles"
+  val unclassifiedKey   = "UnclassifiedFiles"
+}
+
 trait GoldenGateFileEmission extends CustomFileEmission { this: Annotation =>
   override def baseFileName(annotations: firrtl.AnnotationSeq) = {
     annotations.collectFirst{ case OutputBaseFilenameAnnotation(name) => name }.get
   }
+  /** Used to build a manifest that organizes output files based on what downstream flows consume it. **/
+  def downstreamDependencies: Set[DownstreamFlows.FlowType]
 }
 
 /**
@@ -20,7 +36,10 @@ trait GoldenGateFileEmission extends CustomFileEmission { this: Annotation =>
   * @param fileSuffix The string to append to base output file name
   *
   */
-case class GoldenGateOutputFileAnnotation(body: String, fileSuffix: String)
+case class GoldenGateOutputFileAnnotation(
+  body: String,
+  fileSuffix: String,
+  downstreamDependencies: Set[DownstreamFlows.FlowType])
     extends NoTargetAnnotation with GoldenGateFileEmission {
   def suffix = Some(fileSuffix)
   def getBytes = body.getBytes
@@ -30,17 +49,17 @@ object GoldenGateOutputFileAnnotation {
   /**
     * Sugar to add a new output file from a chisel source (e.g., in a bridge, platform shim)
     */
-  def annotateFromChisel(body: String, fileSuffix: String): Unit = {
-    annotate(new ChiselAnnotation { def toFirrtl = GoldenGateOutputFileAnnotation(body, fileSuffix) })
+  def annotateFromChisel(body: String, fileSuffix: String, downstreamDependencies: Set[DownstreamFlows.FlowType]): Unit = {
+    annotate(new ChiselAnnotation { def toFirrtl = GoldenGateOutputFileAnnotation(body, fileSuffix, downstreamDependencies) })
   }
 }
 
 /**
   * Wraps a StringBuilder to incrementally build up an output file annotation.
   */
-class OutputFileBuilder(header: String, fileSuffix: String) {
+class OutputFileBuilder(header: String, fileSuffix: String, downstreamDependencies: Set[DownstreamFlows.FlowType]) {
   private val sb = new StringBuilder(header)
   def getBuilder = sb
   def append(str: String) = sb.append(str)
-  def toAnnotation = GoldenGateOutputFileAnnotation(sb.toString, fileSuffix)
+  def toAnnotation = GoldenGateOutputFileAnnotation(sb.toString, fileSuffix, downstreamDependencies)
 }
diff --git a/sim/midas/src/main/scala/midas/stage/GoldenGateStage.scala b/sim/midas/src/main/scala/midas/stage/GoldenGateStage.scala
@@ -21,7 +21,9 @@ class GoldenGateStage extends Stage with PreservesAll[Phase] {
         new firrtl.stage.phases.AddDefaults,
         new firrtl.stage.phases.Checks,
         new firrtl.stage.phases.AddCircuit,
-        new midas.stage.GoldenGateCompilerPhase)
+        new midas.stage.GoldenGateCompilerPhase,
+        midas.stage.phases.GenerateOutputFileManifest
+        )
       .map(DeletedWrapper(_))
 
 
diff --git a/sim/midas/src/main/scala/midas/stage/RuntimeConfigGenerationPhase.scala b/sim/midas/src/main/scala/midas/stage/RuntimeConfigGenerationPhase.scala
@@ -23,6 +23,7 @@ import logger._
 private [stage] case class RuntimeConfigurationFile(body: String) extends NoTargetAnnotation with GoldenGateFileEmission {
   override def suffix = None
   def getBytes = body.getBytes
+  def downstreamDependencies = Set(DownstreamFlows.RuntimeDeployment)
 }
 
 class RuntimeConfigGenerationPhase extends Phase {
diff --git a/sim/midas/src/main/scala/midas/stage/phases/GenerateOutputFileManifest.scala b/sim/midas/src/main/scala/midas/stage/phases/GenerateOutputFileManifest.scala
@@ -0,0 +1,53 @@
+// See LICENSE for license details.
+
+package midas.stage.phases
+
+import midas.stage.{GoldenGateFileEmission, GoldenGateOutputFileAnnotation, DownstreamFlows}
+import firrtl.{AnnotationSeq, EmittedVerilogCircuitAnnotation, EmittedVerilogModuleAnnotation}
+import firrtl.options.{TargetDirAnnotation, Phase, Dependency, CustomFileEmission}
+
+import org.json4s.native.Serialization
+
+import collection.immutable.ListMap
+
+/**
+  * Extracts a TargetDir-relative path to an EmittedFile.
+  */
+object TargetDirRelativePath {
+  def apply(emittedAnno: CustomFileEmission, annotations: AnnotationSeq): String = {
+    val targetDir  = annotations.collectFirst { case TargetDirAnnotation(name) => name }.get
+    emittedAnno.filename(annotations).getAbsolutePath.stripPrefix(s"${targetDir}/")
+  }
+}
+
+/** Emits an manifest file that sorts all known output files into groups based on 
+ * where those files will be used. See [[midas.stage.DownstreamFlows]].
+ */
+object GenerateOutputFileManifest extends Phase {
+
+  override val prerequisites = Seq(Dependency[midas.stage.GoldenGateCompilerPhase])
+
+  def transform(annotations: AnnotationSeq): AnnotationSeq = {
+    val ggOutputFiles   = annotations.collect { case anno: GoldenGateFileEmission => anno }
+
+    // Group GG annotations based on the flows in which they are used.
+    val unclassifiedFiles = "Unclassified" -> ggOutputFiles.filter(_.downstreamDependencies.isEmpty)
+    val classifiedFiles   = DownstreamFlows.allFlows.map { tpe =>
+      tpe.toString -> ggOutputFiles.filter(_.downstreamDependencies.contains(tpe)) 
+    }
+
+    val ggFileTuples = for ((flowType, fileAnnos) <- (classifiedFiles :+ unclassifiedFiles)) yield {
+      flowType -> fileAnnos.map { anno => TargetDirRelativePath(anno, annotations) }
+    }
+
+    // Capture all verilog files (GG does not control emission of these) and put them in a seperate field
+    val verilogCircuitPath  = annotations.collectFirst { case anno: EmittedVerilogCircuitAnnotation => TargetDirRelativePath(anno, annotations) }
+    val verilogModulesPath  = annotations.collect      { case anno: EmittedVerilogModuleAnnotation  => TargetDirRelativePath(anno, annotations) }
+    val emittedVerilogTuple = (DownstreamFlows.emittedVerilogKey -> (verilogCircuitPath ++: verilogModulesPath).toSeq )
+
+
+    implicit val formats = org.json4s.DefaultFormats
+    val body = Serialization.writePretty(ListMap((ggFileTuples :+ emittedVerilogTuple):_*))
+    GoldenGateOutputFileAnnotation(body, DownstreamFlows.fileManifestSuffix, Set()) +: annotations
+  }
+}
diff --git a/sim/midas/src/main/scala/midas/widgets/Widget.scala b/sim/midas/src/main/scala/midas/widgets/Widget.scala
@@ -14,6 +14,7 @@ import freechips.rocketchip.diplomacy._
 import freechips.rocketchip.util.ParameterizedBundle
 
 import scala.collection.mutable
+import midas.stage.DownstreamFlows
 
 // The AXI4-lite key for the simulation control bus
 case object CtrlNastiKey extends Field[NastiParameters]
@@ -303,6 +304,7 @@ trait HasWidgets {
     GoldenGateOutputFileAnnotation.annotateFromChisel(
       // Append an extra \n to prevent the file from being empty.
       body = widgets.map(_.defaultPlusArgs).flatten.mkString("\n") + "\n",
-      fileSuffix = ".runtime.conf"
+      fileSuffix = ".runtime.conf",
+      downstreamDependencies = Set(DownstreamFlows.RuntimeDeployment),
     )
 }
diff --git a/sim/midas/src/test/scala/midas/passes/AutoILATransformSpec.scala b/sim/midas/src/test/scala/midas/passes/AutoILATransformSpec.scala
@@ -13,8 +13,7 @@ import freechips.rocketchip.config.{Config, Parameters}
 import firrtl.annotations._
 import firrtl.transforms.BlackBoxInlineAnno
 import firrtl.testutils._
-import midas.stage.OutputBaseFilenameAnnotation
-import midas.stage.GoldenGateOutputFileAnnotation
+import midas.stage.{OutputBaseFilenameAnnotation, GoldenGateOutputFileAnnotation, DownstreamFlows}
 
 class BaseAutoILAConfig extends Config((site, here, up) => { case EnableAutoILA => true })
 
@@ -99,6 +98,7 @@ class AutoILATransformSpec extends MiddleTransformSpec with FirrtlRunners {
          |  CONFIG.ALL_PROBE_SAME_MU_CNT {2}] [get_ips ila_firesim]
          |""".stripMargin,
       ".ila_firesim.ipgen.tcl",
+      Set(DownstreamFlows.BitstreamCompile)
     )
 
     def bbAnno = BlackBoxInlineAnno(

Original file line number	Diff line number	Diff line change
`@@ -4,6 +4,7 @@ import chisel3._`
`4`	`4`	`import chisel3.util._`
`5`	`5`	`import freechips.rocketchip.amba.axi4._`
`6`	`6`	`import midas.stage.GoldenGateOutputFileAnnotation`
	`7`	`+import midas.stage.DownstreamFlows`
`7`	`8`
`8`	`9`	`/** An AXI4 bundle definition whose names should match the interfaces exposed on Xilinx IP blocks. aresetn and clock are`
`9`	`10`	`* omitted, and no user fields are provided.`
`@@ -209,5 +210,6 @@ class AXI4ClockConverter(`
`209`	`210`	`\| [get_ips ${desiredName}]`
`210`	`211`	`\|""".stripMargin,`
`211`	`212`	`s".${desiredName}.ipgen.tcl",`
	`213`	`+ Set(DownstreamFlows.BitstreamCompile)`
`212`	`214`	`)`
`213`	`215`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,7 @@`
`1`	`1`	`package midas.platform.xilinx`
`2`	`2`
`3`	`3`	`import chisel3._`
`4`		`-import midas.stage.GoldenGateOutputFileAnnotation`
	`4`	`+import midas.stage.{DownstreamFlows, GoldenGateOutputFileAnnotation}`
`5`	`5`
`6`	`6`	`class MMCM(inputFreqMHz: Double, outputFreqMHz: Double, override val desiredName: String) extends BlackBox {`
`7`	`7`	`val io = IO(new Bundle {`
`@@ -27,5 +27,6 @@ class MMCM(inputFreqMHz: Double, outputFreqMHz: Double, override val desiredName`
`27`	`27`	`\| [get_ips ${desiredName}]`
`28`	`28`	`\|""".stripMargin,`
`29`	`29`	`s".${desiredName}.ipgen.tcl",`
	`30`	`+ Set(DownstreamFlows.BitstreamCompile),`
`30`	`31`	`)`
`31`	`32`	`}`