From 742553f04df4822b1b733c2b0b7ddef571065ba2 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Fri, 13 Nov 2026 12:09:56 +0800 Subject: [PATCH] Upgrade the bootcamp to Chisel 7 Move the notebooks and the toolchain from Chisel 3.6 to Chisel 7.16 (Scala 2.13.18, CIRCT firtool 1.160, ChiselSim on Verilator 5, almond 0.14.5), the same stack as CoralNPU and A-Minimal-Chisel-Project. chiseltest is replaced by ChiselSim, so 2.6_chiseltest.ipynb becomes 2.6_chiselsim.ipynb. FixedPoint and dsptools no longer exist: the notebooks use SInt and small typeclasses written in place, and the rocket-chip AXI4 section is dropped. The Scala FIRRTL library is gone too, so module 4 teaches the CHIRRTL text and the CIRCT flow. runtest.py drives the Scala kernel with jupyter_client. The Dockerfile (Alpine 3.22, JDK 17, Verilator, preinstalled firtool), Binder, Install.md, README and the GitHub Actions workflow (ubuntu-24.04, newer action majors, dependabot, an offline smoke test) follow. --- .dockerignore | 3 + .github/dependabot.yml | 12 + .github/workflows/docker.yml | 38 +- .gitignore | 2 + 0_demo.ipynb | 9 +- 1_intro_to_scala.ipynb | 10 +- 2.1_first_module.ipynb | 24 +- 2.2_comb_logic.ipynb | 21 +- 2.3_control_flow.ipynb | 21 +- 2.4_sequential_logic.ipynb | 68 ++- 2.5_exercise.ipynb | 184 +----- 2.6_chiselsim.ipynb | 588 ++++++++++++++++++ 2.6_chiseltest.ipynb | 505 ---------------- 3.1_parameters.ipynb | 25 +- 3.2_collections.ipynb | 26 +- 3.2_interlude.ipynb | 31 +- 3.3_higher-order_functions.ipynb | 11 +- 3.4_functional_programming.ipynb | 107 ++-- 3.5_object_oriented_programming.ipynb | 18 +- 3.6_types.ipynb | 186 ++++-- 4.1_firrtl_ast.ipynb | 427 +++++++++----- 4.2_firrtl_ast_traversal.ipynb | 714 ++++++++++++++++++++-- 4.3_firrtl_common_idioms.ipynb | 817 ++++++++++++++++++++++---- 4.4_firrtl_add_ops_per_module.ipynb | 581 ++++++++++++++---- Dockerfile | 111 ++-- Install.md | 39 +- README.md | 11 +- binder/apt.txt | 4 +- binder/environment.yml | 6 + binder/postBuild | 22 +- runtest.py | 194 +++--- source/load-ivy.sc | 514 +++++++--------- tests/test_visualizer.py | 92 +++ 33 files changed, 3626 insertions(+), 1795 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 2.6_chiselsim.ipynb delete mode 100644 2.6_chiseltest.ipynb create mode 100644 binder/environment.yml create mode 100644 tests/test_visualizer.py diff --git a/.dockerignore b/.dockerignore index 0f53d94..73a8918 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,3 +13,6 @@ __pycache__ *.swp *.swo *~ +build/ +target/ +out/ diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..54d141e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +version: 2 +updates: + # Keep the GitHub Actions used by the workflows current + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + # Keep the base image of the Dockerfile current + - package-ecosystem: docker + directory: / + schedule: + interval: weekly diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index fead2c6..5b7fd55 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -14,34 +14,52 @@ concurrency: jobs: docker: name: Build Docker Image and Publish (only on push) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: contents: read actions: write steps: - name: Check out the repo - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to Docker Hub if: github.event_name == 'push' - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} + - name: Build image for the smoke test (amd64) + uses: docker/build-push-action@v7 + with: + context: . + platforms: linux/amd64 + load: true + tags: chisel-bootcamp:test + cache-from: type=gha,scope=build + cache-to: type=gha,mode=max,scope=build + + - name: Run notebooks (smoke test, offline) + # The image bundles every dependency, so no network is needed. + run: | + docker run --rm --network none chisel-bootcamp:test \ + python3 runtest.py 0_demo.ipynb 2.1_first_module.ipynb 2.6_chiselsim.ipynb 4.1_firrtl_ast.ipynb + docker run --rm --network none chisel-bootcamp:test python3 tests/test_visualizer.py + - name: Build and push - uses: docker/build-push-action@v6 + if: github.event_name == 'push' + uses: docker/build-push-action@v7 with: context: . - platforms: ${{ github.event_name == 'push' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} - push: ${{ github.event_name == 'push' }} + platforms: linux/amd64,linux/arm64 + push: true tags: sysprog21/chisel-bootcamp:latest - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=build + cache-to: type=gha,mode=max,scope=build diff --git a/.gitignore b/.gitignore index ffecbf5..b37318c 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ # Jupyter-scala scripts coursier almond +/build +*.sv diff --git a/0_demo.ipynb b/0_demo.ipynb index c5ab59c..e961dd6 100644 --- a/0_demo.ipynb +++ b/0_demo.ipynb @@ -36,8 +36,7 @@ }, "outputs": [], "source": [ - "val path = System.getProperty(\"user.dir\") + \"/source/load-ivy.sc\"\n", - "interp.load.module(ammonite.ops.Path(java.nio.file.FileSystems.getDefault().getPath(path)))" + "interp.load.module(os.pwd / \"source\" / \"load-ivy.sc\")" ] }, { @@ -47,8 +46,8 @@ "outputs": [], "source": [ "import chisel3._\n", - "import chisel3.util._\n", - "import chisel3.iotesters.{ChiselFlatSpec, Driver, PeekPokeTester}" + "import chisel3.Input // avoid a name clash with almond.input.Input\n", + "import chisel3.util._\n" ] }, { @@ -222,7 +221,7 @@ "mimetype": "text/x-scala", "name": "scala", "nbconvert_exporter": "script", - "version": "2.12.8" + "version": "2.13.18" } }, "nbformat": 4, diff --git a/1_intro_to_scala.ipynb b/1_intro_to_scala.ipynb index 37f86b2..b8cde9a 100644 --- a/1_intro_to_scala.ipynb +++ b/1_intro_to_scala.ipynb @@ -401,9 +401,9 @@ "\n", "```scala\n", "import chisel3._\n", - "import chisel3.iotesters.{ChiselFlatSpec, Driver, PeekPokeTester}\n", + "import chisel3.util.{Counter, Queue}\n", "```\n", - "The first imports all the classes and methods in the chisel3 package; the underscore here works as a wildcard. The second imports specific classes from the chisel3.iotesters package." + "The first imports all the classes and methods in the chisel3 package; the underscore here works as a wildcard. The second imports specific classes (here `Counter` and `Queue`) from the chisel3.util package." ] }, { @@ -538,7 +538,9 @@ " println(\"counter is about to wrap\")\n", "}\n", "\n", - "x inc() // Scala allows the dots to be omitted; this can be useful for making embedded DSL's look more natural" + "// Scala allows the dot and parentheses to be omitted when a method takes exactly one argument;\n", + "// this can be useful for making embedded DSL's look more natural\n", + "println(x.counter max 3) // same as x.counter.max(3); a method without arguments, like inc(), still needs `x.inc()`" ] }, { @@ -651,7 +653,7 @@ "mimetype": "text/x-scala", "name": "scala", "nbconvert_exporter": "script", - "version": "2.12.8" + "version": "2.13.18" } }, "nbformat": 4, diff --git a/2.1_first_module.ipynb b/2.1_first_module.ipynb index ba37e6b..299ee81 100644 --- a/2.1_first_module.ipynb +++ b/2.1_first_module.ipynb @@ -35,8 +35,7 @@ "metadata": {}, "outputs": [], "source": [ - "val path = System.getProperty(\"user.dir\") + \"/source/load-ivy.sc\"\n", - "interp.load.module(ammonite.ops.Path(java.nio.file.FileSystems.getDefault().getPath(path)))" + "interp.load.module(os.pwd / \"source\" / \"load-ivy.sc\")" ] }, { @@ -51,7 +50,12 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": "import chisel3._\nimport chisel3.util._\nimport chiseltest._\nimport chiseltest.RawTester.test" + "source": [ + "import chisel3._\n", + "import chisel3.Input // avoid a name clash with almond.input.Input\n", + "import chisel3.util._\n", + "import chisel3.simulator.PeekPokeAPI._" + ] }, { "cell_type": "markdown", @@ -190,7 +194,7 @@ "// Scala Code: `test` runs the unit test. \n", "// test takes a user Module and has a code block that applies pokes and expects to the \n", "// circuit under test (c)\n", - "test(new Passthrough()) { c =>\n", + "simulate(new Passthrough()) { c =>\n", " c.io.in.poke(0.U) // Set our input to value 0\n", " c.io.out.expect(0.U) // Assert that the output correctly has 0\n", " c.io.in.poke(1.U) // Set our input to value 1\n", @@ -230,13 +234,13 @@ "source": [ "// Test with width 10\n", "\n", - "test(???) { c =>\n", + "simulate(???) { c =>\n", " ???\n", "}\n", "\n", "// Test with width 20\n", "\n", - "test(???) { c =>\n", + "simulate(???) { c =>\n", " ???\n", "}\n", "\n", @@ -252,14 +256,14 @@ "\n", "
\n", "
\n",
-    "test(new PassthroughGenerator(10)) { c =>\n",
+    "simulate(new PassthroughGenerator(10)) { c =>\n",
     "    c.io.in.poke(0.U)\n",
     "    c.io.out.expect(0.U)\n",
     "    c.io.in.poke(1023.U)\n",
     "    c.io.out.expect(1023.U)\n",
     "}\n",
     "\n",
-    "test(new PassthroughGenerator(20)) { c =>\n",
+    "simulate(new PassthroughGenerator(20)) { c =>\n",
     "    c.io.in.poke(0.U)\n",
     "    c.io.out.expect(0.U)\n",
     "    c.io.in.poke(1048575.U)\n",
@@ -349,7 +353,7 @@
     "    println(s\"Print during generation: Input is ${io.in}\")\n",
     "}\n",
     "\n",
-    "test(new PrintingModule ) { c =>\n",
+    "simulate(new PrintingModule ) { c =>\n",
     "    c.io.in.poke(3.U)\n",
     "    c.clock.step(5) // circuit will print\n",
     "    \n",
@@ -371,7 +375,7 @@
    "mimetype": "text/x-scala",
    "name": "scala",
    "nbconvert_exporter": "script",
-   "version": "2.12.10"
+   "version": "2.13.18"
   }
  },
  "nbformat": 4,
diff --git a/2.2_comb_logic.ipynb b/2.2_comb_logic.ipynb
index 72fe59f..fa73a1c 100644
--- a/2.2_comb_logic.ipynb
+++ b/2.2_comb_logic.ipynb
@@ -33,8 +33,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "val path = System.getProperty(\"user.dir\") + \"/source/load-ivy.sc\"\n",
-    "interp.load.module(ammonite.ops.Path(java.nio.file.FileSystems.getDefault().getPath(path)))"
+    "interp.load.module(os.pwd / \"source\" / \"load-ivy.sc\")"
    ]
   },
   {
@@ -44,9 +43,9 @@
    "outputs": [],
    "source": [
     "import chisel3._\n",
+    "import chisel3.Input // avoid a name clash with almond.input.Input\n",
     "import chisel3.util._\n",
-    "import chiseltest._\n",
-    "import chiseltest.RawTester.test"
+    "import chisel3.simulator.PeekPokeAPI._"
    ]
   },
   {
@@ -108,7 +107,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "We create two `val`s. The first adds two Scala `Int`s, so `println` prints out the integer 2. The second `val` adds two *Chisel* `UInt`s together, so `println` sees this as a hardware node and prints out the type name and pointer (`chisel3.core.UInt@d`). Note that `1.U` is a type cast from a Scala `Int` (1) to a Chisel `UInt` literal.\n",
+    "We create two `val`s. The first adds two Scala `Int`s, so `println` prints out the integer 2. The second `val` adds two *Chisel* `UInt`s together, so `println` sees this as a hardware node and prints out a description of the node (its type and an internal name, not a number). Note that `1.U` is a type cast from a Scala `Int` (1) to a Chisel `UInt` literal.\n",
     "\n",
     "We need to drive the output to something, so we just connect it to the input for now, as with the passthrough module in the previous tutorial.\n",
     "\n",
@@ -180,7 +179,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "test(new MyOperators) {c =>\n",
+    "simulate(new MyOperators) {c =>\n",
     "  c.io.out_add.expect(5.U)\n",
     "  c.io.out_sub.expect(1.U)\n",
     "  c.io.out_mul.expect(8.U)\n",
@@ -216,7 +215,7 @@
     "\n",
     "println(getVerilog(new MyOperatorsTwo))\n",
     "\n",
-    "test(new MyOperatorsTwo) { c =>\n",
+    "simulate(new MyOperatorsTwo) { c =>\n",
     "  c.io.out_mux.expect(3.U)\n",
     "  c.io.out_cat.expect(5.U)\n",
     "}\n",
@@ -268,7 +267,7 @@
     "  ???\n",
     "}\n",
     "\n",
-    "test(new MAC) { c =>\n",
+    "simulate(new MAC) { c =>\n",
     "  val cycles = 100\n",
     "  import scala.util.Random\n",
     "  for (i <- 0 until cycles) {\n",
@@ -345,7 +344,7 @@
     "  ???\n",
     "}\n",
     "\n",
-    "test(new Arbiter) { c =>\n",
+    "simulate(new Arbiter) { c =>\n",
     "  import scala.util.Random\n",
     "  val data = Random.nextInt(65536)\n",
     "  c.io.fifo_data.poke(data.U)\n",
@@ -427,7 +426,7 @@
     "}\n",
     "\n",
     "for (saturate <- Seq(true, false)) {\n",
-    "  test(new ParameterizedAdder(saturate)) { c =>\n",
+    "  simulate(new ParameterizedAdder(saturate)) { c =>\n",
     "    // 100 random tests\n",
     "    val cycles = 100\n",
     "    import scala.util.Random\n",
@@ -498,7 +497,7 @@
    "mimetype": "text/x-scala",
    "name": "scala",
    "nbconvert_exporter": "script",
-   "version": "2.12.10"
+   "version": "2.13.18"
   }
  },
  "nbformat": 4,
diff --git a/2.3_control_flow.ipynb b/2.3_control_flow.ipynb
index 57bee37..1fdfc00 100644
--- a/2.3_control_flow.ipynb
+++ b/2.3_control_flow.ipynb
@@ -34,8 +34,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "val path = System.getProperty(\"user.dir\") + \"/source/load-ivy.sc\"\n",
-    "interp.load.module(ammonite.ops.Path(java.nio.file.FileSystems.getDefault().getPath(path)))"
+    "interp.load.module(os.pwd / \"source\" / \"load-ivy.sc\")"
    ]
   },
   {
@@ -45,9 +44,9 @@
    "outputs": [],
    "source": [
     "import chisel3._\n",
+    "import chisel3.Input // avoid a name clash with almond.input.Input\n",
     "import chisel3.util._\n",
-    "import chiseltest._\n",
-    "import chiseltest.RawTester.test"
+    "import chisel3.simulator.PeekPokeAPI._"
    ]
   },
   {
@@ -81,7 +80,7 @@
     "}\n",
     "\n",
     "//  Test LastConnect\n",
-    "test(new LastConnect) { c => c.io.out.expect(4.U) } // Assert that the output correctly has 4\n",
+    "simulate(new LastConnect) { c => c.io.out.expect(4.U) } // Assert that the output correctly has 4\n",
     "println(\"SUCCESS!!\") // Scala Code: if we get here, our tests passed!"
    ]
   },
@@ -143,7 +142,7 @@
     "}\n",
     "\n",
     "// Test Max3\n",
-    "test(new Max3) { c =>\n",
+    "simulate(new Max3) { c =>\n",
     "  // verify that the max of the three inputs is correct\n",
     "  c.io.in1.poke(6.U)\n",
     "  c.io.in2.poke(4.U)\n",
@@ -265,7 +264,7 @@
     "\n",
     "\n",
     "// Here's the tester\n",
-    "test(new Sort4) { c =>\n",
+    "simulate(new Sort4) { c =>\n",
     "  // verify the inputs are sorted\n",
     "  c.io.in0.poke(3.U)\n",
     "  c.io.in1.poke(6.U)\n",
@@ -311,7 +310,7 @@
    "outputs": [],
    "source": [
     "// Here's the tester\n",
-    "test(new Sort4) { c =>\n",
+    "simulate(new Sort4) { c =>\n",
     "  // verify the all possible ordering of 4 numbers are sorted\n",
     "  List(1, 2, 3, 4).permutations.foreach { case i0 :: i1 :: i2 :: i3 :: Nil =>\n",
     "    println(s\"Sorting $i0 $i1 $i2 $i3\")\n",
@@ -457,7 +456,7 @@
     "}\n",
     "\n",
     "// Test Polynomial\n",
-    "test(new Polynomial) { c =>\n",
+    "simulate(new Polynomial) { c =>\n",
     "  for(x <- 0 to 20) {\n",
     "    for(select <- 0 to 2) {\n",
     "      c.io.select.poke(select.U)\n",
@@ -580,7 +579,7 @@
     "\n",
     "\n",
     "// Test\n",
-    "test(new GradLife) { c =>\n",
+    "simulate(new GradLife) { c =>\n",
     "  // verify that the hardware matches the golden model\n",
     "  for (state <- 0 to 3) {\n",
     "    for (coffee <- List(true, false)) {\n",
@@ -648,7 +647,7 @@
    "mimetype": "text/x-scala",
    "name": "scala",
    "nbconvert_exporter": "script",
-   "version": "2.12.10"
+   "version": "2.13.18"
   }
  },
  "nbformat": 4,
diff --git a/2.4_sequential_logic.ipynb b/2.4_sequential_logic.ipynb
index d9c11c5..47b2f0e 100644
--- a/2.4_sequential_logic.ipynb
+++ b/2.4_sequential_logic.ipynb
@@ -33,8 +33,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "val path = System.getProperty(\"user.dir\") + \"/source/load-ivy.sc\"\n",
-    "interp.load.module(ammonite.ops.Path(java.nio.file.FileSystems.getDefault().getPath(path)))"
+    "interp.load.module(os.pwd / \"source\" / \"load-ivy.sc\")"
    ]
   },
   {
@@ -44,9 +43,9 @@
    "outputs": [],
    "source": [
     "import chisel3._\n",
+    "import chisel3.Input // avoid a name clash with almond.input.Input\n",
     "import chisel3.util._\n",
-    "import chiseltest._\n",
-    "import chiseltest.RawTester.test"
+    "import chisel3.simulator.PeekPokeAPI._"
    ]
   },
   {
@@ -82,7 +81,7 @@
     "  io.out := register\n",
     "}\n",
     "\n",
-    "test(new RegisterModule) { c =>\n",
+    "simulate(new RegisterModule) { c =>\n",
     "  for (i <- 0 until 100) {\n",
     "    c.io.in.poke(i.U)\n",
     "    c.clock.step(1)\n",
@@ -107,12 +106,12 @@
     "\n",
     "The astute observer will notice that previous testers testing combinational logic did not call `step()`. This is because calling `poke()` on an input immediately propagates the updated values through combinational logic. Calling `step()` is only needed to update state elements in sequential logic. \n",
     "\n",
-    "The code block below will show the verilog generated by `RegisterModule`.\n",
+    "The code block below will show the SystemVerilog generated by `RegisterModule`.\n",
     "\n",
     "Note:\n",
     "* The module has an input for clock (and reset) that you didn't add- this is the implicit clock\n",
     "* The variable `register` shows up as `reg [11:0]`, as expected\n",
-    "* There is a block sectioned off by `ifdef Randomize` that initialized the register to some random variable before simulation starts\n",
+    "* There is a block sectioned off by `ifdef ENABLE_INITIAL_REG_` / `ifdef RANDOMIZE_REG_INIT` that initializes the register to some random value before simulation starts (this is why a plain `Reg` has no predictable value until you write to it)\n",
     "* `register` is updated on `posedge clock`"
    ]
   },
@@ -164,7 +163,7 @@
     "  io.out := RegNext(io.in + 1.U)\n",
     "}\n",
     "\n",
-    "test(new RegNextModule) { c =>\n",
+    "simulate(new RegNextModule) { c =>\n",
     "  for (i <- 0 until 100) {\n",
     "    c.io.in.poke(i.U)\n",
     "    c.clock.step(1)\n",
@@ -178,7 +177,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "The Verilog looks almost the same as before, though the register name is generated instead of explicity defined."
+    "The SystemVerilog looks almost the same as before, though the register name is generated instead of explicity defined."
    ]
   },
   {
@@ -236,18 +235,32 @@
     "  io.out := register\n",
     "}\n",
     "\n",
-    "println(getVerilog(new RegInitModule))"
+    "println(getVerilog(new RegInitModule))\n",
+    "\n",
+    "simulate(new RegInitModule) { c =>\n",
+    "  c.io.out.expect(0.U)   // right after the reset: the register holds its reset value, not random junk\n",
+    "  c.io.in.poke(41.U)\n",
+    "  c.clock.step(1)\n",
+    "  c.io.out.expect(42.U)\n",
+    "}\n",
+    "println(\"SUCCESS!!\")"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Note that the generated verilog now has a block that checks `if (reset)` to reset the register to 0.\n",
+    "Note that the generated SystemVerilog now has a block that checks `if (reset)` to reset the register to 0.\n",
     "Also note that this is inside the `always @(posedge clock)` block.\n",
     "Chisel's implicit reset is active high and synchronous.\n",
-    "The register is still initialized to random junk before reset is called.\n",
-    "The `PeekPokeTesters` always call reset before running your test, but you can manually call reset as well using the `reset(n)` function, where reset is high for `n` cycles."
+    "The register is still initialized to random junk until reset is applied.\n",
+    "\n",
+    "**Important:** `simulate(...)` applies a reset to the module (one clock cycle with `reset` high) before running your test body, like the testers of earlier Chisel versions did.\n",
+    "That is what gives a `RegInit` register its value: after the reset it holds `0` in the test above.\n",
+    "A plain `Reg`, on the other hand, has no reset value, so it holds a *random* value at power-up (ChiselSim randomizes uninitialized registers on purpose, to catch tests that accidentally depend on them) and keeps it until the circuit writes to it.\n",
+    "So if your test needs a register to start at a known value, use `RegInit`.\n",
+    "\n",
+    "If you ever need to reset again in the middle of a test, drive the `reset` input yourself like any other input: `c.reset.poke(true.B); c.clock.step(1); c.reset.poke(false.B)`. The reset is synchronous, so it only takes effect on a clock edge."
    ]
   },
   {
@@ -260,7 +273,8 @@
     "They have last connect semantics and can be assigned to conditionally with `when`, `elsewhen`, and `otherwise`.\n",
     "\n",
     "**Example: Register Control Flow**
\n", - "The following example finds the maximum value in a sequence of inputs using conditional register assignments." + "The following example finds the maximum value in a sequence of inputs using conditional register assignments.\n", + "Thanks to the reset applied by `simulate`, `max` starts at 0 in the test below. Had we declared it with a plain `Reg(UInt(10.W))` instead of `RegInit`, the first `expect` would fail, because the register would start out with a random value." ] }, { @@ -282,7 +296,7 @@ " io.max := max\n", "}\n", "\n", - "test(new FindMax) { c =>\n", + "simulate(new FindMax) { c =>\n", " c.io.max.expect(0.U)\n", " c.io.in.poke(1.U)\n", " c.clock.step(1)\n", @@ -358,7 +372,7 @@ "\n", "\"shift\n", "\n", - "A basic Module skeleton, testvector, and Driver invocation is provided below. The first register has been provided for you." + "A basic Module skeleton and a test are provided below. The first register has been provided for you." ] }, { @@ -378,7 +392,7 @@ " ???\n", "}\n", "\n", - "test(new MyShiftRegister()) { c =>\n", + "simulate(new MyShiftRegister()) { c =>\n", " var state = c.init\n", " for (i <- 0 until 10) {\n", " // poke in LSB of i (i % 2)\n", @@ -438,7 +452,7 @@ "// test different depths\n", "for (i <- Seq(3, 4, 8, 24, 65)) {\n", " println(s\"Testing n=$i\")\n", - " test(new MyOptionalShiftRegister(n = i)) { c =>\n", + " simulate(new MyOptionalShiftRegister(n = i)) { c =>\n", " val inSeq = Seq(0, 1, 1, 1, 0, 1, 1, 0, 0, 1)\n", " var state = c.init\n", " var i = 0\n", @@ -489,11 +503,11 @@ "Clocks and resets can be overridden separately or together with `withClock() {}`, `withReset() {}`, and `withClockAndReset() {}`.\n", "The following code blocks will give examples of using these functions.\n", "\n", - "One thing to be aware of is that `reset` (as of this tutorial's writing) is always synchronous and of type `Bool`.\n", + "One thing to be aware of is that the default `reset` is synchronous and of type `Bool` (Chisel also supports asynchronous resets, see `Reset`, `AsyncReset` and `RequireAsyncReset`).\n", "Clocks have their own type in Chisel (`Clock`) and should be declared as such.\n", "*`Bool`s can be converted to `Clock`s by calling `asClock()` on them, but you should be careful that you aren't doing something silly.*\n", "\n", - "Also note that `chisel-testers` do not currently have complete support for multi-clock designs.\n", + "Also note that driving several clocks from a test is more involved than the single-clock `c.clock.step()` used in this module; the examples below therefore only elaborate the hardware.\n", "\n", "**Example: Multi-Clock Module**
\n", "A module with multiple clocks and reset signals." @@ -507,18 +521,16 @@ }, "outputs": [], "source": [ - "// we need to import multi-clock features\n", - "import chisel3.experimental.{withClock, withReset, withClockAndReset}\n", - "\n", + "// withClock, withReset and withClockAndReset are part of chisel3._, which we already imported\n", "class ClockExamples extends Module {\n", " val io = IO(new Bundle {\n", " val in = Input(UInt(10.W))\n", " val alternateReset = Input(Bool())\n", " val alternateClock = Input(Clock())\n", - " val outImplicit = Output(UInt())\n", - " val outAlternateReset = Output(UInt())\n", - " val outAlternateClock = Output(UInt())\n", - " val outAlternateBoth = Output(UInt())\n", + " val outImplicit = Output(UInt(10.W))\n", + " val outAlternateReset = Output(UInt(10.W))\n", + " val outAlternateClock = Output(UInt(10.W))\n", + " val outAlternateBoth = Output(UInt(10.W))\n", " })\n", "\n", " val imp = RegInit(0.U(10.W))\n", @@ -584,7 +596,7 @@ "mimetype": "text/x-scala", "name": "scala", "nbconvert_exporter": "script", - "version": "2.12.10" + "version": "2.13.18" } }, "nbformat": 4, diff --git a/2.5_exercise.ipynb b/2.5_exercise.ipynb index ecc664e..c8b40e4 100644 --- a/2.5_exercise.ipynb +++ b/2.5_exercise.ipynb @@ -13,7 +13,7 @@ "source": [ "# Module 2.5: Putting it all Together: An FIR Filter\n", "**Prev: [Sequential Logic](2.4_sequential_logic.ipynb)**
\n", - "**Next: [ChiselTest (was chisel-testers2)](2.6_chiseltest.ipynb)**\n", + "**Next: [Testing with ChiselSim](2.6_chiselsim.ipynb)**\n", "\n", "## Motivation\n", "Now that you've learned the basics of Chisel, let's use that knowledge to build a FIR (finite impulse response) filter module! FIR filters are very common in digital signal processing applications. Also, the FIR filter will reappear frequently in module 3, so it's important that you don't filter out this module by skipping ahead! If you are unfamiliar with FIR filters, head over to the article on [trusty Wikipedia](https://en.wikipedia.org/wiki/Finite_impulse_response) to learn more.\n", @@ -27,8 +27,7 @@ "metadata": {}, "outputs": [], "source": [ - "val path = System.getProperty(\"user.dir\") + \"/source/load-ivy.sc\"\n", - "interp.load.module(ammonite.ops.Path(java.nio.file.FileSystems.getDefault().getPath(path)))" + "interp.load.module(os.pwd / \"source\" / \"load-ivy.sc\")" ] }, { @@ -38,9 +37,9 @@ "outputs": [], "source": [ "import chisel3._\n", + "import chisel3.Input // avoid a name clash with almond.input.Input\n", "import chisel3.util._\n", - "import chiseltest._\n", - "import chiseltest.RawTester.test" + "import chisel3.simulator.PeekPokeAPI._" ] }, { @@ -74,7 +73,7 @@ "Note that both the input and output are 8-bit unsigned integers. You will need to save necessary state (like delayed signal values) using constructs like shift registers. Use the provided testers to check your implementation.\n", "Registers with constant inputs can be created using a `ShiftRegister` of shift value 1, or by using the `RegNext` construct.\n", "\n", - "Note: for the tests to pass, your registers must be initialized to `0.U`." + "Note: for the tests to pass, your registers must be initialized to `0.U` (e.g. with `RegInit` or `RegNext(x, 0.U)`). `simulate` resets the module before the test body runs, which gives these registers their value (see [Module 2.4](2.4_sequential_logic.ipynb))." ] }, { @@ -108,7 +107,7 @@ "outputs": [], "source": [ "// Simple sanity check: a element with all zero coefficients should always produce zero\n", - "test(new My4ElementFir(0, 0, 0, 0)) { c =>\n", + "simulate(new My4ElementFir(0, 0, 0, 0)) { c =>\n", " c.io.in.poke(0.U)\n", " c.io.out.expect(0.U)\n", " c.clock.step(1)\n", @@ -130,7 +129,7 @@ "outputs": [], "source": [ "// Simple 4-point moving average\n", - "test(new My4ElementFir(1, 1, 1, 1)) { c =>\n", + "simulate(new My4ElementFir(1, 1, 1, 1)) { c =>\n", " c.io.in.poke(1.U)\n", " c.io.out.expect(1.U) // 1, 0, 0, 0\n", " c.clock.step(1)\n", @@ -158,7 +157,7 @@ "outputs": [], "source": [ "// Nonsymmetric filter\n", - "test(new My4ElementFir(1, 2, 3, 4)) { c =>\n", + "simulate(new My4ElementFir(1, 2, 3, 4)) { c =>\n", " c.io.in.poke(1.U)\n", " c.io.out.expect(1.U) // 1*1, 0*2, 0*3, 0*4\n", " c.clock.step(1)\n", @@ -247,171 +246,6 @@ "visualize(() => new MyManyDynamicElementVecFir(4))" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "# DspBlock\n", - "\n", - "Integrating DSP components into a larger system can be challenging and error prone.\n", - "The [rocket section of the dsptools repository](https://github.com/ucb-bar/dsptools/tree/master/rocket) consists of useful generators that should help with such tasks.\n", - "\n", - "One of the core abstractions is the notion of a `DspBlock`.\n", - "A `DspBlock` has:\n", - "* AXI-4 Stream input and output\n", - "* Memory-mapped status and control (in this example, AXI4)\n", - "\n", - "\n", - "\n", - "`DspBlock`s use diplomatic interfaces from rocket.\n", - "[This site](https://www.lowrisc.org/docs/diplomacy/) has a good overview of the basic of diplomacy, but don't worry too much about how it's working for this example.\n", - "Diplomacy really shines when you're connecting a lot of different blocks together to form a complex SoC.\n", - "In this example, we're just making a single peripheral.\n", - "The `StandaloneBlock` traits are mixed in to make diplomatic interfaces work as top-level IOs.\n", - "You only need them when the `DspBlock` is being used as a top level interface without any diplomatic connections.\n", - "\n", - "The following code wraps the FIR filter in AXI4 interfaces.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import dspblocks._\n", - "import freechips.rocketchip.amba.axi4._\n", - "import freechips.rocketchip.amba.axi4stream._\n", - "import freechips.rocketchip.config._\n", - "import freechips.rocketchip.diplomacy._\n", - "import freechips.rocketchip.regmapper._\n", - "\n", - "//\n", - "// Base class for all FIRBlocks.\n", - "// This can be extended to make TileLink, AXI4, APB, AHB, etc. flavors of the FIR filter\n", - "//\n", - "abstract class FIRBlock[D, U, EO, EI, B <: Data](val nFilters: Int, val nTaps: Int)(implicit p: Parameters)\n", - "// HasCSR means that the memory interface will be using the RegMapper API to define status and control registers\n", - "extends DspBlock[D, U, EO, EI, B] with HasCSR {\n", - " // diplomatic node for the streaming interface\n", - " // identity node means the output and input are parameterized to be the same\n", - " val streamNode = AXI4StreamIdentityNode()\n", - " \n", - " // define the what hardware will be elaborated\n", - " lazy val module = new LazyModuleImp(this) {\n", - " // get streaming input and output wires from diplomatic node\n", - " val (in, _) = streamNode.in(0)\n", - " val (out, _) = streamNode.out(0)\n", - "\n", - " require(in.params.n >= nFilters,\n", - " s\"\"\"AXI-4 Stream port must be big enough for all \n", - " |the filters (need $nFilters,, only have ${in.params.n})\"\"\".stripMargin)\n", - "\n", - " // make registers to store taps\n", - " val taps = Reg(Vec(nFilters, Vec(nTaps, UInt(8.W))))\n", - "\n", - " // memory map the taps, plus the first address is a read-only field that says how many filter lanes there are\n", - " val mmap = Seq(\n", - " RegField.r(64, nFilters.U, RegFieldDesc(\"nFilters\", \"Number of filter lanes\"))\n", - " ) ++ taps.flatMap(_.map(t => RegField(8, t, RegFieldDesc(\"tap\", \"Tap\"))))\n", - "\n", - " // generate the hardware for the memory interface\n", - " // in this class, regmap is abstract (unimplemented). mixing in something like AXI4HasCSR or TLHasCSR\n", - " // will define regmap for the particular memory interface\n", - " regmap(mmap.zipWithIndex.map({case (r, i) => i * 8 -> Seq(r)}): _*)\n", - "\n", - " // make the FIR lanes and connect inputs and taps\n", - " val outs = for (i <- 0 until nFilters) yield {\n", - " val fir = Module(new MyManyDynamicElementVecFir(nTaps))\n", - " \n", - " fir.io.in := in.bits.data((i+1)*8, i*8)\n", - " fir.io.valid := in.valid && out.ready\n", - " fir.io.consts := taps(i) \n", - " fir.io.out\n", - " }\n", - "\n", - " val output = if (outs.length == 1) {\n", - " outs.head\n", - " } else {\n", - " outs.reduce((x: UInt, y: UInt) => Cat(y, x))\n", - " }\n", - "\n", - " out.bits.data := output\n", - " in.ready := out.ready\n", - " out.valid := in.valid\n", - " }\n", - "}\n", - "\n", - "// make AXI4 flavor of FIRBlock\n", - "abstract class AXI4FIRBlock(nFilters: Int, nTaps: Int)(implicit p: Parameters) extends FIRBlock[AXI4MasterPortParameters, AXI4SlavePortParameters, AXI4EdgeParameters, AXI4EdgeParameters, AXI4Bundle](nFilters, nTaps) with AXI4DspBlock with AXI4HasCSR {\n", - " override val mem = Some(AXI4RegisterNode(\n", - " AddressSet(0x0, 0xffffL), beatBytes = 8\n", - " ))\n", - "}\n", - "\n", - "// running the code below will show what firrtl is generated\n", - "// note that LazyModules aren't really chisel modules- you need to call \".module\" on them when invoking the chisel driver\n", - "// also note that AXI4StandaloneBlock is mixed in- if you forget it, you will get weird diplomacy errors because the memory\n", - "// interface expects a master and the streaming interface expects to be connected. AXI4StandaloneBlock will add top level IOs\n", - "// println(chisel3.Driver.emit(() => LazyModule(new AXI4FIRBlock(1, 8)(Parameters.empty) with AXI4StandaloneBlock).module))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Testing\n", - "\n", - "Testing `DspBlock`s is a little different.\n", - "Now we're dealing with memory interfaces and `LazyModule`s.\n", - "dsptools has some features that help test `DspBlock`s.\n", - "\n", - "One important feature is `MemMasterModel`.\n", - "The trait defines functions like `memReadWord` and `memWriteWord`- generic functions for generating memory traffic.\n", - "This allows you to write one generic test that can be specialized to the memory interface you are using- for example, you write one test and then specialize it for the TileLink and AXI4 interfaces.\n", - "\n", - "The code below tests the `FIRBlock` this way." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import dsptools.tester.MemMasterModel\n", - "import freechips.rocketchip.amba.axi4\n", - "\n", - "abstract class FIRBlockTester[D, U, EO, EI, B <: Data](c: FIRBlock[D, U, EO, EI, B]) extends PeekPokeTester(c.module) with MemMasterModel {\n", - " // check that address 0 is the number of filters\n", - " require(memReadWord(0) == c.nFilters)\n", - " // write 1 to all the taps\n", - " for (i <- 0 until c.nFilters * c.nTaps) {\n", - " memWriteWord(8 + i * 8, 1)\n", - " }\n", - "}\n", - "\n", - "// specialize the generic tester for axi4\n", - "class AXI4FIRBlockTester(c: AXI4FIRBlock with AXI4StandaloneBlock) extends FIRBlockTester(c) with AXI4MasterModel {\n", - " def memAXI = c.ioMem.get\n", - "}\n", - "\n", - "// invoking testers on lazymodules is a little strange.\n", - "// note that the firblocktester takes a lazymodule, not a module (it calls .module in \"extends PeekPokeTester()\").\n", - "val lm = LazyModule(new AXI4FIRBlock(1, 8)(Parameters.empty) with AXI4StandaloneBlock)\n", - "chisel3.iotesters.Driver(() => lm.module) { _ => new AXI4FIRBlockTester(lm) }" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "**Exercise: TileLink**
\n", - "\n", - "Add a version of `FIRBlock` that uses TileLink for its memory interconnect, and extend the `FIRBlockTester` to use TileLink." - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -435,7 +269,7 @@ "mimetype": "text/x-scala", "name": "scala", "nbconvert_exporter": "script", - "version": "2.12.10" + "version": "2.13.18" } }, "nbformat": 4, diff --git a/2.6_chiselsim.ipynb b/2.6_chiselsim.ipynb new file mode 100644 index 0000000..dd6519d --- /dev/null +++ b/2.6_chiselsim.ipynb @@ -0,0 +1,588 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Chisel" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Module 2.6: Testing with ChiselSim\n", + "**Prev: [Putting it all Together: An FIR Filter](2.5_exercise.ipynb)**
\n", + "**Next: [Generators: Parameters](3.1_parameters.ipynb)**\n", + "\n", + "## Motivation\n", + "So far we have used `simulate(...)` to test our circuits without saying much about how it works.\n", + "`simulate` is Chisel's built-in simulation and testing library, **ChiselSim** (package `chisel3.simulator`).\n", + "It replaces the older `chisel-testers`/`iotesters` and the separate `chiseltest` library, which were used with earlier versions of Chisel.\n", + "ChiselSim provides the following benefits:\n", + "\n", + "- it is part of Chisel itself, so there is no extra dependency to manage\n", + "- it is suitable for both unit tests and system integration tests\n", + "- it is designed for composable abstractions and layering: a test is just ordinary Scala code, so you can build your own helper functions out of the primitives\n", + "- it compiles your design to SystemVerilog and runs it on a real simulator (Verilator in this bootcamp), so what you simulate is what you would synthesize\n", + "- it works well with unit-test frameworks like ScalaTest (see `chisel3.simulator.scalatest.ChiselSim`) when you move from a notebook to a real project\n", + "\n", + "In this module we review the basic operations (`poke`, `peek`, `expect` and `step`), and then learn how to test modules with `Decoupled` (ready/valid) interfaces by writing small helper functions ourselves.\n", + "\n", + "## Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "interp.load.module(os.pwd / \"source\" / \"load-ivy.sc\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import chisel3._\n", + "import chisel3.Input // avoid a name clash with almond.input.Input\n", + "import chisel3.util._\n", + "import chisel3.experimental.BundleLiterals._\n", + "import chisel3.simulator.PeekPokeAPI._" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + ">This bootcamp requires some slight differences from the imports you might see \n", + "elsewhere for Chisel. The `import chisel3.simulator.PeekPokeAPI._` brings in the \n", + "`poke`, `peek`, `expect` and `step` methods on Chisel signals, and the `simulate(...)` function\n", + "used below is a thin wrapper (defined in `source/load-ivy.sc`) around ChiselSim's `EphemeralSimulator.simulate`\n", + "that prints failures nicely in the notebook." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "# Basic Tester implementation\n", + "\n", + "ChiselSim starts with the same basic operations as the older `iotesters`. Here's a brief summary of the basic\n", + "functionality mapping between the older `iotesters` and ChiselSim:\n", + "\n", + "| | iotesters | ChiselSim |\n", + "| :---- | :--- | :--- |\n", + "| poke | poke(c.io.in1, 6) | c.io.in1.poke(6.U) |\n", + "| peek | peek(c.io.out1) | c.io.out1.peek() |\n", + "| expect | expect(c.io.out1, 6) | c.io.out1.expect(6.U) |\n", + "| step | step(1) | c.clock.step(1) |\n", + "| initiate | Driver.execute(...) { c => | simulate(...) { c => |\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's start by looking at the simple pass through module from 2.1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "// Chisel Code, but pass in a parameter to set widths of ports\n", + "class PassthroughGenerator(width: Int) extends Module { \n", + " val io = IO(new Bundle {\n", + " val in = Input(UInt(width.W))\n", + " val out = Output(UInt(width.W))\n", + " })\n", + " io.out := io.in\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Using the old `iotesters` style (which no longer exists in current Chisel) a simple test would look like this:\n", + "\n", + "```scala\n", + "val testResult = Driver(() => new Passthrough()) {\n", + " c => new PeekPokeTester(c) {\n", + " poke(c.io.in, 0) // Set our input to value 0\n", + " expect(c.io.out, 0) // Assert that the output correctly has 0\n", + " poke(c.io.in, 1) // Set our input to value 1\n", + " expect(c.io.out, 1) // Assert that the output correctly has 1\n", + " poke(c.io.in, 2) // Set our input to value 2\n", + " expect(c.io.out, 2) // Assert that the output correctly has 2\n", + " }\n", + "}\n", + "assert(testResult) // Scala Code: if testResult == false, will throw an error\n", + "println(\"SUCCESS!!\") // Scala Code: if we get here, our tests passed!\n", + "```\n", + "\n", + "With ChiselSim the same test is written as follows.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "simulate(new PassthroughGenerator(16)) { c =>\n", + " c.io.in.poke(0.U) // Set our input to value 0\n", + " c.io.out.expect(0.U) // Assert that the output correctly has 0\n", + " c.io.in.poke(1.U) // Set our input to value 1\n", + " c.io.out.expect(1.U) // Assert that the output correctly has 1\n", + " c.io.in.poke(2.U) // Set our input to value 2\n", + " c.io.out.expect(2.U) // Assert that the output correctly has 2\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + ">Just to illustrate the way ChiselSim advances the clock we can\n", + "add some `step` operations to the previous example." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "simulate(new PassthroughGenerator(16)) { c =>\n", + " c.io.in.poke(0.U) // Set our input to value 0\n", + " c.clock.step(1) // advance the clock\n", + " c.io.out.expect(0.U) // Assert that the output correctly has 0\n", + " c.io.in.poke(1.U) // Set our input to value 1\n", + " c.clock.step(1) // advance the clock\n", + " c.io.out.expect(1.U) // Assert that the output correctly has 1\n", + " c.io.in.poke(2.U) // Set our input to value 2\n", + " c.clock.step(1) // advance the clock\n", + " c.io.out.expect(2.U) // Assert that the output correctly has 2\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "## What to notice in the above example\n", + "\n", + "ChiselSim's `simulate` method requires very little boilerplate: there is no separate tester class, the\n", + "test body is just a Scala function that receives the module under test (`c`).\n", + "\n", + "The `poke`, `peek` and `expect` methods are part of each individual `io` element.\n", + "This gives important hints to the tester to make better checking of types.\n", + "The clock is also a signal of the module (`c.clock`), so `step` is a method on it: `c.clock.step(n)`.\n", + "\n", + "Another difference is that values poked and expected are Chisel literals.\n", + "Although pretty simple here, it also provides stronger checking in more advanced and interesting examples, including `Bundle` literals, which we will see below.\n", + "\n", + "Three more things to be aware of:\n", + "- `simulate` first applies a reset to the module (one clock cycle with `reset` high), so `RegInit` registers start at their reset values before your test body runs. Registers without an initial value (plain `Reg`) hold random values until written.\n", + "- `peek()` returns a Chisel literal, so use `.litValue` to get a Scala `BigInt` (for example `c.io.out.peek().litValue`), and `peekBoolean()` on a `Bool` to get a Scala `Boolean`.\n", + "- Like `poke` and `expect`, a failing `expect` simply throws a Scala exception, which is how the test fails.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Modules with Decoupled Interfaces\n", + "In this section we will look at how to test modules with `Decoupled` interfaces.\n", + "`Decoupled` takes a chisel data type and provides it with `ready` and `valid` signals.\n", + "ChiselSim has no special support for these interfaces, but we do not need any: a test is ordinary Scala code, so we can write small helper functions that drive `ready` and `valid` for us, and reuse them in every test of every `Decoupled` circuit.\n", + "\n", + "## A queue example\n", + "The `QueueModule` passes through data whose type is determined by `ioType`. There are `entries` state elements inside the `QueueModule` meaning it can hold that many elements before it exerts backpressure." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class QueueModule[T <: Data](ioType: T, entries: Int) extends Module {\n", + " val in = IO(Flipped(Decoupled(ioType)))\n", + " val out = IO(Decoupled(ioType))\n", + " out <> Queue(in, entries)\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "// Tests are plain Scala, so we can write our own helpers.\n", + "\n", + "// Add (enqueue) one element to a Decoupled input interface.\n", + "// `drive` pokes the payload, e.g. `_.poke(5.U)`.\n", + "// We hold `valid` high until the DUT is `ready`; the transfer happens on the clock edge.\n", + "def enqueue[T <: Data](port: DecoupledIO[T], clock: Clock, timeout: Int = 1000)(drive: T => Unit): Unit = {\n", + " port.valid.poke(true.B)\n", + " drive(port.bits)\n", + " var cycles = 0\n", + " while (!port.ready.peekBoolean()) {\n", + " clock.step(1)\n", + " cycles += 1\n", + " assert(cycles < timeout, s\"timeout: ready was not asserted after $timeout cycles\")\n", + " }\n", + " clock.step(1) // the handshake (valid && ready) completes on this edge\n", + " port.valid.poke(false.B)\n", + "}\n", + "\n", + "// Remove (dequeue) one element from a Decoupled output interface and check it.\n", + "// `check` examines the payload, e.g. `_.expect(5.U)`.\n", + "// We raise `ready` and wait until the DUT presents a valid element.\n", + "def expectDequeue[T <: Data](port: DecoupledIO[T], clock: Clock, timeout: Int = 1000)(check: T => Unit): Unit = {\n", + " port.ready.poke(true.B)\n", + " var cycles = 0\n", + " while (!port.valid.peekBoolean()) {\n", + " clock.step(1)\n", + " cycles += 1\n", + " assert(cycles < timeout, s\"timeout: valid was not asserted after $timeout cycles\")\n", + " }\n", + " check(port.bits)\n", + " clock.step(1) // the handshake completes on this edge\n", + " port.ready.poke(false.B)\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Enqueue and dequeue helpers\n", + "Before running the test body, `simulate` resets the module for one clock cycle (this is what initializes the `Queue`'s internal state). If a test ever needs to restart the circuit, drive the reset like any other input: `c.reset.poke(true.B); c.clock.step(1); c.reset.poke(false.B)`.\n", + "\n", + "\n", + "In this example we define our own helper functions to insert and extract values from the `queue`. \n", + "\n", + "| function | description |\n", + "| :--- | :--- |\n", + "| enqueue | Adds (enqueues) one element to a `Decoupled` input interface, waiting until the DUT is `ready` |\n", + "| expectDequeue | Removes (dequeues) one element from a `Decoupled` output interface, waiting until it is `valid`, and checks its value |\n", + "---\n", + "\n", + "The functions are generic in the payload type `T <: Data`. The last argument is a function that does the type-specific part: poking or checking the `bits` field. \n", + "The `ready`/`valid` handshake itself is the same for every `Decoupled` interface, so we write it only once.\n", + "\n", + ">Note: Remember that a transfer happens on a clock edge where both `valid` and `ready` are high. The helpers therefore keep `valid` (or `ready`) high until the handshake can happen, step the clock once, and then lower the signal again. The `timeout` makes a hung handshake fail the test instead of looping forever.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "simulate(new QueueModule(UInt(9.W), entries = 200)) { c =>\n", + " c.out.ready.poke(false.B)\n", + " c.in.valid.poke(false.B)\n", + "\n", + " val testVector = Seq.tabulate(200){ i => i.U }\n", + "\n", + " testVector.foreach { value =>\n", + " enqueue(c.in, c.clock)(_.poke(value))\n", + " expectDequeue(c.out, c.clock)(_.expect(value))\n", + " }\n", + "}\n", + "println(\"SUCCESS!!\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Enqueuing and dequeuing whole sequences\n", + "Because a test is just Scala, enqueuing or dequeuing a whole sequence is just a `foreach` over the helper functions we wrote above.\n", + "\n", + "| operation | description |\n", + "| :--- | :--- |\n", + "| `testVector.foreach(v => enqueue(c.in, c.clock)(_.poke(v)))` | Continues to add (enqueue) elements from the `Seq` to a `Decoupled` input interface, one at a time, until the sequence is exhausted |\n", + "| `testVector.foreach(v => expectDequeue(c.out, c.clock)(_.expect(v)))` | Removes (dequeues) elements from a `Decoupled` output interface, one at a time, and compares each one to the next element of the `Seq` |\n", + "---\n", + "> Note: The example below works fine but, as written, all the enqueues must finish before the dequeues can begin. This example would fail if the `testVector`'s size is made larger than the queue depth, because the queue would fill up and `enqueue` could never complete (our `timeout` assertion reports this). Try it yourself to see what the failure looks like. In the next section we will show how to construct this type of test properly.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "simulate(new QueueModule(UInt(9.W), entries = 200)) { c =>\n", + " c.out.ready.poke(false.B)\n", + " c.in.valid.poke(false.B)\n", + "\n", + " val testVector = Seq.tabulate(100){ i => i.U }\n", + "\n", + " testVector.foreach { v => enqueue(c.in, c.clock)(_.poke(v)) }\n", + " testVector.foreach { v => expectDequeue(c.out, c.clock)(_.expect(v)) }\n", + "}\n", + "println(\"SUCCESS!!\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "> One important take away from the last section is that the functions we just saw, `enqueue` and `expectDequeue`, are not complicated special case logic in ChiselSim.\n", + "Rather they are examples of the kinds of harness building that ChiselSim encourages you to do on top of its primitives (`poke`, `peek`, `expect` and `step`). Building up your own library of such helpers is a good way to keep your tests short and readable." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Driving Several Interfaces at Once" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Sometimes sending and receiving must happen at the same time. For example, a queue must be emptied while it is being filled if we want to push more elements through it than it can hold.\n", + "Some other testing libraries provide `fork` and `join` to run several test threads concurrently. ChiselSim's peek/poke API does not have them: the test is a single sequential Scala program.\n", + "\n", + "That is not a limitation in practice. Hardware is synchronous, so all the \"concurrency\" is between clock edges. We simply write **one loop that runs once per clock cycle** and, in each iteration, drives the inputs of *every* interface, samples the handshakes, and then calls `step(1)`.\n", + "\n", + "In the example below, the producer side offers the next element while there are elements left to send, and the consumer side is always ready. Each cycle we check whether a handshake (`valid && ready`) is happening on each side *before* the clock edge, and update our bookkeeping accordingly.\n", + "The loop is bounded by a cycle limit so that a bug cannot hang the test.\n", + "\n", + ">In this style there is nothing hidden: you decide the order in which signals are poked and sampled in each cycle, so the behavior is completely deterministic." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "simulate(new QueueModule(UInt(9.W), entries = 200)) { c =>\n", + "\n", + " val n = 300 // more elements than the queue can hold!\n", + " var sent = 0\n", + " var received = 0\n", + " var cycles = 0\n", + "\n", + " c.out.ready.poke(true.B) // the consumer is always ready\n", + "\n", + " while (received < n) {\n", + " // producer: offer the next element while there are some left\n", + " if (sent < n) {\n", + " c.in.valid.poke(true.B)\n", + " c.in.bits.poke(sent.U)\n", + " } else {\n", + " c.in.valid.poke(false.B)\n", + " }\n", + "\n", + " // sample the handshakes that are about to happen on the next clock edge\n", + " val enqFires = sent < n && c.in.ready.peekBoolean()\n", + " val deqFires = c.out.valid.peekBoolean() // out.ready is always true\n", + "\n", + " if (deqFires) {\n", + " c.out.bits.expect(received.U) // data must come out in order\n", + " received += 1\n", + " }\n", + " if (enqFires) sent += 1\n", + "\n", + " c.clock.step(1)\n", + " cycles += 1\n", + " assert(cycles < 10 * n, \"timeout\")\n", + " }\n", + "}\n", + "println(\"SUCCESS!!\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Decoupled helpers with GCD\n", + "In this section we will use our helper functions to implement tests of *Greatest Common Divisor* **GCD**.\n", + "Let's start by defining our IO bundles. As you can see, there is no boilerplate needed to use `Bundle` *literals* (we imported `chisel3.experimental.BundleLiterals._` in the setup cell)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class GcdInputBundle(val w: Int) extends Bundle {\n", + " val value1 = UInt(w.W)\n", + " val value2 = UInt(w.W)\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class GcdOutputBundle(val w: Int) extends Bundle {\n", + " val value1 = UInt(w.W)\n", + " val value2 = UInt(w.W)\n", + " val gcd = UInt(w.W)\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's look at a *Decoupled* version of **GCD**. We've modified it a bit here to use the `Decoupled` wrapper that adds a `ready` and a `valid` signal to the input and output Bundle. The `Flipped` wrapper takes the `Decoupled` `GcdInputBundle` which by default is created as an output and converts each field to the opposite direction (recursively). The data elements of the bundled arguments to `Decoupled` are placed in the top level field `bits`. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "/**\n", + " * Compute GCD using subtraction method.\n", + " * Subtracts the smaller of registers x and y from the larger until register y is zero.\n", + " * value input register x is then the Gcd\n", + " * returns a packet of information with the two input values and their GCD\n", + " */\n", + "class DecoupledGcd(width: Int) extends Module {\n", + "\n", + " val input = IO(Flipped(Decoupled(new GcdInputBundle(width))))\n", + " val output = IO(Decoupled(new GcdOutputBundle(width)))\n", + "\n", + " val xInitial = Reg(UInt())\n", + " val yInitial = Reg(UInt())\n", + " val x = Reg(UInt())\n", + " val y = Reg(UInt())\n", + " val busy = RegInit(false.B)\n", + "\n", + " input.ready := ! busy\n", + " output.valid := false.B\n", + " output.bits := DontCare\n", + "\n", + " when(busy) {\n", + " // during computation keep subtracting the smaller from the larger\n", + " when(x > y) {\n", + " x := x - y\n", + " }.otherwise {\n", + " y := y - x\n", + " }\n", + " when(x === 0.U || y === 0.U) {\n", + " // when one of them becomes zero computation is over (the gcd is the other one),\n", + " // signal valid data to output if the output is ready\n", + " output.bits.value1 := xInitial\n", + " output.bits.value2 := yInitial\n", + " output.bits.gcd := x | y\n", + " output.valid := true.B\n", + " busy := ! output.ready\n", + " }\n", + " }.otherwise {\n", + " when(input.valid) {\n", + " // valid data available and no computation in progress, grab new values and start\n", + " val bundle = input.deq()\n", + " x := bundle.value1\n", + " y := bundle.value2\n", + " xInitial := bundle.value1\n", + " yInitial := bundle.value2\n", + " busy := true.B\n", + " }\n", + " }\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Our test looks pretty much the same as the earlier Queue tests.\n", + "But there's more going on because the computation takes multiple cycles, so the input is not `ready` while each GCD is being computed.\n", + "The good news is that test side of this is simple and consistent across different Decoupled circuits: our `enqueue` and `expectDequeue` helpers wait for `ready` and `valid` for us.\n", + "\n", + "We also introduce here the Chisel `Bundle` literal notation. Consider the line\n", + "```scala\n", + "new GcdInputBundle(16).Lit(_.value1 -> x.U, _.value2 -> y.U)\n", + "```\n", + "`GcdInputBundle` defined above has two fields `value1` and `value2`.\n", + "We create a bundle literal by first creating a bundle then calling its `.Lit` method.\n", + "That method takes a variable argument list of key/value pairs, where the key (e.g. `_.value1`) is the field name and the value (e.g. x.U) is a chisel hardware literal, the Scala `Int` x is converted into a Chisel `UInt` literal.\n", + "The `_.` in front of the field name is necessary to bind the name value to the bundle internals. \n", + "A whole bundle literal can be passed to `poke` and `expect`.\n", + "\n", + ">This may not be the perfect notation but in extensive development discussions it was viewed as\n", + "the best balance between minimizing boilerplate and the notational limitations available in Scala.\n", + "\n", + "The test below also waits 5 cycles before accepting each result (`c.output.ready` stays low during that time) to create backpressure on the GCD module.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "simulate(new DecoupledGcd(16)) { dut =>\n", + " dut.input.valid.poke(false.B)\n", + " dut.output.ready.poke(false.B)\n", + "\n", + " val testValues = for { x <- 1 to 10; y <- 1 to 10} yield (x, y)\n", + "\n", + " for ((x, y) <- testValues) {\n", + " val input = new GcdInputBundle(16).Lit(_.value1 -> x.U, _.value2 -> y.U)\n", + " val expected = new GcdOutputBundle(16).Lit(_.value1 -> x.U, _.value2 -> y.U, _.gcd -> BigInt(x).gcd(BigInt(y)).U)\n", + "\n", + " enqueue(dut.input, dut.clock)(_.poke(input))\n", + " dut.clock.step(5) // wait some cycles before receiving the output to create backpressure\n", + " expectDequeue(dut.output, dut.clock)(_.expect(expected))\n", + " }\n", + "}\n", + "println(\"SUCCESS!!\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "# You're done!\n", + "\n", + "[Return to the top.](#top)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Scala", + "language": "scala", + "name": "scala" + }, + "language_info": { + "codemirror_mode": "text/x-scala", + "file_extension": ".scala", + "mimetype": "text/x-scala", + "name": "scala", + "nbconvert_exporter": "script", + "version": "2.13.18" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/2.6_chiseltest.ipynb b/2.6_chiseltest.ipynb deleted file mode 100644 index 7e8cc4d..0000000 --- a/2.6_chiseltest.ipynb +++ /dev/null @@ -1,505 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\"Chisel" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "##### Module 2.6: More on ChiselTest\n", - "**Prev: [Putting it all Together: An FIR Filter](2.5_exercise.ipynb)**
\n", - "**Next: [Generators: Parameters](3.1_parameters.ipynb)**\n", - "\n", - "## Motivation\n", - "The Chisel team has been working on an improved testing framework. \"ChiselTest\", it provides the following improvements .\n", - "\n", - "- suitable for both unit tests and system integration tests\n", - "- designed for composable abstractions and layering\n", - "- highly usable, encouraging unit tests by making it as easy, painless (avoiding boilerplate and other nonsense), and useful as possible to write them\n", - "\n", - "### Planned\n", - "- ablity to target multiple backends and simulators (possibly requiring a link to Scala, if the testvector is not static, or using a limited test constructing API subset, when synthesizing to FPGA)\n", - "- will be included in base chisel3, to avoid packaging and dependency nightmares\n", - "\n", - "\n", - "## Setup" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "val path = System.getProperty(\"user.dir\") + \"/source/load-ivy.sc\"\n", - "interp.load.module(ammonite.ops.Path(java.nio.file.FileSystems.getDefault().getPath(path)))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import chisel3._\n", - "import chisel3.util._\n", - "import chisel3.experimental._\n", - "import chisel3.experimental.BundleLiterals._\n", - "import chiseltest._\n", - "import chiseltest.RawTester.test" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - ">This bootcamp requires some slight differences from the imports you might see \n", - "elsewhere for chisel. The `import chiseltest.RawTester.test` brings in \n", - "version of `test(...)` below that is designed specifically for the bootcamp" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "# Basic Tester implementation\n", - "\n", - "ChiselTest starts with the same basic operations as iotesters. Here's a brief summary of the basic\n", - "functionality mapping between the older iotesters and the new ChiselTest\n", - "\n", - "| | iotesters | ChiselTest |\n", - "| :---- | :--- | :--- |\n", - "| poke | poke(c.io.in1, 6) | c.io.in1.poke(6.U) |\n", - "| peek | peek(c.io.out1) | c.io.out1.peek() |\n", - "| expect | expect(c.io.out1, 6) | c.io.out1.expect(6.U) |\n", - "| step | step(1) | c.io.clock.step(1) |\n", - "| initiate | Driver.execute(...) { c => | test(...) { c => |\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's start by looking at the simple pass through module from 2.1" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "// Chisel Code, but pass in a parameter to set widths of ports\n", - "class PassthroughGenerator(width: Int) extends Module { \n", - " val io = IO(new Bundle {\n", - " val in = Input(UInt(width.W))\n", - " val out = Output(UInt(width.W))\n", - " })\n", - " io.out := io.in\n", - "}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Using the old style a simple test would look like this\n", - "\n", - "```scala\n", - "val testResult = Driver(() => new Passthrough()) {\n", - " c => new PeekPokeTester(c) {\n", - " poke(c.io.in, 0) // Set our input to value 0\n", - " expect(c.io.out, 0) // Assert that the output correctly has 0\n", - " poke(c.io.in, 1) // Set our input to value 1\n", - " expect(c.io.out, 1) // Assert that the output correctly has 1\n", - " poke(c.io.in, 2) // Set our input to value 2\n", - " expect(c.io.out, 2) // Assert that the output correctly has 2\n", - " }\n", - "}\n", - "assert(testResult) // Scala Code: if testResult == false, will throw an error\n", - "println(\"SUCCESS!!\") // Scala Code: if we get here, our tests passed!\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "test(new PassthroughGenerator(16)) { c =>\n", - " c.io.in.poke(0.U) // Set our input to value 0\n", - " c.io.out.expect(0.U) // Assert that the output correctly has 0\n", - " c.io.in.poke(1.U) // Set our input to value 1\n", - " c.io.out.expect(1.U) // Assert that the output correctly has 1\n", - " c.io.in.poke(2.U) // Set our input to value 2\n", - " c.io.out.expect(2.U) // Assert that the output correctly has 2\n", - "}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - ">Just to illustrate the way ChiselTest advances the clock we can\n", - "add some `step` operations to the previous examples." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "test(new PassthroughGenerator(16)) { c =>\n", - " c.io.in.poke(0.U) // Set our input to value 0\n", - " c.clock.step(1) // advance the clock\n", - " c.io.out.expect(0.U) // Assert that the output correctly has 0\n", - " c.io.in.poke(1.U) // Set our input to value 1\n", - " c.clock.step(1) // advance the clock\n", - " c.io.out.expect(1.U) // Assert that the output correctly has 1\n", - " c.io.in.poke(2.U) // Set our input to value 2\n", - " c.clock.step(1) // advance the clock\n", - " c.io.out.expect(2.U) // Assert that the output correctly has 2\n", - "}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "## What to notice in the above example\n", - "\n", - "ChiselTest's `test` method requires a bit less boiler plate. What was the `PeekPokeTester` is now\n", - "built into the process.\n", - "\n", - "The `poke` and `expect` methods are now part of each individual `io` element.\n", - "This gives important hints to the tester to make better checking of types.\n", - "The `peek` and `step` operations are also now methods on `io` elements.\n", - "\n", - "Another difference is that values poked and expected are Chisel literals.\n", - "Although pretty simple here, it also provides stronger checking in more advanced and interesting examples.\n", - "This will be further enhanced with coming improvements in the ability to specify `Bundle` literals\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": "# Modules with Decoupled Interfaces\nIn this section we will look at some of ChiselTest's tools for working with `Decoupled` interfaces.\n`Decoupled` takes a chisel data type and provides it with `ready` and `valid` signals.\nChiselTest provides some nice tools for automating and reliably testing these interfaces.\n\n## A queue example\nThe `QueueModule` passes through data whose type is determined by `ioType`. There are `entries` state elements inside the `QueueModule` meaning it can hold that many elements before it exerts backpressure." - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class QueueModule[T <: Data](ioType: T, entries: Int) extends MultiIOModule {\n", - " val in = IO(Flipped(Decoupled(ioType)))\n", - " val out = IO(Decoupled(ioType))\n", - " out <> Queue(in, entries)\n", - "}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## EnqueueNow and expectDequeueNow\n", - "*ChiselTest* has some built in methods for dealing with circuits with Decoupled interfaces in the IOs. In this example we will see how to insert and extract values from the `queue`. \n", - "\n", - "| method | description |\n", - "| :--- | :--- |\n", - "| enqueueNow | Add (enqueue) one element to a `Decoupled` input interface |\n", - "| expectDequeueNow | Removes (dequeues) one element from a `Decoupled` output interface |\n", - "---\n", - "\n", - "\n", - ">Note: There is some required boiler plate `initSource`, `setSourceClock`, etc that is necessary to ensure that the `ready` and `valid` fields are\n", - "all initialized correctly at the beginning of the test.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "test(new QueueModule(UInt(9.W), entries = 200)) { c =>\n", - " // Example testsequence showing the use and behavior of Queue\n", - " c.in.initSource()\n", - " c.in.setSourceClock(c.clock)\n", - " c.out.initSink()\n", - " c.out.setSinkClock(c.clock)\n", - " \n", - " val testVector = Seq.tabulate(200){ i => i.U }\n", - "\n", - " testVector.zip(testVector).foreach { case (in, out) =>\n", - " c.in.enqueueNow(in)\n", - " c.out.expectDequeueNow(out)\n", - " }\n", - "}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## EnqueueSeq and DequeueSeq \n", - "Now we are going to introduce two new methods that deal with enqueuing and dequeuing operations in single operations.\n", - "\n", - "| method | description |\n", - "| :--- | :--- |\n", - "| enqueueSeq | Continues to add (enqueue) elements from the `Seq` to a `Decoupled` input interface, one at a time, until the sequence is exhausted |\n", - "| expectDequeueSeq | Removes (dequeues) elements from a `Decoupled` output interface, one at a time, and compares each one to the next element of the `Seq` |\n", - "---\n", - "> Note: The example below works fine but, as written, the `enqueueSeq` must finish before the `expectDequeueSeq` can begin. This example would fail if the `testVector`'s size is made larger than the queue depth, because the queue would fill up and not be able to complete the `enqueueSeq`. Try it yourself to see what the failure looks like. In the next section we will show to construct this type of test properly.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "test(new QueueModule(UInt(9.W), entries = 200)) { c =>\n", - " // Example testsequence showing the use and behavior of Queue\n", - " c.in.initSource()\n", - " c.in.setSourceClock(c.clock)\n", - " c.out.initSink()\n", - " c.out.setSinkClock(c.clock)\n", - " \n", - " val testVector = Seq.tabulate(100){ i => i.U }\n", - "\n", - " c.in.enqueueSeq(testVector)\n", - " c.out.expectDequeueSeq(testVector)\n", - "}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": "> One more important take away from the last section is that the functions we just saw, `enqueueNow`, \n`enqueueSeq`, `expectDequeueNow`, and `expectDequeueSeq` are not complicated special case logic in ChiselTest.\nRather they are examples of the kinds of harness building that ChiselTest encourages you to build from the ChiselTest primitives. To see how these methods are implemented check out [TestAdapters.scala](https://github.com/ucb-bar/chiseltest/blob/main/src/main/scala/chiseltest/TestAdapters.scala)" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Fork and Join in ChiselTest" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": "In this section we will look at running sections of a unit test concurrently. In order to do this we will introduce two new features of ChiselTest.\n\n| method | description |\n| :--- | :--- |\n| fork | launches a concurrent code block, additional forks can be run concurrently to this one via the .fork appended to end of the code block of the preceeding fork |\n| join | re-unites multiple related forks back into the calling thread |\n---\n\nIn the example below two `fork`s are chained together, and then `join`ed. In the first `fork` block the `enqueueSeq` will continue to add elements until exhausted. The second `fork` block will `expectDequeueSeq` on each cycle when data is available.\n\n>The threads created by fork are run in a deterministic order, largely according to their order as specified in code, and certain bug-prone operations that depend on other threads are forbidden with runtime checks. " - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "test(new QueueModule(UInt(9.W), entries = 200)) { c =>\n", - " // Example testsequence showing the use and behavior of Queue\n", - " c.in.initSource()\n", - " c.in.setSourceClock(c.clock)\n", - " c.out.initSink()\n", - " c.out.setSinkClock(c.clock)\n", - " \n", - " val testVector = Seq.tabulate(300){ i => i.U }\n", - "\n", - " fork {\n", - " c.in.enqueueSeq(testVector)\n", - " }.fork {\n", - " c.out.expectDequeueSeq(testVector)\n", - " }.join()\n", - "}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Using Fork and Join with GCD\n", - "In this section we will use the fork join methods to implement tests of *Greatest Common Denominator* **GCD**.\n", - "Let's start by defining our IO bundles. We are going to add a bit of boiler plate here to allow us to use `Bundle` *literals*. As the comments say, it is hoped that we will soon have support for autogeneration of the literal support code." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class GcdInputBundle(val w: Int) extends Bundle {\n", - " val value1 = UInt(w.W)\n", - " val value2 = UInt(w.W)\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class GcdOutputBundle(val w: Int) extends Bundle {\n", - " val value1 = UInt(w.W)\n", - " val value2 = UInt(w.W)\n", - " val gcd = UInt(w.W)\n", - "}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now let's look at a *Decoupled* version of **GCD**. We've modified it a bit here to use the `Decoupled` wrapper that adds a `ready` and a `valid` signal to the input and output Bundle. The `Flipped` wrapper takes the `Decoupled` `GcdInputBundle` which by default is created as an output and converts each field to the opposite direction (recursively). The data elements of the bundled arguments to `Decoupled` are placed in the top level field `bits`. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "/**\n", - " * Compute GCD using subtraction method.\n", - " * Subtracts the smaller of registers x and y from the larger until register y is zero.\n", - " * value input register x is then the Gcd\n", - " * returns a packet of information with the two input values and their GCD\n", - " */\n", - "class DecoupledGcd(width: Int) extends MultiIOModule {\n", - "\n", - " val input = IO(Flipped(Decoupled(new GcdInputBundle(width))))\n", - " val output = IO(Decoupled(new GcdOutputBundle(width)))\n", - "\n", - " val xInitial = Reg(UInt())\n", - " val yInitial = Reg(UInt())\n", - " val x = Reg(UInt())\n", - " val y = Reg(UInt())\n", - " val busy = RegInit(false.B)\n", - " val resultValid = RegInit(false.B)\n", - "\n", - " input.ready := ! busy\n", - " output.valid := resultValid\n", - " output.bits := DontCare\n", - "\n", - " when(busy) {\n", - " // during computation keep subtracting the smaller from the larger\n", - " when(x > y) {\n", - " x := x - y\n", - " }.otherwise {\n", - " y := y - x\n", - " }\n", - " when(y === 0.U) {\n", - " // when y becomes zero computation is over,\n", - " // signal valid data to output if the output is ready\n", - " output.bits.value1 := xInitial\n", - " output.bits.value2 := yInitial\n", - " output.bits.gcd := x\n", - " output.valid := true.B\n", - " busy := ! output.ready\n", - " }\n", - " }.otherwise {\n", - " when(input.valid) {\n", - " // valid data available and no computation in progress, grab new values and start\n", - " val bundle = input.deq()\n", - " x := bundle.value1\n", - " y := bundle.value2\n", - " xInitial := bundle.value1\n", - " yInitial := bundle.value2\n", - " busy := true.B\n", - " resultValid := false.B\n", - " }\n", - " }\n", - "}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Our test looks pretty much the same as the earlier Queue tests.\n", - "But there's more going on because the computation take multiple cycles so the input enqueue process is blocked as each GCD is computed.\n", - "The good news is that test side of this is simple and consistent across different Decoupled circuits.\n", - "\n", - "Also introduced here is the new Chisel3 `Bundle` literal notation. Consider the line\n", - "```scala\n", - "new GcdInputBundle(16).Lit(_.value1 -> x.U, _.value2 -> y.U)\n", - "```\n", - "`GcdInputBundle` defined above has two fields `value1` and `value2`.\n", - "We create a bundle literal by first creating a bundle then calling its `.Lit` method.\n", - "That method takes a variable argument list of key/value pairs, where the key (e.g. `_.value`) is the field name and the value (e.g. x.U) is a chisel hardware literal, the Scala `Int` x is converted into a Chisel `UInt` literal.\n", - "The `_.` in front of the field name is necessary to bind the name value to the bundle internals. \n", - "\n", - ">This may not be the perfect notation but in extensive development discussions it was viewed as\n", - "the best balance between minimizing boilerplate and the notational limitations available in Scala.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "test(new DecoupledGcd(16)) { dut =>\n", - " dut.input.initSource().setSourceClock(dut.clock)\n", - " dut.output.initSink().setSinkClock(dut.clock)\n", - "\n", - " val testValues = for { x <- 1 to 10; y <- 1 to 10} yield (x, y)\n", - " val inputSeq = testValues.map { case (x, y) =>\n", - " (new GcdInputBundle(16)).Lit(_.value1 -> x.U, _.value2 -> y.U)\n", - " }\n", - " val resultSeq = testValues.map { case (x, y) =>\n", - " new GcdOutputBundle(16).Lit(_.value1 -> x.U, _.value2 -> y.U, _.gcd -> BigInt(x).gcd(BigInt(y)).U)\n", - " }\n", - "\n", - " fork {\n", - " dut.input.enqueueSeq(inputSeq)\n", - " }.fork {\n", - " for (expected <- resultSeq) {\n", - " dut.output.expectDequeue(expected)\n", - " dut.clock.step(5) // wait some cycles before receiving the next output to create backpressure\n", - " }\n", - " }.join()\n", - "}\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "---\n", - "# You're done!\n", - "\n", - "[Return to the top.](#top)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Scala", - "language": "scala", - "name": "scala" - }, - "language_info": { - "codemirror_mode": "text/x-scala", - "file_extension": ".scala", - "mimetype": "text/x-scala", - "name": "scala", - "nbconvert_exporter": "script", - "version": "2.12.10" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} \ No newline at end of file diff --git a/3.1_parameters.ipynb b/3.1_parameters.ipynb index 47f662f..1b051cb 100644 --- a/3.1_parameters.ipynb +++ b/3.1_parameters.ipynb @@ -12,7 +12,7 @@ "metadata": {}, "source": [ "# Module 3.1: Generators: Parameters\n", - "**Prev: [ChiselTest (was chisel-testers2)](2.6_chiseltest.ipynb)**
\n", + "**Prev: [Testing with ChiselSim](2.6_chiselsim.ipynb)**
\n", "**Next: [Generators: Collections](3.2_collections.ipynb)**\n", "\n", "## Motivation\n", @@ -31,8 +31,7 @@ "metadata": {}, "outputs": [], "source": [ - "val path = System.getProperty(\"user.dir\") + \"/source/load-ivy.sc\"\n", - "interp.load.module(ammonite.ops.Path(java.nio.file.FileSystems.getDefault().getPath(path)))" + "interp.load.module(os.pwd / \"source\" / \"load-ivy.sc\")" ] }, { @@ -42,9 +41,9 @@ "outputs": [], "source": [ "import chisel3._\n", + "import chisel3.Input // avoid a name clash with almond.input.Input\n", "import chisel3.util._\n", - "import chiseltest._\n", - "import chiseltest.RawTester.test" + "import chisel3.simulator.PeekPokeAPI._" ] }, { @@ -82,7 +81,7 @@ "**Example: Parameterized Chisel Object**
\n", "Chisel modules can be parameterized the same way.\n", "The following module has parameters for the widths of all its inputs and outputs.\n", - "Running the code block will print generated Verilog.\n", + "Running the code block will print the generated (System)Verilog.\n", "Play with the parameters and check that the output changes to reflect new parameters." ] }, @@ -217,7 +216,7 @@ "\n", "\n", "// Here are the testers\n", - "test(new Sort4(true)) { c => \n", + "simulate(new Sort4(true)) { c => \n", " c.io.in0.poke(3.U)\n", " c.io.in1.poke(6.U)\n", " c.io.in2.poke(9.U)\n", @@ -245,7 +244,7 @@ " c.io.out2.expect(6.U)\n", " c.io.out3.expect(13.U)\n", "}\n", - "test(new Sort4(false)) { c =>\n", + "simulate(new Sort4(false)) { c =>\n", " c.io.in0.poke(3.U)\n", " c.io.in1.poke(6.U)\n", " c.io.in2.poke(9.U)\n", @@ -601,7 +600,7 @@ " io.carryOut := sum(1)\n", "}\n", "\n", - "test(new HalfFullAdder(false)) { c =>\n", + "simulate(new HalfFullAdder(false)) { c =>\n", " require(!c.hasCarry, \"DUT must be half adder\")\n", " // 0 + 0 = 0\n", " c.io.a.poke(0.U)\n", @@ -622,7 +621,7 @@ " c.io.carryOut.expect(0.U)\n", "}\n", "\n", - "test(new HalfFullAdder(true)) { c =>\n", + "simulate(new HalfFullAdder(true)) { c =>\n", " require(c.hasCarry, \"DUT must be half adder\")\n", " c.io.carryIn.get.poke(0.U)\n", " // 0 + 0 + 0 = 0\n", @@ -856,7 +855,7 @@ "class BinaryMealy(val mp: BinaryMealyParams) extends Module {\n", " val io = IO(new Bundle {\n", " val in = Input(Bool())\n", - " val out = Output(UInt())\n", + " val out = Output(UInt(1.W))\n", " })\n", "\n", " val state = RegInit(UInt(), mp.s0.U)\n", @@ -899,7 +898,7 @@ "\n", "val testParams = BinaryMealyParams(nStates, s0, stateTransition, output)\n", "\n", - "test(new BinaryMealy(testParams)) { c =>\n", + "simulate(new BinaryMealy(testParams)) { c =>\n", " c.io.in.poke(false.B)\n", " c.io.out.expect(0.U)\n", " c.clock.step(1)\n", @@ -954,7 +953,7 @@ "mimetype": "text/x-scala", "name": "scala", "nbconvert_exporter": "script", - "version": "2.12.10" + "version": "2.13.18" } }, "nbformat": 4, diff --git a/3.2_collections.ipynb b/3.2_collections.ipynb index 1c3e069..ab44d76 100644 --- a/3.2_collections.ipynb +++ b/3.2_collections.ipynb @@ -30,15 +30,15 @@ "metadata": {}, "outputs": [], "source": [ - "val path = System.getProperty(\"user.dir\") + \"/source/load-ivy.sc\"\n", - "interp.load.module(ammonite.ops.Path(java.nio.file.FileSystems.getDefault().getPath(path)))" + "interp.load.module(os.pwd / \"source\" / \"load-ivy.sc\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Note we add a new import here because `mutable.ArrayBuffer` lives in `scala.collections`." + "Note we add a new import here because `mutable.ArrayBuffer` lives in `scala.collection`.\n", + "We import only the `mutable` package, not all of `scala.collection._`: the latter would also replace the default (immutable) `Seq` by `scala.collection.Seq`, which Chisel's `Vec(...)` and `VecInit(...)` do not accept in Scala 2.13." ] }, { @@ -48,10 +48,10 @@ "outputs": [], "source": [ "import chisel3._\n", + "import chisel3.Input // avoid a name clash with almond.input.Input\n", "import chisel3.util._\n", - "import chiseltest._\n", - "import chiseltest.RawTester.test\n", - "import scala.collection._" + "import chisel3.simulator.PeekPokeAPI._\n", + "import scala.collection.mutable" ] }, { @@ -203,7 +203,7 @@ "source": [ "val goldenModel = new ScalaFirFilter(Seq(1, 1, 1, 1))\n", "\n", - "test(new My4ElementFir(1, 1, 1, 1)) { c =>\n", + "simulate(new My4ElementFir(1, 1, 1, 1)) { c =>\n", " for(i <- 0 until 100) {\n", " val input = scala.util.Random.nextInt(8)\n", "\n", @@ -314,7 +314,7 @@ "source": [ "val goldenModel = new ScalaFirFilter(Seq(1, 1, 1, 1))\n", "\n", - "test(new MyManyElementFir(Seq(1, 1, 1, 1), 8)) { c =>\n", + "simulate(new MyManyElementFir(Seq(1, 1, 1, 1), 8)) { c =>\n", " for(i <- 0 until 100) {\n", " val input = scala.util.Random.nextInt(8)\n", "\n", @@ -353,10 +353,10 @@ " * run a test comparing software and hardware filters\n", " * run for at least twice as many samples as taps\n", " */\n", - "def runOneTest(taps: Seq[Int]) {\n", + "def runOneTest(taps: Seq[Int]): Unit = {\n", " val goldenModel = new ScalaFirFilter(taps)\n", "\n", - " test(new MyManyElementFir(taps, 32)) { c =>\n", + " simulate(new MyManyElementFir(taps, 32)) { c =>\n", " for(i <- 0 until 2 * taps.length) {\n", " val input = r()\n", "\n", @@ -458,7 +458,7 @@ "source": [ "val goldenModel = new ScalaFirFilter(Seq(1, 1, 1, 1))\n", "\n", - "test(new MyManyDynamicElementVecFir(4)) { c =>\n", + "simulate(new MyManyDynamicElementVecFir(4)) { c =>\n", " c.io.consts(0).poke(1.U)\n", " c.io.consts(1).poke(1.U)\n", " c.io.consts(2).poke(1.U)\n", @@ -526,7 +526,7 @@ "metadata": {}, "outputs": [], "source": [ - "test(new RegisterFile(2) ) { c =>\n", + "simulate(new RegisterFile(2) ) { c =>\n", " def readExpect(addr: Int, value: Int, port: Int = 0): Unit = {\n", " c.io.raddr(port).poke(addr.U)\n", " c.io.rdata(port).expect(value.U)\n", @@ -602,7 +602,7 @@ "mimetype": "text/x-scala", "name": "scala", "nbconvert_exporter": "script", - "version": "2.12.10" + "version": "2.13.18" } }, "nbformat": 4, diff --git a/3.2_interlude.ipynb b/3.2_interlude.ipynb index d246118..983c29a 100644 --- a/3.2_interlude.ipynb +++ b/3.2_interlude.ipynb @@ -27,8 +27,7 @@ "metadata": {}, "outputs": [], "source": [ - "val path = System.getProperty(\"user.dir\") + \"/source/load-ivy.sc\"\n", - "interp.load.module(ammonite.ops.Path(java.nio.file.FileSystems.getDefault().getPath(path)))" + "interp.load.module(os.pwd / \"source\" / \"load-ivy.sc\")" ] }, { @@ -38,9 +37,9 @@ "outputs": [], "source": [ "import chisel3._\n", + "import chisel3.Input // avoid a name clash with almond.input.Input\n", "import chisel3.util._\n", - "import chiseltest._\n", - "import chiseltest.RawTester.test" + "import chisel3.simulator.PeekPokeAPI._" ] }, { @@ -78,7 +77,9 @@ "\n", "`Queue` creates a FIFO (first-in, first-out) queue with Decoupled interfaces on both sides, allowing backpressure. Both the data type and number of elements are configurable. \n", "\n", - ">Notice how the first few times we print out the `peek`ed value we call `litValue` on it. This converts the chisel literal that is returned into a `BigInt`. Later similar calls we do not call `litValue` and you can see extra information about the value `peek` returns. like the type and the width." + ">Notice that we call `litValue` on each `peek`ed value. This converts the Chisel literal that `peek` returns into a Scala `BigInt`; without it you would see extra information about the value, like its type and width.\n", + "\n", + ">Also note that the queue's memory is not reset, so `io.out.bits` holds an arbitrary (possibly random) value whenever `io.out.valid` is 0. Only look at `bits` when `valid` is asserted." ] }, { @@ -87,7 +88,7 @@ "metadata": {}, "outputs": [], "source": [ - "test(new Module {\n", + "simulate(new Module {\n", " // Example circuit using a Queue\n", " val io = IO(new Bundle {\n", " val in = Flipped(Decoupled(UInt(8.W)))\n", @@ -158,7 +159,7 @@ "metadata": {}, "outputs": [], "source": [ - "test(new Module {\n", + "simulate(new Module {\n", " // Example circuit using a priority arbiter\n", " val io = IO(new Bundle {\n", " val in = Flipped(Vec(2, Decoupled(UInt(8.W))))\n", @@ -217,7 +218,7 @@ "metadata": {}, "outputs": [], "source": [ - "test(new Module {\n", + "simulate(new Module {\n", " // Example circuit using PopCount\n", " val io = IO(new Bundle {\n", " val in = Input(UInt(8.W))\n", @@ -247,7 +248,7 @@ "metadata": {}, "outputs": [], "source": [ - "test(new Module {\n", + "simulate(new Module {\n", " // Example circuit using Reverse\n", " val io = IO(new Bundle {\n", " val in = Input(UInt(8.W))\n", @@ -288,7 +289,7 @@ "metadata": {}, "outputs": [], "source": [ - "test(new Module {\n", + "simulate(new Module {\n", " // Example circuit using UIntToOH\n", " val io = IO(new Bundle {\n", " val in = Input(UInt(4.W))\n", @@ -316,7 +317,7 @@ "metadata": {}, "outputs": [], "source": [ - "test(new Module {\n", + "simulate(new Module {\n", " // Example circuit using OHToUInt\n", " val io = IO(new Bundle {\n", " val in = Input(UInt(16.W))\n", @@ -366,7 +367,7 @@ "metadata": {}, "outputs": [], "source": [ - "test(new Module {\n", + "simulate(new Module {\n", " // Example circuit using PriorityMux\n", " val io = IO(new Bundle {\n", " val in_sels = Input(Vec(2, Bool()))\n", @@ -401,7 +402,7 @@ "metadata": {}, "outputs": [], "source": [ - "test(new Module {\n", + "simulate(new Module {\n", " // Example circuit using Mux1H\n", " val io = IO(new Bundle {\n", " val in_sels = Input(Vec(2, Bool()))\n", @@ -451,7 +452,7 @@ "metadata": {}, "outputs": [], "source": [ - "test(new Module {\n", + "simulate(new Module {\n", " // Example circuit using Mux1H\n", " val io = IO(new Bundle {\n", " val count = Input(Bool())\n", @@ -506,7 +507,7 @@ "mimetype": "text/x-scala", "name": "scala", "nbconvert_exporter": "script", - "version": "2.12.10" + "version": "2.13.18" } }, "nbformat": 4, diff --git a/3.3_higher-order_functions.ipynb b/3.3_higher-order_functions.ipynb index b8f2142..1d2f61d 100644 --- a/3.3_higher-order_functions.ipynb +++ b/3.3_higher-order_functions.ipynb @@ -27,8 +27,7 @@ "metadata": {}, "outputs": [], "source": [ - "val path = System.getProperty(\"user.dir\") + \"/source/load-ivy.sc\"\n", - "interp.load.module(ammonite.ops.Path(java.nio.file.FileSystems.getDefault().getPath(path)))" + "interp.load.module(os.pwd / \"source\" / \"load-ivy.sc\")" ] }, { @@ -38,9 +37,9 @@ "outputs": [], "source": [ "import chisel3._\n", + "import chisel3.Input // avoid a name clash with almond.input.Input\n", "import chisel3.util._\n", - "import chiseltest._\n", - "import chiseltest.RawTester.test" + "import chisel3.simulator.PeekPokeAPI._" ] }, { @@ -306,7 +305,7 @@ " ???\n", "}\n", "\n", - "test(new MyRoutingArbiter(4)) { c =>\n", + "simulate(new MyRoutingArbiter(4)) { c =>\n", " // verify that the computation is correct\n", " // Set input defaults\n", " for(i <- 0 until 4) {\n", @@ -396,7 +395,7 @@ "mimetype": "text/x-scala", "name": "scala", "nbconvert_exporter": "script", - "version": "2.12.10" + "version": "2.13.18" } }, "nbformat": 4, diff --git a/3.4_functional_programming.ipynb b/3.4_functional_programming.ipynb index dd02494..80d0c2d 100644 --- a/3.4_functional_programming.ipynb +++ b/3.4_functional_programming.ipynb @@ -27,15 +27,15 @@ "metadata": {}, "outputs": [], "source": [ - "val path = System.getProperty(\"user.dir\") + \"/source/load-ivy.sc\"\n", - "interp.load.module(ammonite.ops.Path(java.nio.file.FileSystems.getDefault().getPath(path)))" + "interp.load.module(os.pwd / \"source\" / \"load-ivy.sc\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "This module uses the Chisel `FixedPoint` type, which currently resides in the experimental package." + "This module does not need any special library: the neuron exercise at the end uses ordinary `SInt` values as fixed-point numbers.\n", + "Chisel 3 had an experimental `FixedPoint` type, but it has been removed, so we will keep track of the binary point ourselves." ] }, { @@ -45,11 +45,9 @@ "outputs": [], "source": [ "import chisel3._\n", + "import chisel3.Input // avoid a name clash with almond.input.Input\n", "import chisel3.util._\n", - "import chiseltest._\n", - "import chiseltest.RawTester.test\n", - "import chisel3.experimental._\n", - "import chisel3.internal.firrtl.KnownBinaryPoint" + "import chisel3.simulator.PeekPokeAPI._" ] }, { @@ -172,11 +170,9 @@ "metadata": {}, "outputs": [], "source": [ - "import scala.util.Random\n", - "\n", "// both x and y call the nextInt function, but x is evaluated immediately and y is a function\n", - "val x = Random.nextInt\n", - "def y = Random.nextInt\n", + "val x = scala.util.Random.nextInt()\n", + "def y = scala.util.Random.nextInt()\n", "\n", "// x was previously evaluated, so it is a constant\n", "println(s\"x = $x\")\n", @@ -378,7 +374,7 @@ "Those last three lines could be easily combined into one. Also notice how we've handled bitwidth growth conservatively to avoid loss.\n", "\n", "**Example: FIR Filter Tester**
\n", - "Let's test our FIR! Previously, we provided a custom golden model. This time we'll use Breeze, a Scala library of useful linear algebra and signal processing functions, as a golden model for our FIR filter. The code below compares the Chisel output with the golden model output, and any errors cause the tester to fail.\n", + "Let's test our FIR! Previously, we provided a custom golden model. This time the golden model is a plain Scala function that computes the convolution of the input samples with the window coefficients. Testers are just Scala code, so any Scala function can serve as a reference. The code below compares the Chisel output with the golden model output, and any errors cause the tester to fail.\n", "\n", "Try uncommenting the print statment at the end just after the expect call. Also try changing the window from triangular to Hamming." ] @@ -391,9 +387,13 @@ "source": [ "// math imports\n", "import scala.math.{pow, sin, Pi}\n", - "import breeze.signal.{filter, OptOverhang}\n", - "import breeze.signal.support.{CanFilter, FIRKernel1D}\n", - "import breeze.linalg.DenseVector\n", + "\n", + "// golden model: out(n) = sum over k of coeffs(k) * in(n-k), computed with Long to avoid overflow\n", + "def goldenFir(in: Seq[Int], coeffs: Seq[Int]): Seq[Long] = {\n", + " in.indices.map { n =>\n", + " coeffs.indices.filter(_ <= n).map(k => coeffs(k).toLong * in(n - k)).sum\n", + " }\n", + "}\n", "\n", "// test parameters\n", "val length = 7\n", @@ -401,7 +401,7 @@ "val window = TriangularWindow\n", "\n", "// test our FIR\n", - "test(new MyFir(length, bitwidth, window)) { c =>\n", + "simulate(new MyFir(length, bitwidth, window)) { c =>\n", " \n", " // test data\n", " val n = 100 // input length\n", @@ -411,20 +411,15 @@ " // sample data, scale to between 0 and 2^bitwidth\n", " val max_value = pow(2, bitwidth)-1\n", " val sine = (0 until n).map(i => (max_value/2 + max_value/2*sin(2*Pi*sine_freq/samp_freq*i)).toInt)\n", - " //println(s\"input = ${sine.toArray.deep.mkString(\", \")}\")\n", + " //println(s\"input = ${sine.mkString(\", \")}\")\n", "\n", " // coefficients\n", " val coeffs = window(length, bitwidth)\n", - " //println(s\"coeffs = ${coeffs.toArray.deep.mkString(\", \")}\")\n", + " //println(s\"coeffs = ${coeffs.mkString(\", \")}\")\n", "\n", - " // use breeze filter as golden model; need to reverse coefficients\n", - " val expected = filter(\n", - " DenseVector(sine.toArray),\n", - " FIRKernel1D(DenseVector(coeffs.reverse.toArray), 1.0, \"\"),\n", - " OptOverhang.None\n", - " )\n", - " expected.toArray // seems to be necessary\n", - " //println(s\"exp_out = ${expected.toArray.deep.mkString(\", \")}\") // this seems to be necessary\n", + " // use the plain Scala convolution as golden model\n", + " val expected = goldenFir(sine, coeffs)\n", + " //println(s\"exp_out = ${expected.mkString(\", \")}\")\n", "\n", " // push data through our FIR and check the result\n", " c.reset.poke(true.B)\n", @@ -433,10 +428,8 @@ " for (i <- 0 until n) {\n", " c.io.in.poke(sine(i).U)\n", " if (i >= length-1) { // wait for all registers to be initialized since we didn't zero-pad the data\n", - " val expectValue = expected(i-length+1)\n", - " //println(s\"expected value is $expectValue\")\n", - " c.io.out.expect(expected(i-length+1).U)\n", - " //println(s\"cycle $i, got ${c.io.out.peek()}, expect ${expected(i-length+1)}\")\n", + " c.io.out.expect(expected(i).U)\n", + " //println(s\"cycle $i, got ${c.io.out.peek()}, expect ${expected(i)}\")\n", " }\n", " c.clock.step(1)\n", " }\n", @@ -456,7 +449,22 @@ "\n", "![Neuron](https://upload.wikimedia.org/wikipedia/commons/thumb/6/60/ArtificialNeuronModel_english.png/600px-ArtificialNeuronModel_english.png)\n", "\n", - "First, complete the following code to create a neuron generator. The parameter `inputs` gives the number of inputs. The parameter `act` is a function that implements the logic of the activation function. We'll make the inputs and outputs 16-bit fixed point values with 8 fractional bits." + "First, complete the following code to create a neuron generator. The parameter `inputs` gives the number of inputs. The parameter `act` is a function that implements the logic of the activation function. We'll make the inputs and outputs 16-bit fixed point values with 8 fractional bits.\n", + "\n", + "Since Chisel no longer has a fixed-point type, we represent a fixed-point number as an `SInt` that holds the real value multiplied by $2^8$. For example, 1.0 is stored as the integer 256 and -0.5 as -128. Adding two such numbers is just an `SInt` addition, but the product of two numbers with 8 fractional bits has 16 fractional bits, so after each multiplication we shift right by 8 (an arithmetic shift for `SInt`) to get back to 8 fractional bits. The helper below converts a Scala `Double` to this representation at elaboration or test time." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "// number of fractional bits of our fixed-point representation\n", + "val FracBits = 8\n", + "\n", + "// convert a Scala Double to a fixed-point SInt literal (the value multiplied by 2^FracBits)\n", + "def toFixed(x: Double): SInt = math.round(x * (1 << FracBits)).toInt.S" ] }, { @@ -465,11 +473,11 @@ "metadata": {}, "outputs": [], "source": [ - "class Neuron(inputs: Int, act: FixedPoint => FixedPoint) extends Module {\n", + "class Neuron(inputs: Int, act: SInt => SInt) extends Module {\n", " val io = IO(new Bundle {\n", - " val in = Input(Vec(inputs, FixedPoint(16.W, 8.BP)))\n", - " val weights = Input(Vec(inputs, FixedPoint(16.W, 8.BP)))\n", - " val out = Output(FixedPoint(16.W, 8.BP))\n", + " val in = Input(Vec(inputs, SInt(16.W))) // 8 fractional bits\n", + " val weights = Input(Vec(inputs, SInt(16.W))) // 8 fractional bits\n", + " val out = Output(SInt(16.W)) // 8 fractional bits\n", " })\n", " \n", " ???\n", @@ -485,7 +493,8 @@ "\n", "
\n", "
\n",
-    "  val mac = io.in.zip(io.weights).map{ case(a:FixedPoint, b:FixedPoint) => a*b}.reduce(_+_)\n",
+    "  // each product has 16 fractional bits, shift right to get back to 8 fractional bits\n",
+    "  val mac = io.in.zip(io.weights).map{ case(a, b) => (a*b) >> FracBits }.reduce(_ +& _)\n",
     "  io.out := act(mac)\n",
     "
" ] @@ -514,7 +523,7 @@ " x & \\text{if } x \\gt 0\n", " \\end{cases}$\n", "\n", - "Implement these two functions below. You can specify a fixed-point literal like `-3.14.F(8.BP)`. " + "Implement these two functions below. You can specify a fixed-point literal like `toFixed(-3.14)`. " ] }, { @@ -523,8 +532,8 @@ "metadata": {}, "outputs": [], "source": [ - "val Step: FixedPoint => FixedPoint = ???\n", - "val ReLU: FixedPoint => FixedPoint = ???" + "val Step: SInt => SInt = ???\n", + "val ReLU: SInt => SInt = ???" ] }, { @@ -536,8 +545,8 @@ "\n", "
\n", "
\n",
-    "val Step: FixedPoint => FixedPoint = x => Mux(x <= 0.F(8.BP), 0.F(8.BP), 1.F(8.BP))\n",
-    "val ReLU: FixedPoint => FixedPoint = x => Mux(x <= 0.F(8.BP), 0.F(8.BP), x)\n",
+    "val Step: SInt => SInt = x => Mux(x <= 0.S, toFixed(0.0), toFixed(1.0))\n",
+    "val ReLU: SInt => SInt = x => Mux(x <= 0.S, toFixed(0.0), x)\n",
     "
" ] }, @@ -547,7 +556,7 @@ "source": [ "Finally, let's create a tester that checks the correctness of our Neuron. With the step activation function, neurons may be used as logic gate approximators. Proper selection of weights and bias can perform binary functions. We'll test our neuron using AND logic. Complete the following tester to check our neuron with the step function.\n", "\n", - "Note that since the circuit is purely combinational, the `reset(5)` and `step(1)` calls are not necessary." + "Note that since the circuit is purely combinational, the reset and `step(1)` calls are not necessary." ] }, { @@ -557,7 +566,7 @@ "outputs": [], "source": [ "// test our Neuron \n", - "test(new Neuron(2, Step)) { c =>\n", + "simulate(new Neuron(2, Step)) { c =>\n", " val inputs = Seq(Seq(-1, -1), Seq(-1, 1), Seq(1, -1), Seq(1, 1))\n", "\n", " // make this a sequence of two values\n", @@ -568,11 +577,11 @@ " c.clock.step(5)\n", " c.reset.poke(false.B)\n", " for (i <- inputs) {\n", - " c.io.in(0).poke(i(0).F(8.BP))\n", - " c.io.in(1).poke(i(1).F(8.BP))\n", - " c.io.weights(0).poke(weights(0).F(16.W, 8.BP))\n", - " c.io.weights(1).poke(weights(1).F(16.W, 8.BP))\n", - " c.io.out.expect((if (i(0) + i(1) > 0) 1 else 0).F(16.W, 8.BP))\n", + " c.io.in(0).poke(toFixed(i(0)))\n", + " c.io.in(1).poke(toFixed(i(1)))\n", + " c.io.weights(0).poke(toFixed(weights(0)))\n", + " c.io.weights(1).poke(toFixed(weights(1)))\n", + " c.io.out.expect(toFixed(if (i(0) + i(1) > 0) 1.0 else 0.0))\n", " c.clock.step(1)\n", " }\n", " \n", @@ -615,7 +624,7 @@ "mimetype": "text/x-scala", "name": "scala", "nbconvert_exporter": "script", - "version": "2.12.10" + "version": "2.13.18" } }, "nbformat": 4, diff --git a/3.5_object_oriented_programming.ipynb b/3.5_object_oriented_programming.ipynb index ed6af8d..8b6bbd7 100644 --- a/3.5_object_oriented_programming.ipynb +++ b/3.5_object_oriented_programming.ipynb @@ -30,8 +30,7 @@ "metadata": {}, "outputs": [], "source": [ - "val path = System.getProperty(\"user.dir\") + \"/source/load-ivy.sc\"\n", - "interp.load.module(ammonite.ops.Path(java.nio.file.FileSystems.getDefault().getPath(path)))" + "interp.load.module(os.pwd / \"source\" / \"load-ivy.sc\")" ] }, { @@ -41,10 +40,9 @@ "outputs": [], "source": [ "import chisel3._\n", + "import chisel3.Input // avoid a name clash with almond.input.Input\n", "import chisel3.util._\n", - "import chisel3.experimental._\n", - "import chiseltest._\n", - "import chiseltest.RawTester.test" + "import chisel3.simulator.PeekPokeAPI._" ] }, { @@ -335,7 +333,7 @@ "# Inheritance with Chisel\n", "You've seen `Module`s and `Bundle`s before, but it's important to realize what's really going on.\n", "Every Chisel module you make is a class extending the base type `Module`.\n", - "Every Chisel IO you make is a class extending the base type `Bundle` (or, in some special cases, `Bundle`'s supertype [`Record`](https://github.com/freechipsproject/chisel3/blob/v3.0.0/chiselFrontend/src/main/scala/chisel3/core/Aggregate.scala#L415)).\n", + "Every Chisel IO you make is a class extending the base type `Bundle` (or, in some special cases, `Bundle`'s supertype [`Record`](https://github.com/chipsalliance/chisel/blob/v7.16.0/core/src/main/scala/chisel3/Aggregate.scala#L806)).\n", "Chisel hardware types like `UInt` or `Bundle` all have `Data` as a supertype.\n", "We'll explore using object oriented programming to create hierarchical hardware blocks and explore object reuse. You'll learn more about types and `Data` in the next Module on type generic generators.\n", "\n", @@ -397,7 +395,7 @@ "source": [ "// test our gray coder\n", "val bitwidth = 4\n", - "test(new GrayCoder(bitwidth)) { c =>\n", + "simulate(new GrayCoder(bitwidth)) { c =>\n", " def toBinary(i: Int, digits: Int = 8) = {\n", " String.format(\"%\" + digits + \"s\", i.toBinaryString).replace(' ', '0')\n", " }\n", @@ -456,7 +454,7 @@ " val read_counter = withClock(io.read_clock) { Counter(io.read_enable && !io.empty, depth*2)._1 }\n", " \n", " // encode\n", - " val encoder = new GrayCoder(write_counter.getWidth)\n", + " val encoder = Module(new GrayCoder(write_counter.getWidth))\n", " encoder.io.in := write_counter\n", " encoder.io.encode := true.B\n", " \n", @@ -464,7 +462,7 @@ " val sync = withClock(io.read_clock) { ShiftRegister(encoder.io.out, 2) }\n", " \n", " // decode\n", - " val decoder = new GrayCoder(read_counter.getWidth)\n", + " val decoder = Module(new GrayCoder(read_counter.getWidth))\n", " decoder.io.in := sync\n", " decoder.io.encode := false.B\n", " \n", @@ -496,7 +494,7 @@ "mimetype": "text/x-scala", "name": "scala", "nbconvert_exporter": "script", - "version": "2.12.10" + "version": "2.13.18" } }, "nbformat": 4, diff --git a/3.6_types.ipynb b/3.6_types.ipynb index f50b39d..96ddd7b 100644 --- a/3.6_types.ipynb +++ b/3.6_types.ipynb @@ -35,8 +35,7 @@ "metadata": {}, "outputs": [], "source": [ - "val path = System.getProperty(\"user.dir\") + \"/source/load-ivy.sc\"\n", - "interp.load.module(ammonite.ops.Path(java.nio.file.FileSystems.getDefault().getPath(path)))" + "interp.load.module(os.pwd / \"source\" / \"load-ivy.sc\")" ] }, { @@ -46,9 +45,9 @@ "outputs": [], "source": [ "import chisel3._\n", + "import chisel3.Input // avoid a name clash with almond.input.Input\n", "import chisel3.util._\n", - "import chiseltest._\n", - "import chiseltest.RawTester.test" + "import chisel3.simulator.PeekPokeAPI._" ] }, { @@ -231,6 +230,8 @@ "The most general is `asTypeOf()`, which is shown below.\n", "Some chisel objects also define `asUInt()` and `asSInt()` as well as some others.\n", "\n", + "Running it as is fails during elaboration with a `ChiselException` saying that the connection between the `SInt` sink and the `UInt` source failed because they have different types.\n", + "\n", "If you remove the `//` from the code block below, the example should work for you.\n" ] }, @@ -248,7 +249,7 @@ " io.out := io.in//.asTypeOf(io.out)\n", "}\n", "\n", - "test(new TypeConvertDemo) { c =>\n", + "simulate(new TypeConvertDemo) { c =>\n", " c.io.in.poke(3.U)\n", " c.io.out.expect(3.S)\n", " c.io.in.poke(15.U)\n", @@ -270,6 +271,8 @@ "The following example shows an example of a \"generator\" that can add two literals of type `UInt` or `SInt`.\n", "Later sections will talk more about writing type-generic generators.\n", "\n", + "The first two `println`s print the generated SystemVerilog; the third one intentionally fails with `java.lang.Exception: I give up!` because `UInt` and `SInt` cannot be mixed.\n", + "\n", "**Note: there are much better and safer ways to write type-generic generators in Scala**." ] }, @@ -301,7 +304,7 @@ "source": [ "It is good to remember that Chisel types generally should not be value matched.\n", "Scala's match executes during circuit elaboration, but what you probably want is a post-elaboration comparison.\n", - "The following gives a syntax error:" + "The following gives a syntax error (reported by the Scala parser, before any Chisel code runs):" ] }, { @@ -509,7 +512,9 @@ "\n", "For other types, Chisel will let you connect them, but may truncate/pad bits as appropriate.\n", "* Bool/UInt to Bool/UInt\n", - "* Bundle to Bundle" + "* Bundle to Bundle (only the common subelements are connected)\n", + "\n", + "Connecting a `Clock` to a `Bool` is rejected at elaboration time. Try uncommenting the first connection in the module below to see the error message." ] }, { @@ -539,10 +544,10 @@ " //io.out := io.c // won't work due to different types\n", "\n", " // Okay, but Chisel will truncate the input width to 1 to match the output.\n", - "// io.out := io.in\n", + " io.out := io.in\n", "\n", - "// // Compiles; Chisel will connect the common subelements of the two Bundles (in this case, 'a').\n", - "// io.bundleOut := io.bundleIn\n", + " // Compiles; Chisel will connect the common subelements of the two Bundles (in this case, 'a').\n", + " io.bundleOut := io.bundleIn\n", "}\n", "\n", "println(getVerilog(new BadTypeModule))" @@ -644,7 +649,7 @@ "`Data` can be used in IOs and supports `:=`, wires, regs, etc.\n", "\n", "Registers are a good example of polymorphic code in Chisel.\n", - "Look at the implementation of `RegEnable` (a register with a `Bool` enable signal) [here](https://github.com/freechipsproject/chisel3/blob/v3.0.0/src/main/scala/chisel3/util/Reg.scala#L10).\n", + "Look at the implementation of `RegEnable` (a register with a `Bool` enable signal) [here](https://github.com/chipsalliance/chisel/blob/v7.16.0/src/main/scala/chisel3/util/Reg.scala#L8).\n", "The apply function is templated for `[T <: Data]`, which means `RegEnable` will work for all Chisel hardware types.\n", "\n", "Some operations are only defined on subtypes of `Bits`, for example `+`.\n", @@ -681,7 +686,6 @@ " \n", " val in = Input(gen)\n", " val out = Output(Vec(n + 1, gen)) // + 1 because in is included in out\n", - " override def cloneType: this.type = (new ShiftRegisterIO(gen, n)).asInstanceOf[this.type]\n", "}\n", "\n", "class ShiftRegister[T <: Data](gen: T, n: Int) extends Module {\n", @@ -689,12 +693,12 @@ " \n", " io.out.foldLeft(io.in) { case (in, out) =>\n", " out := in\n", - " RegNext(in)\n", + " RegNext(in, 0.U.asTypeOf(gen)) // registers reset to zero\n", " }\n", "}\n", "\n", "visualize(() => new ShiftRegister(SInt(6.W), 3))\n", - "test(new ShiftRegister(SInt(6.W), 3)) { c => \n", + "simulate(new ShiftRegister(SInt(6.W), 3)) { c => \n", " println(s\"Testing ShiftRegister of type ${c.io.in} and depth ${c.io.out.length}\")\n", " for (i <- 0 until 10) {\n", " c.io.in.poke(i.S) // magic literal creation\n", @@ -708,6 +712,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "Note that the `Bundle` above does not need to override `cloneType` (as it had to in older versions of Chisel): the Chisel compiler plugin generates it for us.\n", + "\n", "We generally recommend avoiding to use inheritance with type generics.\n", "It can be very tricky to do properly and can get frustrating quickly." ] @@ -721,30 +727,71 @@ "\n", "The example above was limited to simple operations that could be performed on any instance of `Data` such as `:=` or `RegNext()`.\n", "When generating DSP circuits, we would like to do mathematical operations like addition and multiplication.\n", - "The `dsptools` library provides tools for writing type parameterized DSP generators.\n", + "Libraries such as `dsptools` (which is not used in this bootcamp) provide tools for writing type parameterized DSP generators; here we build a tiny version of the same idea ourselves.\n", "\n", - "Here is an example of writing a multiply-accumulate module.\n", - "It can be used to generate a multiply-accumulate (MAC) for `FixedPoint`, `SInt`, or even `DspComplex[T]` (the complex number type provided by `dsptools`).\n", - "The syntax of the type bound is a little different because `dsptools` uses typeclasses.\n", - "They are beyond the scope of this notebook.\n", - "Read the `dsptools` readme and documentation for more information on using typeclasses.\n", + "The key tool is a *typeclass*.\n", + "A typeclass is a trait (here `Ring[T]`) that describes operations on some type `T` without `T` having to inherit from it.\n", + "You then provide an *instance* of the typeclass for each type that supports the operations (here one for `UInt` and one for `SInt`).\n", + "The syntax of the type bound is a little different from the one above:\n", "\n", - "`T <: Data : Ring` means that `T` is a subtype of `Data` and is also a `Ring` .\n", - "`Ring` is defined in `dsptools` as a number with `+` and `*` (among other operations).\n", + "`T <: Data : Ring` means that `T` is a subtype of `Data` and that an implicit `Ring[T]` instance exists.\n", + "Our `Ring` is a number with `0`, `1`, `+`, `-` and `*` (see [this Wikipedia article](https://en.wikipedia.org/wiki/Ring_(mathematics)) for details about rings).\n", + "The `RingOps` implicit class adds the operators `+`, `-` and `*` to every type that has a `Ring`, so generic code can use the usual syntax.\n", "\n", - "_An alternative to `Ring` would be `Real`, but that would not allow us to make a MAC for `DspComplex()` because complex numbers are not `Real`._\n", - "\n" + "Here is the typeclass definition, followed by an example of using it to write a multiply-accumulate module.\n", + "It can be used to generate a multiply-accumulate (MAC) for `UInt`, `SInt`, or anything else for which we write a `Ring` instance (we will do so for complex numbers below)." ] }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "// A typeclass describing \"number-like\" hardware types: they have 0, 1, +, - and *.\n", + "trait Ring[T <: Data] {\n", + " def zero: T\n", + " def one: T\n", + " def plus(a: T, b: T): T\n", + " def minus(a: T, b: T): T\n", + " def times(a: T, b: T): T\n", + "}\n", + "\n", + "object Ring {\n", + " // summon the Ring instance of T, so we can write Ring[T].zero\n", + " def apply[T <: Data](implicit ring: Ring[T]): Ring[T] = ring\n", + "\n", + " // instances for the built-in types; living in the companion object, the compiler finds them automatically\n", + " implicit object UIntRing extends Ring[UInt] {\n", + " def zero: UInt = 0.U\n", + " def one: UInt = 1.U\n", + " def plus(a: UInt, b: UInt): UInt = a + b\n", + " def minus(a: UInt, b: UInt): UInt = a - b\n", + " def times(a: UInt, b: UInt): UInt = a * b\n", + " }\n", + " implicit object SIntRing extends Ring[SInt] {\n", + " def zero: SInt = 0.S\n", + " def one: SInt = 1.S\n", + " def plus(a: SInt, b: SInt): SInt = a + b\n", + " def minus(a: SInt, b: SInt): SInt = a - b\n", + " def times(a: SInt, b: SInt): SInt = a * b\n", + " }\n", + "}\n", + "\n", + "// Lets us write a + b, a - b and a * b for any T that has a Ring instance\n", + "implicit class RingOps[T <: Data](lhs: T)(implicit ring: Ring[T]) {\n", + " def +(rhs: T): T = ring.plus(lhs, rhs)\n", + " def -(rhs: T): T = ring.minus(lhs, rhs)\n", + " def *(rhs: T): T = ring.times(lhs, rhs)\n", + "}" + ], + "execution_count": null, + "outputs": [] + }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "import chisel3.experimental._\n", - "import dsptools.numbers._\n", - "\n", "class Mac[T <: Data : Ring](genIn : T, genOut: T) extends Module {\n", " val io = IO(new Bundle {\n", " val a = Input(genIn)\n", @@ -756,8 +803,7 @@ "}\n", "\n", "println(getVerilog(new Mac(UInt(4.W), UInt(6.W)) ))\n", - "println(getVerilog(new Mac(SInt(4.W), SInt(6.W)) ))\n", - "println(getVerilog(new Mac(FixedPoint(4.W, 3.BP), FixedPoint(6.W, 4.BP))))\n" + "println(getVerilog(new Mac(SInt(4.W), SInt(6.W)) ))" ] }, { @@ -791,16 +837,13 @@ "class MacTestModule extends Module {\n", " val io = IO(new Bundle {\n", " val uin = Input(UInt(4.W))\n", - " val uout = Output(UInt())\n", + " val uout = Output(UInt(16.W))\n", " val sin = Input(SInt(4.W))\n", - " val sout = Output(SInt())\n", - " //val fin = Input(FixedPoint(16.W, 12.BP))\n", - " //val fout = Output(FixedPoint())\n", + " val sout = Output(SInt(16.W))\n", " })\n", " // for each IO pair, do out = in * in + in\n", " io.uout := Mac(io.uin, io.uin, io.uin)\n", " io.sout := Mac(io.sin, io.sin, io.sin)\n", - " //io.fout := Mac(io.fin, io.fin, io.fin)\n", "}\n", "println(getVerilog(new MacTestModule))" ] @@ -849,7 +892,7 @@ " ??? // your code\n", "}\n", "\n", - "test(new Integrator(SInt(4.W), SInt(8.W))) { c =>\n", + "simulate(new Integrator(SInt(4.W), SInt(8.W))) { c =>\n", " c.io.in.poke(3.S)\n", " c.io.out.expect(0.S)\n", " c.clock.step(1)\n", @@ -876,8 +919,8 @@ "\n", "class Integrator\\[T <: Data : Ring\\](genIn: T, genReg: T) extends Module {\n", " val io = IO(new Bundle {\n", - " val in = Input(genIn.cloneType)\n", - " val out = Output(genReg.cloneType)\n", + " val in = Input(genIn)\n", + " val out = Output(genReg)\n", " })\n", " \n", " val reg = RegInit(genReg, Ring[T].zero) // init to zero\n", @@ -906,24 +949,24 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "**Example: DspComplex**
\n", - "`DspComplex` is a custom data type defined in **dsptools** [here](https://github.com/ucb-bar/dsptools/blob/v1.0.0/src/main/scala/dsptools/numbers/chisel_concrete/DspComplex.scala#L59).\n", + "**Example: Complex Numbers**
\n", + "Complex numbers are a classic example of a custom data type.\n", + "The `dsptools` library has a `DspComplex` type; we write a minimal version of our own.\n", "The key line to understand is this:\n", "```scala\n", - "class DspComplex[T <: Data:Ring](val real: T, val imag: T) extends Bundle { ... }\n", + "class Complex[T <: Data : Ring](val real: T, val imag: T) extends Bundle\n", "```\n", "\n", - "`DspComplex` is a type-generic container.\n", + "`Complex` is a type-generic container.\n", "That means the real and imaginary parts of a complex number can be any type as long as they satisfy the type constraints, given by `T <: Data : Ring`.\n", "\n", "`T <: Data` means `T` is a subtype of `chisel3.Data`, the base type for Chisel objects.\n", - "This means that `DspComplex` only works for objects that are Chisel types and not arbitrary Scala types.\n", + "This means that `Complex` only works for objects that are Chisel types and not arbitrary Scala types.\n", "\n", - "`T : Ring` means that a Ring typeclass implementation for `T` exists.\n", - "`Ring` typeclasses define `+` and `*` operators as well as additive and multiplicative identities (see [this Wikipedia article](https://en.wikipedia.org/wiki/Ring_(mathematics)) for details about rings).\n", - "**dsptools** defines typeclasses for commonly used Chisel types [here](https://github.com/ucb-bar/dsptools/tree/v1.0.0/src/main/scala/dsptools/numbers/chisel_types).\n", + "`T : Ring` means that a `Ring` typeclass implementation for `T` exists.\n", + "`Ring` typeclasses define `+` and `*` operators as well as additive and multiplicative identities.\n", "\n", - "**dsptools** also defines a `Ring` typeclass for `DspComplex`, so we can reuse our MAC generator with complex numbers:" + "Below we also define a `Ring` typeclass instance for `Complex[T]` (using the rule $(a+bi)(c+di) = (ac-bd) + (ad+bc)i$), so we can reuse our MAC generator with complex numbers:" ] }, { @@ -932,7 +975,27 @@ "metadata": {}, "outputs": [], "source": [ - "println(getVerilog(new Mac(DspComplex(SInt(4.W), SInt(4.W)), DspComplex(SInt(6.W), SInt(6.W))) ))" + "// A complex number whose real and imaginary parts can be any type T\n", + "class Complex[T <: Data : Ring](val real: T, val imag: T) extends Bundle\n", + "\n", + "// Complex numbers form a Ring as well, as long as their parts do\n", + "implicit def complexRing[T <: Data : Ring]: Ring[Complex[T]] = new Ring[Complex[T]] {\n", + " private def make(re: T, im: T): Complex[T] = {\n", + " val result = Wire(new Complex(chiselTypeOf(re), chiselTypeOf(im)))\n", + " result.real := re\n", + " result.imag := im\n", + " result\n", + " }\n", + " def zero: Complex[T] = make(Ring[T].zero, Ring[T].zero)\n", + " def one: Complex[T] = make(Ring[T].one, Ring[T].zero)\n", + " def plus(a: Complex[T], b: Complex[T]): Complex[T] = make(a.real + b.real, a.imag + b.imag)\n", + " def minus(a: Complex[T], b: Complex[T]): Complex[T] = make(a.real - b.real, a.imag - b.imag)\n", + " // (a + bi)(c + di) = (ac - bd) + (ad + bc)i\n", + " def times(a: Complex[T], b: Complex[T]): Complex[T] =\n", + " make(a.real * b.real - a.imag * b.imag, a.real * b.imag + a.imag * b.real)\n", + "}\n", + "\n", + "println(getVerilog(new Mac(new Complex(SInt(4.W), SInt(4.W)), new Complex(SInt(6.W), SInt(6.W)))))" ] }, { @@ -978,10 +1041,13 @@ " // Implement this!\n", " }\n", "}\n", - "trait SignMagnitudeRing extends Ring[SignMagnitude] {\n", + "implicit object SignMagnitudeRing extends Ring[SignMagnitude] {\n", " def plus(f: SignMagnitude, g: SignMagnitude): SignMagnitude = {\n", " f + g\n", " }\n", + " def minus(f: SignMagnitude, g: SignMagnitude): SignMagnitude = {\n", + " f - g\n", + " }\n", " def times(f: SignMagnitude, g: SignMagnitude): SignMagnitude = {\n", " f * g\n", " }\n", @@ -992,22 +1058,12 @@ " one\n", " }\n", " def zero: SignMagnitude = {\n", - " val zero = Wire(new SignMagnitude(Some(0)))\n", + " val zero = Wire(new SignMagnitude(Some(1)))\n", " zero.sign := false.B\n", " zero.magnitude := 0.U\n", " zero\n", " }\n", - " def negate(f: SignMagnitude): SignMagnitude = {\n", - " -f\n", - " }\n", - " \n", - " // Leave unimplemented for this example\n", - " def minusContext(f: SignMagnitude, g: SignMagnitude): SignMagnitude = ???\n", - " def negateContext(f: SignMagnitude): SignMagnitude = ???\n", - " def plusContext(f: SignMagnitude,g: SignMagnitude): SignMagnitude = ???\n", - " def timesContext(f: SignMagnitude,g: SignMagnitude): SignMagnitude = ???\n", - "}\n", - "implicit object SignMagnitudeRingImpl extends SignMagnitudeRing" + "}" ] }, { @@ -1018,7 +1074,7 @@ "source": [ "import chisel3.experimental.BundleLiterals._\n", "\n", - "test(new Mac(new SignMagnitude(Some(4)), new SignMagnitude(Some(5)))) { c =>\n", + "simulate(new Mac(new SignMagnitude(Some(4)), new SignMagnitude(Some(5)))) { c =>\n", " c.io.a.poke(chiselTypeOf(c.io.a).Lit(_.sign -> false.B, _.magnitude -> 3.U))\n", " c.io.b.poke(chiselTypeOf(c.io.b).Lit(_.sign -> false.B, _.magnitude -> 3.U))\n", " c.io.c.poke(chiselTypeOf(c.io.c).Lit(_.sign -> false.B, _.magnitude -> 2.U))\n", @@ -1038,7 +1094,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Look at the verilog to see if the output looks reasonable:" + "Look at the generated SystemVerilog to see if the output looks reasonable:" ] }, { @@ -1055,7 +1111,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "`SignMagnitude` even works with `DspComplex`!" + "`SignMagnitude` even works with our `Complex` type!" ] }, { @@ -1064,7 +1120,7 @@ "metadata": {}, "outputs": [], "source": [ - "println(getVerilog(new Mac(DspComplex(new SignMagnitude(Some(4)), new SignMagnitude(Some(4))), DspComplex(new SignMagnitude(Some(5)), new SignMagnitude(Some(5))))))" + "println(getVerilog(new Mac(new Complex(new SignMagnitude(Some(4)), new SignMagnitude(Some(4))), new Complex(new SignMagnitude(Some(5)), new SignMagnitude(Some(5))))))" ] }, { @@ -1127,7 +1183,7 @@ "mimetype": "text/x-scala", "name": "scala", "nbconvert_exporter": "script", - "version": "2.12.10" + "version": "2.13.18" } }, "nbformat": 4, diff --git a/4.1_firrtl_ast.ipynb b/4.1_firrtl_ast.ipynb index cfe21e8..112e4c0 100644 --- a/4.1_firrtl_ast.ipynb +++ b/4.1_firrtl_ast.ipynb @@ -1,6 +1,7 @@ { "cells": [ { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -8,20 +9,22 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "# Module 4.1: Introduction to FIRRTL\n", - "\n", "**Prev: [Generators: Types](3.6_types.ipynb)**
\n", "**Next: [FIRRTL AST Traversal](4.2_firrtl_ast_traversal.ipynb)**\n", "\n", "## Motivation\n", "You've learned some Scala and written some Chisel, and for 90% of users, that should be enough to become a Chisel aficionado.\n", "\n", - "However, some use cases are better expressed as a programmatic transformation of a Chisel design, rather than as a generator.\n", + "However, some use cases are better expressed as a programmatic analysis or transformation of a finished Chisel design, rather than as a generator.\n", + "\n", + "For example, suppose we want to count the number of registers in a design, or the number of adders in every module of a hierarchy. This would be difficult to do as a generator, because a generator only sees the Scala objects it is building at that moment. Instead, we can look at the *elaborated* design, which is what the FIRRTL language describes.\n", "\n", - "For example, suppose we want to count the number of registers in a design. This would be difficult to do as a generator, so instead, we can write a FIRRTL pass to do it for us.\n", + "In this module we will learn to read FIRRTL, and then write small Scala programs that parse, analyze and rewrite it.\n", "\n", "## Setup" ] @@ -29,39 +32,58 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ - "val path = System.getProperty(\"user.dir\") + \"/source/load-ivy.sc\"\n", - "interp.load.module(ammonite.ops.Path(java.nio.file.FileSystems.getDefault().getPath(path)))" + "interp.load.module(os.pwd / \"source\" / \"load-ivy.sc\")" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "import chisel3._\n", + "import chisel3.Input // avoid a name clash with almond.input.Input\n", "import chisel3.util._\n", - "import chisel3.iotesters.{ChiselFlatSpec, Driver, PeekPokeTester}\n", - "import firrtl._" + "import chisel3.reflect.DataMirror" ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "## What is FIRRTL?\n", "As you've probably become aware, when you execute a Chisel design, it elaborates (executes the surrounding Scala code) to construct an instance of your generator, with all Scala parameters resolved.\n", "\n", - "Instead of directly emitting Verilog, Chisel emits an intermediate representation called FIRRTL, which represents the elaborated (parameter-resolved) RTL instance. It can be serialized (converted to a String for writing to a file), and this serialized syntax is human readable. Internally, however, it is not represented as a long string. Instead, it is a datastructure organized as a tree of nodes, called an abstract-syntax-tree (AST).\n", + "Chisel does not directly emit Verilog. Instead, it emits an intermediate representation called **FIRRTL** (Flexible Intermediate Representation for RTL), which represents the elaborated (parameter-resolved) RTL instance. The text that Chisel itself produces is sometimes called **CHIRRTL** (\"Chisel FIRRTL\"), because it still leaves some work, like width inference, to the compiler.\n", + "\n", + "The compilation flow of Chisel 7 looks like this:\n", + "\n", + "```\n", + " Chisel (Scala)\n", + " | elaboration (running your generator)\n", + " v\n", + " CHIRRTL text <- human readable; what this notebook is about\n", + " | ----------------------------------------- firtool (CIRCT) ----\n", + " | parse, infer widths, check, optimize\n", + " v\n", + " FIRRTL dialect (MLIR)\n", + " | lower types, remove `when`, ...\n", + " v\n", + " HW dialect (MLIR)\n", + " | emit\n", + " v\n", + " SystemVerilog\n", + "```\n", + "\n", + "Everything below the CHIRRTL line is done by [CIRCT](https://circt.llvm.org)'s `firtool`, a native program that Chisel downloads and runs for you. In older versions of Chisel (3.x and before), the same job was done by a FIRRTL compiler written in Scala, and a whole chapter of this bootcamp was spent on writing Scala \"transforms\" for it. That Scala library no longer exists: the FIRRTL *language* is the contract between Chisel and the compiler, and any tool that can read the text can work with it.\n", "\n", - "Let's take a look! We will take a simple Chisel design, elaborate it, and inspect what FIRRTL it generates!\n", + "That is also good news for us. FIRRTL text is simple, line-oriented and well specified, so with a few dozen lines of Scala we can build our own data structure (an abstract-syntax-tree, or AST) from it and analyze it. This is what Modules 4.2 to 4.4 do.\n", + "\n", + "Let's take a look! We will take a simple Chisel design, elaborate it, and inspect what FIRRTL it generates.\n", "\n", "First, we define a Chisel module, which delays its input signal by two cycles." ] @@ -69,9 +91,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "class DelayBy2(width: Int) extends Module {\n", @@ -86,242 +106,324 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "Next, let's elaborate it, serialize, and print out the FIRRTL it generates." + "Next, let's elaborate it and print the FIRRTL it generates. `getFirrtl` is a helper of this bootcamp (see `source/load-ivy.sc`); it is a thin wrapper around `circt.stage.ChiselStage.emitCHIRRTL`." ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ - "println((new chisel3.stage.ChiselStage).emitChirrtl(new DelayBy2(4)))\n" + "println(getFirrtl(new DelayBy2(4)))" ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "As you can see, the serialized FIRRTL looks very similar to what our Chisel design would look like, with all generator parameters resolved." + "As you can see, the serialized FIRRTL looks very similar to what our Chisel design looks like, with all generator parameters resolved. Let's go through it from top to bottom:\n", + "\n", + "* `FIRRTL version 7.0.0` is the version of the language specification.\n", + "* `circuit DelayBy2 :` starts the circuit, whose name is its top module. Everything after the `%[[` up to the matching `]]` is a JSON list of *annotations*: extra information for the compiler that is not part of the circuit itself. Here, Chisel only says which modules are candidates for de-duplication (`DedupGroupAnnotation`). Libraries and user code can add their own.\n", + "* The `layer` declarations (`Verification`, `Assert`, ...) declare optional pieces of hardware, which can be switched on or off at build time. Chisel always declares the standard ones for verification code such as `printf`, `assert`, `assume` and `cover`. They are not relevant for our design.\n", + "* `public module DelayBy2 :` is a module definition. Only the top module is `public`: it is the one visible from the outside.\n", + "* `input clock : Clock` and `input reset : UInt<1>` are the implicit clock and reset ports that every Chisel `Module` has. Note that a Chisel `Bundle` is *not* flattened, it is a single port with an aggregate type, `output io : { flip in : UInt<4>, out : UInt<4>}`. A `flip` field has the opposite direction of the port, so `io.in` is an input.\n", + "* `reg r0 : UInt, clock` declares a register named after the Scala `val` that holds it. Its type is just `UInt`: the width is unknown, to be inferred by the compiler (from `io.in` in this case). A register with a reset value would be declared as `regreset`.\n", + "* `connect r0, io.in` assigns a source to a sink. It is the only kind of assignment in FIRRTL, and, as in Chisel, a later connect overrides an earlier one (last-connect semantics).\n", + "* `@[cmd3.sc 6:19]` at the end of a line is a *source locator*: it records the Scala file, line and column that created this line of FIRRTL. This is how tools can report errors in terms of your Chisel code. (Here the file is the notebook cell that was executed.)\n", + "\n", + "### Following the pipeline\n", + "We can ask Chisel to stop the pipeline at the later stages as well. `ChiselStage` has an `emit...` function for each of them. Note that you have to write `_root_.circt` because `chisel3.util` contains another package named `circt`, which our `import chisel3.util._` makes visible.\n", + "\n", + "First, the FIRRTL dialect. This is the CHIRRTL after `firtool` has parsed it, checked it and inferred all widths. It is written in MLIR syntax (we only print the interesting module, not the layer declarations):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "val firrtlDialect = _root_.circt.stage.ChiselStage.emitFIRRTLDialect(new DelayBy2(4))\n", + "println(firrtlDialect.linesIterator.dropWhile(!_.contains(\"firrtl.module\")).takeWhile(!_.startsWith(\" }\")).mkString(\"\\n\"))" ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "## The FIRRTL AST\n", + "Notice that the widths are known now (`!firrtl.uint<4>` for `r0`), that the bundle is now a list of scalar ports `%io_in` and `%io_out`, and that the output port is driven through a helper wire.\n", "\n", - "As mentioned earlier, the FIRRTL representation can be serialized as a String, but internally, it is a datastructure called an AST (abstract syntax tree). This data structure is a tree of nodes, where one node can contain children nodes. There are no cycles in this datastructure.\n", - "\n", - "Let's take a look at what the internal datastructure looks like:" + "Next is the HW dialect, where the design is described with hardware primitives (`seq.firreg` is a register) instead of FIRRTL's types:" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ - "val firrtlSerialization = (new chisel3.stage.ChiselStage).emitChirrtl(new DelayBy2(4))\n", - "val firrtlAST = firrtl.Parser.parse(firrtlSerialization.split(\"\\n\").toIterator, Parser.GenInfo(\"file.fir\"))\n", - "\n", - "println(firrtlAST)" + "val hwDialect = _root_.circt.stage.ChiselStage.emitHWDialect(new DelayBy2(4))\n", + "println(hwDialect.linesIterator.dropWhile(!_.contains(\"hw.module\")).takeWhile(!_.startsWith(\" }\")).mkString(\"\\n\"))" ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "Obviously, the serialization of a datastructure isn't as pretty, but you can see some of the classes and such that internally represent the RTL design. Let's try to pretty that up a bit to make it understandable." + "Finally, SystemVerilog, which is what `getVerilog` returns. `emitSystemVerilog` also accepts options for `firtool`, which is how you control the generated code. For example, the options below remove the debug information and the random initialization of registers (which firtool normally emits for simulation). `firtool --help` lists all of them. The output also contains some extra files for the verification layers, which we cut off:" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ - "println(stringifyAST(firrtlAST))" + "val sv = _root_.circt.stage.ChiselStage.emitSystemVerilog(\n", + " new DelayBy2(4),\n", + " firtoolOpts = Array(\"--disable-all-randomization\", \"--strip-debug-info\")\n", + ")\n", + "println(sv.split(\"// ----- 8<\")(0))" ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "This is the internal datastructure that holds the FIRRTL AST. It is a tree structure whose root node is **Circuit**, which has 3 children: **@[file.fir@2.0]**, **ArrayBuffer**, and **cmd5WrapperHelperDelayBy2**. The following is the definition of `Circuit`'s actual Scala class that was serialized:\"Circuit\n", - "\n" + "### Hierarchy\n", + "Modules that instantiate other modules show up as `inst` statements. Let's define a design with a module `Top` that instantiates `AddMe` twice, with different parameters:" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "metadata": {}, + "outputs": [], "source": [ - "As you can see, it has three children nodes: `info: Info`, `Modules: Seq[DefModule]`, and `main: String`. It extends `FirrtlNode`, of which all FIRRTL AST nodes must do. Ignore the `def mapXXXX` functions for now.\n", - "\n", - "Many FIRRTL nodes contain an `info: Info` field, which the parser can either insert file information like line number and column number, or insert a `NoInfo` token. In this example, **@[file.fir@2.0]** would refer to the FIRRTL file, line 2, column 0.\n", + "class AddMe(val nInputs: Int, val width: Int) extends Module {\n", + " val io = IO(new Bundle {\n", + " val in = Input(Vec(nInputs, UInt(width.W)))\n", + " val out = Output(UInt(width.W))\n", + " })\n", + " io.out := io.in.reduce(_ +& _)\n", + "}\n", "\n", - "The following section will outline all of these FIRRTL nodes in detail." + "class Top extends Module {\n", + " val io = IO(new Bundle {\n", + " val a = Input(UInt(4.W))\n", + " val b = Input(UInt(4.W))\n", + " val y = Output(UInt(5.W))\n", + " val z = Output(UInt(5.W))\n", + " })\n", + " val m0 = Module(new AddMe(2, 4))\n", + " val m1 = Module(new AddMe(3, 4))\n", + " m0.io.in(0) := io.a\n", + " m0.io.in(1) := io.b\n", + " m1.io.in(0) := io.a\n", + " m1.io.in(1) := io.b\n", + " m1.io.in(2) := m0.io.out\n", + " io.y := m0.io.out\n", + " io.z := m1.io.out\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "// Skip the annotations and layers, and show only the modules\n", + "println(getFirrtl(new Top).linesIterator.dropWhile(!_.contains(\"module\")).mkString(\"\\n\"))" ] }, { + "attachments": {}, "cell_type": "markdown", - "metadata": { - "collapsed": true - }, + "metadata": {}, "source": [ - "# FIRRTL Node Descriptions\n", - "\n", - "This section describes common FirrtlNodes found in [firrtl/src/main/scala/firrtl/ir/IR.scala](https://github.com/ucb-bar/firrtl/blob/master/src/main/scala/firrtl/ir/IR.scala).\n", + "A few things to observe:\n", + "* A circuit is a flat list of modules, and *modules are never nested*. `Top` refers to the other two by name with `inst m0 of AddMe`.\n", + "* The two instances have different parameters, so they are different modules, which Chisel names `AddMe` and `AddMe_1`. Two instances with identical parameters would share one module definition.\n", + "* Clock and reset are connected explicitly (`connect m0.clock, clock`), as is everything else. FIRRTL has no implicit wiring.\n", + "* Ports of an instance are accessed like fields of a bundle: `m0.io.in[0]`, `m0.io.out`.\n", + "* The module `Top` is last, and it is the only `public` one.\n", + "\n", + "### Looking at the design from Chisel\n", + "The FIRRTL text is not the only way to look at a design. While Chisel is elaborating, `chisel3.reflect.DataMirror` can tell you about the `Data` objects you hold: whether they are ports, wires or registers, their direction and their width. We use it below to list the ports of a submodule from within its parent. Note that this information is available only *after* a module has been closed (`Module(new AddMe(...))` has returned), because Chisel only then knows all of its ports." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class Parent extends Module {\n", + " val io = IO(new Bundle { val in = Input(UInt(4.W)); val out = Output(UInt(5.W)) })\n", + " val r = RegInit(0.U(3.W))\n", + " val w = Wire(UInt(3.W))\n", + " w := r\n", + " println(s\"r: isReg=${DataMirror.isReg(r)} isWire=${DataMirror.isWire(r)}\")\n", + " println(s\"w: isReg=${DataMirror.isReg(w)} isWire=${DataMirror.isWire(w)}\")\n", + "\n", + " val child = Module(new AddMe(2, 4))\n", + " for ((name, port) <- DataMirror.fullModulePorts(child)) {\n", + " println(f\"child port $name%-8s ${DataMirror.directionOf(port)}%-14s width=${port.widthOption}\")\n", + " }\n", + " child.io.in(0) := io.in\n", + " child.io.in(1) := io.in\n", + " io.out := child.io.out\n", + "}\n", + "val parentFirrtl = getFirrtl(new Parent)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`DataMirror` is handy inside generators, but it cannot see everything: for example, it cannot enumerate the registers or the operators of a module, and it works only while elaborating. The FIRRTL text, on the other hand, contains the *whole* design, so we will focus on it in the rest of this module.\n", "\n", - "For more detail on components not mentioned here, please refer to [The FIRRTL Specification](https://github.com/ucb-bar/firrtl/blob/master/spec/spec.pdf).\n", + "# FIRRTL Language Overview\n", "\n", + "This section describes the most common elements of the FIRRTL language. For more detail on components not mentioned here, please refer to [The FIRRTL Specification](https://github.com/chipsalliance/firrtl-spec).\n", "\n", "## Circuit\n", - "Circuit is the root node of any Firrtl datastructure. There is only ever one Circuit, and that Circuit contains a list of module definitions and the name of the top-level module.\n", - "\n", - "#### FirrtlNode Declaration\n", - "```scala \n", - "Circuit(info: Info, modules: Seq[DefModule], main: String)\n", + "The circuit is the root of any FIRRTL file. There is only ever one circuit, and it contains a list of module definitions and the name of the top-level module.\n", "```\n", - "\n", - "#### Concrete Syntax\n", - "```\n", - "circuit Adder:\n", - " ... //List of modules\n", - "```\n", - "#### In-memory Representation\n", - "```scala\n", - "Circuit(NoInfo, Seq(...), \"Adder\")\n", + "circuit Adder :\n", + " ... // list of modules\n", "```\n", "\n", "## Module\n", - "\n", - "Modules are the unit of modularity within Firrtl and are never directly nested (declaring an instance of a module has its own concrete syntax and AST representation). Each Module has a name, and a list of ports, and a body containing its implementation.\n", - "\n", - "#### FirrtlNode declaration\n", - "```scala\n", - "Module(info: Info, name: String, ports: Seq[Port], body: Stmt) extends DefModule\n", - "```\n", - "\n", - "#### Concrete Syntax\n", + "Modules are the unit of modularity within FIRRTL and are never directly nested (declaring an instance of a module is a statement). Each module has a name, a list of ports, and a body containing its implementation.\n", "```\n", - "module Adder:\n", + "module Adder :\n", " ... // list of ports\n", " ... // statements\n", "```\n", - "#### In-memory representation\n", - "```scala\n", - "Module(NoInfo, \"Adder\", Seq(...), )\n", - "```\n", "\n", "## Port\n", - "A port defines part of a Module's io, and has a name, direction (input or output), and type.\n", - "\n", - "#### FirrtlNode Declaration\n", - "```scala\n", - "class Port(info: Info, name: String, direction: Direction, tpe: Type)\n", - "```\n", - "#### Concrete Syntax\n", - "```\n", - "input x: UInt\n", - "```\n", - "\n", - "#### In-memory representation\n", - "```scala\n", - "Port(NoInfo, \"x\", INPUT, UIntType(UnknownWidth))\n", - "```\n", - "\n", - "## Statement\n", - "A statement is used to describe the components within a module and how they interact. Below are some commonly used statements:\n", - "\n", - "### Block of Statements\n", - "A group of statements. Commonly used as the body field in a Module declaration.\n", - "\n", - "### Wire Declaration\n", - "A wire declaration, containing a name and type. It can be both a source (connected *from*) and a sink (connected *to\").\n", - "#### FirrtlNode declaration\n", - "```scala\n", - "DefWire(info: Info, name: String, tpe: Type)\n", - "```\n", - "#### Concrete syntax\n", + "A port defines part of a module's interface, and has a name, a direction (`input` or `output`), and a type. Types are:\n", "```\n", - "wire w: UInt\n", - "```\n", - "#### In-memory Representation\n", - "```scala\n", - "DefWire(NoInfo, \"w\", UIntType(UnknownWidth))\n", + "input x : UInt<8> // unsigned integer of 8 bits; `UInt` alone means \"infer the width\"\n", + "input s : SInt<8> // signed integer\n", + "input c : Clock // clock, `Reset` and `AsyncReset` are the reset types\n", + "output v : UInt<4>[2] // vector (array) of 2 elements\n", + "output io : { flip in : UInt<4>, out : UInt<4> } // bundle (struct); `flip` reverses the direction of a field\n", "```\n", "\n", - "### Register Declaration\n", - "A register declaration, containing a name, type, clock signal, reset signal, and reset value.\n", - "#### FirrtlNode declaration\n", - "```scala\n", - "DefRegister(info: Info, name: String, tpe: Type, clock: Expression, reset: Expression, init: Expression)\n", + "## Statements\n", + "A statement describes the components within a module and how they interact. The most common ones:\n", "```\n", - "\n", - "### Connection\n", - "Represents a directioned connection from a source to a sink. Note that it abides by last-connect-semantics, as described in Chisel.\n", - "\n", - "#### FirrtlNode declaration\n", - "```scala\n", - "Connect(info: Info, loc: Expression, expr: Expression)\n", + "wire w : UInt<4> // wire, can be both a source and a sink\n", + "reg r : UInt<4>, clock // register without reset\n", + "regreset r : UInt<4>, clock, reset, UInt<4>(0h0) // register with synchronous reset, and its reset value\n", + "node n = add(x, y) // named expression (immutable)\n", + "inst m0 of Adder // instance of another module\n", + "connect w, n // connection, with last-connect semantics\n", + "when cond : // conditional; may be followed by `else :`\n", + " ...\n", + "invalidate w // \"don't care\" value\n", + "printf(...), assert(...) // verification statements\n", + "cmem / smem // memories\n", "```\n", + "Every `when` block is indented. Chisel's `when`/`.elsewhen`/`.otherwise` become nested `when`/`else` blocks.\n", "\n", - "### Other Statements\n", - "Other statement types like `DefMemory`, `DefNode`, `IsInvalid`, `Conditionally`, and others are omitted here; please refer to [firrtl/src/main/scala/firrtl/ir/IR.scala](https://github.com/freechipsproject/firrtl/blob/master/src/main/scala/firrtl/ir/IR.scala) for more detail.\n", - "\n", - "## Expression\n", - "Expressions represent references to declared components or logical and arithmetic operations. Below are some commonly used expressions:\n", - "\n", - "### Reference\n", - "A reference to a declared component, such as a wire, register, or port. It has a name and type field. Note that it does not contain a pointer to the actual declaration, but instead just contains the name as a String.\n", - "\n", - "#### FirrtlNode declaration\n", - "```scala\n", - "Reference(name: String, tpe: Type)\n", + "## Expressions\n", + "Expressions represent references to declared components or logical and arithmetic operations. The most common ones:\n", "```\n", - "\n", - "### DoPrim\n", - "An anonymous primitive operation, such as `Add`, `Sub`, or `And`, `Or`, or subword-selection (`Bits`). The type of operation is indicated by the `op: PrimOp` field. Note that the number of required arguments and constants are determined by the `op`.\n", - "\n", - "#### FirrtlNode declaration\n", - "```scala\n", - "DoPrim(op: PrimOp, args: Seq[Expression], consts: Seq[BigInt], tpe: Type)\n", + "UInt<4>(0h5) // literal: type, then value in hex (0h5), or decimal\n", + "x // reference to a port, wire, register, node or instance\n", + "io.in // sub-field of a bundle, also `m0.io.out` for instance ports\n", + "v[1] // sub-index of a vector\n", + "mux(sel, a, b) // multiplexer\n", + "add(a, b) // primitive operation (primop)\n", + "bits(a, 3, 0) // primops can take integer constants after their arguments\n", "```\n", + "There are only a few dozen primitive operations. Among them: `add`, `sub`, `mul`, `div`, `rem`, `lt`, `leq`, `gt`, `geq`, `eq`, `neq`, `and`, `or`, `xor`, `not`, `shl`, `shr`, `cat`, `bits`, `head`, `tail`, `pad`, `asUInt`, `asSInt`, `andr`, `orr`, `xorr`. Note that FIRRTL's `add` has a result that is one bit wider than its operands, just like Chisel's `+&`. A Chisel `+` is a `tail(add(a, b), 1)`, which drops the carry bit. We will see this in the next sections.\n", "\n", - "### Other Expressions\n", - "Other expressions including `SubField`, `SubIndex`, `SubAccess`, `Mux`, `ValidIf` etc. are described in more detail in [firrtl/src/main/scala/firrtl/ir/IR.scala](https://github.com/ucb-bar/firrtl/blob/master/src/main/scala/firrtl/ir/IR.scala) and [The FIRRTL Specification](https://github.com/ucb-bar/firrtl/blob/master/spec/spec.pdf).\n", - "\n", - "# Back to our example\n", - "\n", - "Let's take another look at the FIRRTL AST from our example. Hopefully, the structure of the design makes more sense!" + "**Exercise: Read the FIRRTL**
\n", + "Below is a module `EnabledReg`, which should keep its value unless `io.en` is true, in which case it loads `io.d`. The register must reset to 0. Complete the module, and then look at the FIRRTL printed by the next line. Which FIRRTL statements did your Chisel code turn into? (Hint: you should find a `regreset`, a `when` and some `connect`s.)" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ - "println(stringifyAST(firrtlAST))" + "class EnabledReg extends Module {\n", + " val io = IO(new Bundle {\n", + " val en = Input(Bool())\n", + " val d = Input(UInt(8.W))\n", + " val q = Output(UInt(8.W))\n", + " })\n", + "\n", + " val r = ??? // a register with an 8-bit width and reset value 0\n", + " ??? // when `en` is true, load `d` into r\n", + " io.q := r\n", + "}\n", + "\n", + "val enabledRegFirrtl = getFirrtl(new EnabledReg)\n", + "println(enabledRegFirrtl.linesIterator.dropWhile(!_.contains(\"module\")).mkString(\"\\n\"))\n", + "assert(enabledRegFirrtl.contains(\"regreset\") && enabledRegFirrtl.contains(\"when io.en\"))\n", + "println(\"SUCCESS!!\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + "\n", + "
\n", + "
\n",
+    "class EnabledReg extends Module {\n",
+    "  val io = IO(new Bundle {\n",
+    "    val en = Input(Bool())\n",
+    "    val d  = Input(UInt(8.W))\n",
+    "    val q  = Output(UInt(8.W))\n",
+    "  })\n",
+    "\n",
+    "  val r = RegInit(0.U(8.W))\n",
+    "  when (io.en) {\n",
+    "    r := io.d\n",
+    "  }\n",
+    "  io.q := r\n",
+    "}\n",
+    "
" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "That's it for this section! In the next section, we will build a Scala data structure for FIRRTL and look at how a program walks it." ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "That's it for this section! In the next section, we will look at how a FIRRTL transformation walks this AST and modifies it." + "# You're done!\n", + "\n", + "[Return to the top.](#top)" ] } ], @@ -335,12 +437,11 @@ "codemirror_mode": "text/x-scala", "file_extension": ".scala", "mimetype": "text/x-scala", - "name": "scala211", + "name": "scala", "nbconvert_exporter": "script", - "pygments_lexer": "scala", - "version": "2.11.11" + "version": "2.13.18" } }, "nbformat": 4, - "nbformat_minor": 2 -} \ No newline at end of file + "nbformat_minor": 1 +} diff --git a/4.2_firrtl_ast_traversal.ipynb b/4.2_firrtl_ast_traversal.ipynb index 6397ca9..8546a62 100644 --- a/4.2_firrtl_ast_traversal.ipynb +++ b/4.2_firrtl_ast_traversal.ipynb @@ -1,6 +1,7 @@ { "cells": [ { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -8,21 +9,49 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "# Module 4.2: FIRRTL AST Traversal\n", - "\n", "**Prev: [Introduction to FIRRTL](4.1_firrtl_ast.ipynb)**
\n", "**Next: [Common Pass Idioms](4.3_firrtl_common_idioms.ipynb)**\n", "\n", - "### Understanding IR node children\n", - "\n", - "Writing a Firrtl pass usually requires writing functions which walk the Firrtl datastructure to either collect information or replace IR nodes with new IR nodes.\n", + "### From text to a tree\n", + "In the previous section we saw that Chisel emits FIRRTL as text. To analyze a design with a program, we want a data structure instead of a string: a tree whose nodes are circuits, modules, statements and expressions. Chisel 7 does not ship such a data structure (the FIRRTL compiler is now `firtool`, which is not a Scala program), so we will write a small one ourselves. This is a good exercise in Scala: case classes, pattern matching and the collection library are all that is needed.\n", "\n", - "The IR datastructure is a tree, where each IR node can have some number of children nodes (which in turn can have more children nodes, etc.). IR nodes without children are called leaves.\n", + "Our AST covers the subset of FIRRTL that Chisel emits for ordinary designs: modules, ports, registers, wires, nodes, instances, connects, `when`/`else` and all expressions. Anything else (printf, memories, ...) is kept as raw text in an `OtherStmt`.\n", "\n", - "Different IR nodes can have different children types. The following table shows the possible children type for each IR node type:\n", + "## Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "interp.load.module(os.pwd / \"source\" / \"load-ivy.sc\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import chisel3._\n", + "import chisel3.Input // avoid a name clash with almond.input.Input\n", + "import chisel3.util._" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## The IR nodes\n", + "The case classes below are our IR. Their names follow the ones of the (former) Scala FIRRTL library. The following table shows the possible children of each IR node type:\n", "\n", "```\n", "+------------+-----------------------------+\n", @@ -30,16 +59,422 @@ "+------------+-----------------------------+\n", "| Circuit | DefModule |\n", "| DefModule | Port, Statement |\n", - "| Port | Type, Direction |\n", - "| Statement | Statement, Expression, Type |\n", - "| Expression | Expression, Type |\n", - "| Type | Type, Width |\n", - "| Width | |\n", - "| Direction | |\n", + "| Statement | Statement, Expression |\n", + "| Expression | Expression |\n", "+------------+-----------------------------+\n", "```\n", "\n", - "### The map function\n", + "Types (`UInt<4>`, bundles, ...) are kept as plain strings, since we do not need to look inside them. Each statement also has an `info` field with its source locator (an empty string if there is none).\n", + "\n", + "The `mapExpr` and `mapStmt` methods apply a function to the children of a node. We will explain them below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "// ---- Expressions ----\n", + "sealed trait Expression {\n", + " // Apply f to the child Expressions of this node (like firrtl's `map`).\n", + " def mapExpr(f: Expression => Expression): Expression = this match {\n", + " case DoPrim(op, args, consts) => DoPrim(op, args.map(f), consts)\n", + " case leaf => leaf\n", + " }\n", + "}\n", + "case class Reference(name: String) extends Expression // x, io.in[0], m0.io.out\n", + "case class Literal(tpe: String, value: String) extends Expression // UInt<3>(0h5)\n", + "case class DoPrim(op: String, args: Seq[Expression], consts: Seq[Int]) extends Expression // add(a, b), bits(a, 3, 0), mux(c, a, b)\n", + "\n", + "// ---- Statements ----\n", + "sealed trait Statement {\n", + " def info: String // source locator, e.g. \"cmd3.sc 6:19\" (\"\" if there is none)\n", + "\n", + " // Apply f to the Expressions directly contained in this statement.\n", + " def mapExpr(f: Expression => Expression): Statement = this match {\n", + " case s: DefRegister => s.copy(clock = f(s.clock))\n", + " case s: DefRegisterReset => s.copy(clock = f(s.clock), reset = f(s.reset), init = f(s.init))\n", + " case s: DefNode => s.copy(value = f(s.value))\n", + " case s: Connect => s.copy(loc = f(s.loc), expr = f(s.expr))\n", + " case s: Conditionally => s.copy(pred = f(s.pred))\n", + " case other => other\n", + " }\n", + " // Apply f to the child Statements of this statement.\n", + " def mapStmt(f: Statement => Statement): Statement = this match {\n", + " case s: Conditionally => s.copy(conseq = s.conseq.map(f), alt = s.alt.map(f))\n", + " case s: OtherStmt => s.copy(children = s.children.map(f))\n", + " case other => other\n", + " }\n", + "}\n", + "sealed trait IsRegister extends Statement { def name: String; def tpe: String }\n", + "case class DefRegister(info: String, name: String, tpe: String, clock: Expression) extends IsRegister\n", + "case class DefRegisterReset(info: String, name: String, tpe: String, clock: Expression,\n", + " reset: Expression, init: Expression) extends IsRegister\n", + "case class DefWire(info: String, name: String, tpe: String) extends Statement\n", + "case class DefNode(info: String, name: String, value: Expression) extends Statement\n", + "case class DefInstance(info: String, name: String, module: String) extends Statement\n", + "case class Connect(info: String, loc: Expression, expr: Expression) extends Statement\n", + "case class Conditionally(info: String, pred: Expression, conseq: Seq[Statement], alt: Seq[Statement]) extends Statement\n", + "// Everything we do not model in detail (printf, assert, mem, ...). Keeps the raw text.\n", + "case class OtherStmt(info: String, text: String, children: Seq[Statement]) extends Statement\n", + "\n", + "// ---- Modules and circuits ----\n", + "case class Port(info: String, direction: String, name: String, tpe: String)\n", + "case class DefModule(info: String, name: String, isPublic: Boolean, ports: Seq[Port], body: Seq[Statement]) {\n", + " def mapStmt(f: Statement => Statement): DefModule = copy(body = body.map(f))\n", + "}\n", + "case class Circuit(main: String, modules: Seq[DefModule], otherDecls: Seq[String] = Nil) {\n", + " def mapModule(f: DefModule => DefModule): Circuit = copy(modules = modules.map(f))\n", + "}" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## A hand-made AST\n", + "Before parsing anything, we can build a tree by hand. This is the FIRRTL for `io.out := io.in + 1` in a hypothetical module, written as an AST:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "// Pretty-prints any of our case classes as an indented tree: one node per line.\n", + "def showTree(x: Any, depth: Int = 0, label: String = \"\"): String = {\n", + " val pad = \" \" * depth + (if (label.isEmpty) \"\" else label + \": \")\n", + " def isNode(v: Any): Boolean = v match {\n", + " case _: Expression | _: Statement | _: Port | _: DefModule | _: Circuit => true\n", + " case _ => false\n", + " }\n", + " x match {\n", + " case p: Product if isNode(p) =>\n", + " val fields = p.productElementNames.toList.zip(p.productIterator.toList)\n", + " val (kids, atoms) = fields.partition {\n", + " case (_, v: Seq[_]) => v.nonEmpty && v.forall(isNode)\n", + " case (_, v) => isNode(v)\n", + " }\n", + " val head = pad + p.productPrefix + atoms.filter(_._2 != \"\").map { case (n, v) => s\" $n=$v\" }.mkString\n", + " (head +: kids.flatMap {\n", + " case (n, v: Seq[_]) => v.map(showTree(_, depth + 1, n))\n", + " case (n, v) => Seq(showTree(v, depth + 1, n))\n", + " }).mkString(\"\\n\")\n", + " case other => pad + other.toString\n", + " }\n", + "}\n", + "\n", + "val addOne = DefModule(\"\", \"AddOne\", true,\n", + " Seq(Port(\"\", \"input\", \"in\", \"UInt<4>\"), Port(\"\", \"output\", \"out\", \"UInt<5>\")),\n", + " Seq(Connect(\"\", Reference(\"out\"), DoPrim(\"add\", Seq(Reference(\"in\"), Literal(\"UInt<1>\", \"0h1\")), Nil))))\n", + "\n", + "println(showTree(addOne))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`showTree` is our own pretty-printer. It prints one node per line, and the children are indented below their parent. Each case class also has the usual `toString`, but for large trees that is hard to read.\n", + "\n", + "## Parsing FIRRTL\n", + "Now, let's fill the tree from text. The FIRRTL emitted by Chisel is line-oriented and indentation-based, so parsing goes in four steps:\n", + "\n", + "1. Split the text into lines. Remember the indentation, and cut the source locator `@[...]` off the end of each line.\n", + "2. Use the indentation to arrange the lines into a tree: lines indented deeper than the previous one are its children.\n", + "3. Parse each line's text into an `Expression` or a `Statement`.\n", + "4. Put it together: skip the header and annotations, and parse each `module`.\n", + "\n", + "Step 1 and 2:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "case class Line(indent: Int, text: String, info: String)\n", + "case class Node(line: Line, kids: Seq[Node])\n", + "\n", + "val infoRegex = \"\"\"^(.*?)\\s*@\\[(.*)\\]\\s*$\"\"\".r\n", + "\n", + "def toLines(text: String): List[Line] =\n", + " text.linesIterator.filter(_.trim.nonEmpty).map { raw =>\n", + " val indent = raw.length - raw.dropWhile(_ == ' ').length\n", + " raw.trim match {\n", + " case infoRegex(code, info) => Line(indent, code, info)\n", + " case code => Line(indent, code, \"\")\n", + " }\n", + " }.toList\n", + "\n", + "// Parses the sibling lines at `indent`; a more indented line starts the children of the line before it.\n", + "def toTree(lines: List[Line], indent: Int): (Vector[Node], List[Line]) = {\n", + " var rest = lines\n", + " val siblings = Vector.newBuilder[Node]\n", + " while (rest.nonEmpty && rest.head.indent == indent) {\n", + " val line = rest.head\n", + " val (kids, remaining) =\n", + " if (rest.tail.nonEmpty && rest.tail.head.indent > indent) toTree(rest.tail, rest.tail.head.indent)\n", + " else (Vector.empty[Node], rest.tail)\n", + " siblings += Node(line, kids)\n", + " rest = remaining\n", + " }\n", + " (siblings.result(), rest)\n", + "}\n", + "\n", + "val sample = \"\"\" module Foo : @[a.scala 1:1]\n", + " input x : UInt<1>\n", + " when x : @[a.scala 2:3]\n", + " connect y, x @[a.scala 3:5]\n", + " else :\n", + " connect y, z\"\"\"\n", + "toTree(toLines(sample), 2)._1.foreach(println)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Step 3 starts with expressions. We split an expression into its operator and its arguments, taking care to split only at the commas that are not nested in parentheses, brackets, braces or angle brackets (think of `mux(c, add(a, b), d)` or the bundle type `{ a : UInt<1>, b : UInt<2> }`). Arguments that are plain integers are the constants of a primitive operation, like the `3` and `0` of `bits(x, 3, 0)`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "// Split on commas that are not nested inside (), [], {} or <>.\n", + "def splitTop(s: String): List[String] = {\n", + " var depth = 0\n", + " val parts = scala.collection.mutable.ListBuffer[String]()\n", + " val cur = new StringBuilder\n", + " for (c <- s) c match {\n", + " case '(' | '[' | '{' | '<' => depth += 1; cur += c\n", + " case ')' | ']' | '}' | '>' => depth -= 1; cur += c\n", + " case ',' if depth == 0 => parts += cur.toString.trim; cur.clear()\n", + " case _ => cur += c\n", + " }\n", + " if (cur.nonEmpty) parts += cur.toString.trim\n", + " parts.toList\n", + "}\n", + "\n", + "val literalRegex = \"\"\"^((?:UInt|SInt)(?:<\\d+>)?)\\((.*)\\)$\"\"\".r\n", + "val primRegex = \"\"\"^([A-Za-z0-9_]+)\\((.*)\\)$\"\"\".r\n", + "val intRegex = \"\"\"^-?\\d+$\"\"\".r\n", + "\n", + "def parseExpr(text: String): Expression = text.trim match {\n", + " case literalRegex(tpe, value) => Literal(tpe, value)\n", + " case primRegex(op, args) =>\n", + " val (consts, exprs) = splitTop(args).partition(a => intRegex.matches(a))\n", + " DoPrim(op, exprs.map(parseExpr), consts.map(_.toInt))\n", + " case ref => Reference(ref)\n", + "}\n", + "\n", + "println(showTree(parseExpr(\"mux(sel, add(a, UInt<3>(0h5)), bits(io.v[1], 3, 0))\")))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, statements, ports and modules. Each kind of line is recognized by a regular expression that is matched with Scala's pattern matching. A `when` is the only statement with nested statements that we model, and it has to deal with its `else :`, which is a *sibling* line of the `when`, but belongs to it. That is why `toStatements` converts the siblings together." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "val portRegex = \"\"\"^(input|output) (\\S+) : (.*)$\"\"\".r\n", + "val regRegex = \"\"\"^reg (\\S+) : (.*)$\"\"\".r\n", + "val regResetRegex = \"\"\"^regreset (\\S+) : (.*)$\"\"\".r\n", + "val wireRegex = \"\"\"^wire (\\S+) : (.*)$\"\"\".r\n", + "val nodeRegex = \"\"\"^node (\\S+) = (.*)$\"\"\".r\n", + "val instRegex = \"\"\"^inst (\\S+) of (\\S+)$\"\"\".r\n", + "val connectRegex = \"\"\"^connect (.*)$\"\"\".r\n", + "val whenRegex = \"\"\"^when (.*) :$\"\"\".r\n", + "val moduleRegex = \"\"\"^(public )?module (\\S+) :$\"\"\".r\n", + "\n", + "// An `else :` node belongs to the `when` right before it, so we convert siblings together.\n", + "def toStatements(nodes: Seq[Node]): Seq[Statement] =\n", + " nodes.foldLeft(Vector.empty[Statement]) {\n", + " case (done, Node(Line(_, \"else :\", _), kids)) =>\n", + " done.last match {\n", + " case w: Conditionally => done.init :+ w.copy(alt = toStatements(kids))\n", + " case _ => sys.error(\"`else` without `when`\")\n", + " }\n", + " case (done, node) => done :+ toStatement(node)\n", + " }\n", + "\n", + "def toStatement(node: Node): Statement = {\n", + " val Node(Line(_, text, info), kids) = node\n", + " text match {\n", + " case regResetRegex(name, rest) =>\n", + " val Seq(tpe, clock, reset, init) = splitTop(rest)\n", + " DefRegisterReset(info, name, tpe, parseExpr(clock), parseExpr(reset), parseExpr(init))\n", + " case regRegex(name, rest) =>\n", + " val Seq(tpe, clock) = splitTop(rest)\n", + " DefRegister(info, name, tpe, parseExpr(clock))\n", + " case wireRegex(name, tpe) => DefWire(info, name, tpe)\n", + " case nodeRegex(name, value) => DefNode(info, name, parseExpr(value))\n", + " case instRegex(name, module) => DefInstance(info, name, module)\n", + " case connectRegex(args) =>\n", + " val Seq(loc, expr) = splitTop(args)\n", + " Connect(info, parseExpr(loc), parseExpr(expr))\n", + " case whenRegex(pred) => Conditionally(info, parseExpr(pred), toStatements(kids), Nil)\n", + " case other => OtherStmt(info, other, toStatements(kids))\n", + " }\n", + "}\n", + "\n", + "def toModule(node: Node): DefModule = {\n", + " val Node(Line(_, text, info), kids) = node\n", + " val moduleRegex(pub, name) = text\n", + " val (portNodes, bodyNodes) = kids.partition(k => portRegex.matches(k.line.text))\n", + " val ports = portNodes.map { k =>\n", + " val portRegex(dir, pname, tpe) = k.line.text\n", + " Port(k.line.info, dir, pname, tpe)\n", + " }\n", + " DefModule(info, name, pub != null, ports, toStatements(bodyNodes))\n", + "}" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Step 4: the circuit. The first lines (the version and the annotations in `%[[ ... ]]`) are skipped, and the `layer` declarations are ignored. Modules are indented by two spaces, so that is the indentation of the first level of our tree." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def parseCircuit(firrtl: String): Circuit = {\n", + " // Skip the \"FIRRTL version\" line and the multi-line annotation block `circuit Top :%[[ ... ]]`.\n", + " val lines = firrtl.linesIterator.dropWhile(!_.startsWith(\"circuit \")).toList\n", + " val header = lines.head\n", + " val afterAnnos = if (header.endsWith(\"%[[\")) lines.tail.dropWhile(_.trim != \"]]\").tail else lines.tail\n", + " val main = header.stripPrefix(\"circuit \").takeWhile(c => c != ' ' && c != ':')\n", + " val (children, _) = toTree(toLines(afterAnnos.mkString(\"\\n\")), 2)\n", + " // `layer` and `extmodule` declarations are not analyzed: we keep their text to serialize it back.\n", + " def rawLines(n: Node, depth: Int): Seq[String] =\n", + " ((\" \" * depth) + n.line.text + (if (n.line.info.isEmpty) \"\" else s\" @[${n.line.info}]\")) +: n.kids.flatMap(rawLines(_, depth + 1))\n", + " val (modules, others) = children.partition(n => moduleRegex.matches(n.line.text))\n", + " Circuit(main, modules.map(toModule), others.flatMap(rawLines(_, 1)))\n", + "}" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's try it on the `DelayBy2` module from the previous section:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class DelayBy2(width: Int) extends Module {\n", + " val io = IO(new Bundle {\n", + " val in = Input(UInt(width.W))\n", + " val out = Output(UInt(width.W))\n", + " })\n", + " val r0 = RegNext(io.in)\n", + " val r1 = RegNext(r0)\n", + " io.out := r1\n", + "}\n", + "\n", + "lazy val delayAST = parseCircuit(getFirrtl(new DelayBy2(4)))\n", + "println(showTree(delayAST))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This is the data structure that holds the FIRRTL AST. It is a tree structure whose root node is a **Circuit**, which has the name of the top module and a list of **DefModule**s. Our module has three **Port**s (`clock`, `reset` and `io`), two registers and three connections. As in the previous section, the `info` fields hold the source locator that tells the line of Scala that produced a node.\n", + "\n", + "A parser is only useful if it does not lose information. We can check this by *serializing* the AST back to text , and parsing again: the result must be the same tree. The `serialize` functions below are the mirror image of the parser, one `match` per kind of node; there is nothing new in them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def serialize(e: Expression): String = e match {\n", + " case Reference(name) => name\n", + " case Literal(tpe, value) => s\"$tpe($value)\"\n", + " case DoPrim(op, args, consts) => (args.map(serialize) ++ consts).mkString(s\"$op(\", \", \", \")\")\n", + "}\n", + "\n", + "def serialize(s: Statement, indent: Int): Seq[String] = {\n", + " val pad = \" \" * indent\n", + " val loc = (i: String) => if (i.isEmpty) \"\" else s\" @[$i]\"\n", + " s match {\n", + " case DefRegister(i, n, t, c) => Seq(s\"${pad}reg $n : $t, ${serialize(c)}${loc(i)}\")\n", + " case DefRegisterReset(i, n, t, c, r, v) => Seq(s\"${pad}regreset $n : $t, ${serialize(c)}, ${serialize(r)}, ${serialize(v)}${loc(i)}\")\n", + " case DefWire(i, n, t) => Seq(s\"${pad}wire $n : $t${loc(i)}\")\n", + " case DefNode(i, n, v) => Seq(s\"${pad}node $n = ${serialize(v)}${loc(i)}\")\n", + " case DefInstance(i, n, m) => Seq(s\"${pad}inst $n of $m${loc(i)}\")\n", + " case Connect(i, l, r) => Seq(s\"${pad}connect ${serialize(l)}, ${serialize(r)}${loc(i)}\")\n", + " case Conditionally(i, p, c, a) =>\n", + " Seq(s\"${pad}when ${serialize(p)} :${loc(i)}\") ++ c.flatMap(serialize(_, indent + 1)) ++\n", + " (if (a.isEmpty) Nil else Seq(s\"${pad}else :\") ++ a.flatMap(serialize(_, indent + 1)))\n", + " case OtherStmt(i, t, kids) => Seq(s\"$pad$t${loc(i)}\") ++ kids.flatMap(serialize(_, indent + 1))\n", + " }\n", + "}\n", + "\n", + "def serialize(m: DefModule): Seq[String] = {\n", + " val loc = (i: String) => if (i.isEmpty) \"\" else s\" @[$i]\"\n", + " Seq(s\" ${if (m.isPublic) \"public \" else \"\"}module ${m.name} :${loc(m.info)}\") ++\n", + " m.ports.map(p => s\" ${p.direction} ${p.name} : ${p.tpe}${loc(p.info)}\") ++ Seq(\"\") ++\n", + " m.body.flatMap(serialize(_, 2)) ++ Seq(\"\")\n", + "}\n", + "\n", + "def serialize(c: Circuit): String =\n", + " (Seq(s\"circuit ${c.main} :\") ++ c.otherDecls ++ c.modules.flatMap(serialize)).mkString(\"\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "println(serialize(delayAST))\n", + "assert(parseCircuit(serialize(delayAST)) == delayAST)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Traversal\n", + "### Understanding the map function\n", + "\n", + "Writing an analysis usually requires writing functions which walk the AST to either collect information or replace nodes with new nodes.\n", + "\n", + "The IR datastructure is a tree, where each IR node can have some number of children nodes (which in turn can have more children nodes, etc.). IR nodes without children are called leaves.\n", "\n", "To write a function that traverses a `Circuit`, we need to first understand the functional programming concept `map`.\n", "\n", @@ -64,7 +499,7 @@ "val s = Seq(\"a\", \"b\", \"c\")\n", "def f(x: String): String = x + x // repeated declaration for clarity\n", "val t = s.map(f)\n", - "println(t) // Seq(\"aa\", \"bb\", \"cc\")\n", + "println(t) // List(\"aa\", \"bb\", \"cc\")\n", "```\n", "```\n", " Seq\n", @@ -72,86 +507,254 @@ "\"aa\" \"bb\" \"cc\"\n", "```\n", "\n", - "#### Understanding Firrtl's map\n", + "#### Our map functions\n", "\n", "We use this \"mapping\" idea to create our own, custom `map` methods on IR nodes. Suppose we have a `DoPrim` expression representing 1 + 1; this can be depicted as a tree of expressions with a root node `DoPrim`:\n", "```\n", " DoPrim\n", " / \\\n", - "UIntValue UIntValue\n", + " Literal Literal\n", "```\n", "\n", "If we have a function `f` that takes an `Expression` argument and returns a new `Expression`, we can \"map\" it onto all children `Expression` of a given IR node, like our `DoPrim`. This would return the following new `DoPrim`, whose children are the result of applying `f` to every `Expression` child of `DoPrim`:\n", "```\n", " DoPrim\n", " / \\\n", - "f(UIntValue) f(UIntValue)\n", + " f(Literal) f(Literal)\n", "```\n", "\n", - "Sometimes IR nodes have children of multiple types. For example, `Conditionally` has both `Expression` and `Statement` children. In this case, the map will only apply its function to the children whose type matches the function's argument type (and return value type):\n", + "Sometimes IR nodes have children of multiple types. For example, `Conditionally` has both `Expression` and `Statement` children. This is why we have two methods, `mapExpr` and `mapStmt`:\n", "```scala\n", - "val c = Conditionally(info, e, s1, s2) // e: Expression, s1, s2: Statement, info: FileInfo\n", + "val c = Conditionally(info, e, Seq(s1), Seq(s2)) // e: Expression, s1, s2: Statement, info: String\n", "def fExp(e: Expression): Expression = ...\n", "def fStmt(s: Statement): Statement = ...\n", - "c.map(fExp) // Conditionally(fExp(e), s1, s2)\n", - "c.map(fStmt) // Conditionally(e, fStmt(s1), fStmt(s2))\n", + "c.mapExpr(fExp) // Conditionally(info, fExp(e), Seq(s1), Seq(s2))\n", + "c.mapStmt(fStmt) // Conditionally(info, e, Seq(fStmt(s1)), Seq(fStmt(s2)))\n", "```\n", + "You can look at their implementation in the case class definitions above. Note that they apply the function only to the *direct* children, they do not recurse. Recursion is up to the function we write, as we will see now.\n", "\n", - "Scala has \"infix notation\", which allows you to drop the `.` and parenthesis when calling a function which has one argument. Often, we write these map functions with infix notation:\n", - "```scala\n", - "c map fExp // equivalent to c.map(fExp)\n", - "c map fStmt // equivalent to c.map(fStmt)\n", - "```\n", + "Scala has \"infix notation\", which allows you to drop the `.` and parentheses when calling a function which has one argument. Often, we write such calls with infix notation: `c mapExpr fExp` is equivalent to `c.mapExpr(fExp)`.\n", "\n", "### Pre-order traversal\n", "\n", - "To traverse a Firrtl tree, we use `map` to write recursive functions which visit every child of every node we care about.\n", + "To traverse the tree, we use `map` to write recursive functions which visit every child of every node we care about.\n", "\n", - "Suppose we want to collect the names of every register declared in the design; we know this requires visiting every `Statement`. However, some `Statement` nodes can have children `Statement`. Thus, we need to write a function that will both check if its input argument is a `DefRegister` and, if not, will recursively apply `f` to all `Statement` children of its input argument:\n", + "Suppose we want to collect the names of every register declared in the design; we know this requires visiting every `Statement`. However, some `Statement` nodes can have children `Statement`s (the body of a `when`). Thus, we need to write a function that will both check if its input argument is a register and, if not, will recursively apply itself to all `Statement` children of its input argument.\n", "\n", - "The following function, `f`, is similar to our described function yet it takes two arguments: a mutable hashset of register names, and a `Statement`. Using function currying, we can pass only the first argument to return a new function with the desired type signature (`Statement=>Statement`):\n", + "The following function `collectRegs` takes two arguments: a mutable set of register names, and a `Statement`. Using function currying, we can pass only the first argument to return a new function with the desired type signature (`Statement => Statement`):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import scala.collection.mutable\n", "\n", - "```scala\n", - "def f(regNames: mutable.HashSet[String]())(s: Statement): Statement = s match {\n", + "def collectRegs(regNames: mutable.LinkedHashSet[String])(s: Statement): Statement = s match {\n", " // If register, add name to regNames\n", - " case r: DefRegister =>\n", + " case r: IsRegister =>\n", " regNames += r.name\n", - " r // Return argument unchanged (ok because DefRegister has no Statement children)\n", - " // If not, apply f(regNames) to all children Statement\n", - " case _ => s map f(regNames) // Note that f(regNames) is of type Statement=>Statement\n", + " r // Return argument unchanged (ok because registers have no Statement children)\n", + " // If not, apply collectRegs(regNames) to all children Statement\n", + " case _ => s.mapStmt(collectRegs(regNames)) // Note that collectRegs(regNames) is of type Statement => Statement\n", "}\n", - "```\n", "\n", - "This pattern is very common in Firrtl, and is called \"pre-order traversal\" because the recursive function matches on the original IR node before recursively applying to its children nodes.\n", + "// A module with a register inside a `when`, to make it interesting\n", + "class WhenReg extends Module {\n", + " val io = IO(new Bundle { val en = Input(Bool()); val d = Input(UInt(4.W)); val q = Output(UInt(4.W)) })\n", + " val a = Reg(UInt(4.W))\n", + " io.q := a\n", + " when (io.en) {\n", + " val b = RegNext(io.d)\n", + " a := b + 1.U\n", + " }\n", + "}\n", "\n", - "### Post-order traversal\n", + "lazy val whenRegAST = parseCircuit(getFirrtl(new WhenReg))\n", + "val regNames = mutable.LinkedHashSet[String]()\n", + "whenRegAST.modules.foreach(m => m.mapStmt(collectRegs(regNames)))\n", + "println(regNames)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This pattern is very common, and is called \"pre-order traversal\" because the recursive function matches on the original IR node before recursively applying to its children nodes. Note that we call `m.mapStmt` only for its side effect on `regNames`, and ignore the new module it returns.\n", "\n", - "We can write the previous example in a \"post-order traversal\" as follows:\n", + "### Post-order traversal\n", "\n", - "```scala\n", - "def f(regNames: mutable.HashSet[String]())(s: Statement): Statement = \n", - " // Not we immediately recurse to the children nodes, then match\n", - " s map f(regName) match {\n", + "We can write the previous example in a \"post-order traversal\" as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def collectRegsPost(regNames: mutable.LinkedHashSet[String])(s: Statement): Statement =\n", + " // Note we immediately recurse to the children nodes, then match\n", + " s.mapStmt(collectRegsPost(regNames)) match {\n", " // If register, add name to regNames\n", - " case r: DefRegister =>\n", + " case r: IsRegister =>\n", " regNames += r.name\n", - " r // Return argument unchanged (ok because DefRegister has no Statement children)\n", + " r\n", " // If not, return s\n", - " case _ => s // Note that all Statement children of s have had f(regNames) already applied\n", + " case other => other // Note that all Statement children of s have had collectRegsPost(regNames) already applied\n", " }\n", - "```\n", "\n", - "While the traversal ordering is different between these two examples, it makes no difference for this use case (and many others). However, it is an important tool to keep in your back pocket for when the traversal ordering matters." + "val regNames2 = mutable.LinkedHashSet[String]()\n", + "whenRegAST.modules.foreach(m => m.mapStmt(collectRegsPost(regNames2)))\n", + "println(regNames2)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "While the traversal ordering is different between these two examples, it makes no difference for this use case (and many others). However, it is an important tool to keep in your back pocket for when the traversal ordering matters.\n", + "\n", + "### The collection-style traversal\n", + "Mutable sets and `map` calls that throw away their result are a bit clumsy for pure analysis. Since our tree is an ordinary Scala data structure, we can also *flatten* it into a sequence and use the whole collections library. Let's define three helpers:\n", + "\n", + "* `allStatements(body)`: every statement of a body, including those nested in `when`s;\n", + "* `directExprs(stmt)`: the expressions that appear directly in a statement;\n", + "* `allExprs(expr)`: an expression and all of its sub-expressions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "// All statements of a body, including the ones nested inside `when`s (pre-order).\n", + "def allStatements(body: Seq[Statement]): Seq[Statement] = body.flatMap {\n", + " case w: Conditionally => w +: (allStatements(w.conseq) ++ allStatements(w.alt))\n", + " case o: OtherStmt => o +: allStatements(o.children)\n", + " case s => Seq(s)\n", + "}\n", + "\n", + "// The Expressions that appear directly in a statement (not the sub-expressions).\n", + "def directExprs(s: Statement): Seq[Expression] = s match {\n", + " case r: DefRegister => Seq(r.clock)\n", + " case r: DefRegisterReset => Seq(r.clock, r.reset, r.init)\n", + " case n: DefNode => Seq(n.value)\n", + " case c: Connect => Seq(c.loc, c.expr)\n", + " case w: Conditionally => Seq(w.pred)\n", + " case _ => Nil\n", + "}\n", + "\n", + "// An Expression and all its sub-expressions (pre-order).\n", + "def allExprs(e: Expression): Seq[Expression] = e match {\n", + " case DoPrim(_, args, _) => e +: args.flatMap(allExprs)\n", + " case leaf => Seq(e)\n", + "}" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With them, \"collect all registers\" becomes a one-liner:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for (m <- whenRegAST.modules) {\n", + " println(m.name + \": \" + allStatements(m.body).collect { case r: IsRegister => r.name })\n", + "}" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "...and \"which primitive operations does this module use, and how often?\" is not much longer. We take all statements, all of their expressions, all of the sub-expressions, and keep the `DoPrim`s:" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], - "source": [] + "source": [ + "for (m <- whenRegAST.modules) {\n", + " val ops = allStatements(m.body).flatMap(directExprs).flatMap(allExprs).collect { case DoPrim(op, _, _) => op }\n", + " println(m.name + \": \" + ops.groupBy(identity).map { case (op, l) => op -> l.size })\n", + "}" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Exercise: Operators used**
\n", + "Write a function `opsUsed` that returns the set of names of the primitive operations that a module uses (e.g. `Set(\"add\", \"tail\", ...)`). It should find the operations inside of `when` blocks too. We test it with a module that computes `(a + b) ^ a` (the `+` is a `tail(add(...), 1)` in FIRRTL)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def opsUsed(m: DefModule): Set[String] = ???\n", + "\n", + "class OpsDemo extends Module {\n", + " val io = IO(new Bundle { val a = Input(UInt(4.W)); val b = Input(UInt(4.W)); val out = Output(UInt(4.W)) })\n", + " io.out := (io.a + io.b) ^ io.a\n", + "}\n", + "lazy val opsDemoAST = parseCircuit(getFirrtl(new OpsDemo))\n", + "println(opsUsed(opsDemoAST.modules.head))\n", + "assert(opsUsed(opsDemoAST.modules.head) == Set(\"add\", \"tail\", \"xor\"))\n", + "println(\"SUCCESS!!\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + "\n", + "
\n", + "
\n",
+    "def opsUsed(m: DefModule): Set[String] =\n",
+    "  allStatements(m.body).flatMap(directExprs).flatMap(allExprs).collect { case DoPrim(op, _, _) => op }.toSet\n",
+    "
" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "That's it for this section! In the next section, we will see how analyses and transformations of a design are commonly structured." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# You're done!\n", + "\n", + "[Return to the top.](#top)" + ] } ], "metadata": { @@ -164,12 +767,11 @@ "codemirror_mode": "text/x-scala", "file_extension": ".scala", "mimetype": "text/x-scala", - "name": "scala211", + "name": "scala", "nbconvert_exporter": "script", - "pygments_lexer": "scala", - "version": "2.11.11" + "version": "2.13.18" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 1 } diff --git a/4.3_firrtl_common_idioms.ipynb b/4.3_firrtl_common_idioms.ipynb index e676cf1..d7c0cdd 100644 --- a/4.3_firrtl_common_idioms.ipynb +++ b/4.3_firrtl_common_idioms.ipynb @@ -1,6 +1,7 @@ { "cells": [ { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -8,145 +9,752 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "# Module 4.3: Common Pass Idioms\n", - "\n", "**Prev: [FIRRTL AST Traversal](4.2_firrtl_ast_traversal.ipynb)**
\n", "**Next: [A FIRRTL Transform Example](4.4_firrtl_add_ops_per_module.ipynb)**\n", "\n", + "In this section we look at the typical tasks that programs working on a FIRRTL AST perform: *collecting* information, *adding* statements and *deleting* statements. We will use the AST from the previous section.\n", + "\n", + "## Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "interp.load.module(os.pwd / \"source\" / \"load-ivy.sc\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import chisel3._\n", + "import chisel3.Input // avoid a name clash with almond.input.Input\n", + "import chisel3.util._\n", + "import chisel3.reflect.DataMirror" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The IR, the parser (`parseCircuit`), `serialize` and the traversal helpers (`allStatements`, `directExprs`, `allExprs`) that we wrote in Module 4.2 are needed again. Notebooks do not share definitions, so the next cell simply contains all of that code once more. Run it, and skip over it:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "// The FIRRTL library of Modules 4.2: IR, parser, serializer and traversal helpers.\n", + "// Nothing new here: just run this cell.\n", + "// ---- Expressions ----\n", + "sealed trait Expression {\n", + " // Apply f to the child Expressions of this node (like firrtl's `map`).\n", + " def mapExpr(f: Expression => Expression): Expression = this match {\n", + " case DoPrim(op, args, consts) => DoPrim(op, args.map(f), consts)\n", + " case leaf => leaf\n", + " }\n", + "}\n", + "case class Reference(name: String) extends Expression // x, io.in[0], m0.io.out\n", + "case class Literal(tpe: String, value: String) extends Expression // UInt<3>(0h5)\n", + "case class DoPrim(op: String, args: Seq[Expression], consts: Seq[Int]) extends Expression // add(a, b), bits(a, 3, 0), mux(c, a, b)\n", + "\n", + "// ---- Statements ----\n", + "sealed trait Statement {\n", + " def info: String // source locator, e.g. \"cmd3.sc 6:19\" (\"\" if there is none)\n", + "\n", + " // Apply f to the Expressions directly contained in this statement.\n", + " def mapExpr(f: Expression => Expression): Statement = this match {\n", + " case s: DefRegister => s.copy(clock = f(s.clock))\n", + " case s: DefRegisterReset => s.copy(clock = f(s.clock), reset = f(s.reset), init = f(s.init))\n", + " case s: DefNode => s.copy(value = f(s.value))\n", + " case s: Connect => s.copy(loc = f(s.loc), expr = f(s.expr))\n", + " case s: Conditionally => s.copy(pred = f(s.pred))\n", + " case other => other\n", + " }\n", + " // Apply f to the child Statements of this statement.\n", + " def mapStmt(f: Statement => Statement): Statement = this match {\n", + " case s: Conditionally => s.copy(conseq = s.conseq.map(f), alt = s.alt.map(f))\n", + " case s: OtherStmt => s.copy(children = s.children.map(f))\n", + " case other => other\n", + " }\n", + "}\n", + "sealed trait IsRegister extends Statement { def name: String; def tpe: String }\n", + "case class DefRegister(info: String, name: String, tpe: String, clock: Expression) extends IsRegister\n", + "case class DefRegisterReset(info: String, name: String, tpe: String, clock: Expression,\n", + " reset: Expression, init: Expression) extends IsRegister\n", + "case class DefWire(info: String, name: String, tpe: String) extends Statement\n", + "case class DefNode(info: String, name: String, value: Expression) extends Statement\n", + "case class DefInstance(info: String, name: String, module: String) extends Statement\n", + "case class Connect(info: String, loc: Expression, expr: Expression) extends Statement\n", + "case class Conditionally(info: String, pred: Expression, conseq: Seq[Statement], alt: Seq[Statement]) extends Statement\n", + "// Everything we do not model in detail (printf, assert, mem, ...). Keeps the raw text.\n", + "case class OtherStmt(info: String, text: String, children: Seq[Statement]) extends Statement\n", + "\n", + "// ---- Modules and circuits ----\n", + "case class Port(info: String, direction: String, name: String, tpe: String)\n", + "case class DefModule(info: String, name: String, isPublic: Boolean, ports: Seq[Port], body: Seq[Statement]) {\n", + " def mapStmt(f: Statement => Statement): DefModule = copy(body = body.map(f))\n", + "}\n", + "case class Circuit(main: String, modules: Seq[DefModule], otherDecls: Seq[String] = Nil) {\n", + " def mapModule(f: DefModule => DefModule): Circuit = copy(modules = modules.map(f))\n", + "}\n", + "\n", + "// ---- Step 1: split into lines and recover the nesting from the indentation ----\n", + "case class Line(indent: Int, text: String, info: String)\n", + "case class Node(line: Line, kids: Seq[Node])\n", + "\n", + "val infoRegex = \"\"\"^(.*?)\\s*@\\[(.*)\\]\\s*$\"\"\".r\n", + "\n", + "def toLines(text: String): List[Line] =\n", + " text.linesIterator.filter(_.trim.nonEmpty).map { raw =>\n", + " val indent = raw.length - raw.dropWhile(_ == ' ').length\n", + " raw.trim match {\n", + " case infoRegex(code, info) => Line(indent, code, info)\n", + " case code => Line(indent, code, \"\")\n", + " }\n", + " }.toList\n", + "\n", + "// Parses the sibling lines at `indent`; a more indented line starts the children of the line before it.\n", + "def toTree(lines: List[Line], indent: Int): (Vector[Node], List[Line]) = {\n", + " var rest = lines\n", + " val siblings = Vector.newBuilder[Node]\n", + " while (rest.nonEmpty && rest.head.indent == indent) {\n", + " val line = rest.head\n", + " val (kids, remaining) =\n", + " if (rest.tail.nonEmpty && rest.tail.head.indent > indent) toTree(rest.tail, rest.tail.head.indent)\n", + " else (Vector.empty[Node], rest.tail)\n", + " siblings += Node(line, kids)\n", + " rest = remaining\n", + " }\n", + " (siblings.result(), rest)\n", + "}\n", + "\n", + "// ---- Step 2: expressions ----\n", + "// Split on commas that are not nested inside (), [], {} or <>.\n", + "def splitTop(s: String): List[String] = {\n", + " var depth = 0\n", + " val parts = scala.collection.mutable.ListBuffer[String]()\n", + " val cur = new StringBuilder\n", + " for (c <- s) c match {\n", + " case '(' | '[' | '{' | '<' => depth += 1; cur += c\n", + " case ')' | ']' | '}' | '>' => depth -= 1; cur += c\n", + " case ',' if depth == 0 => parts += cur.toString.trim; cur.clear()\n", + " case _ => cur += c\n", + " }\n", + " if (cur.nonEmpty) parts += cur.toString.trim\n", + " parts.toList\n", + "}\n", + "\n", + "val literalRegex = \"\"\"^((?:UInt|SInt)(?:<\\d+>)?)\\((.*)\\)$\"\"\".r\n", + "val primRegex = \"\"\"^([A-Za-z0-9_]+)\\((.*)\\)$\"\"\".r\n", + "val intRegex = \"\"\"^-?\\d+$\"\"\".r\n", + "\n", + "def parseExpr(text: String): Expression = text.trim match {\n", + " case literalRegex(tpe, value) => Literal(tpe, value)\n", + " case primRegex(op, args) =>\n", + " val (consts, exprs) = splitTop(args).partition(a => intRegex.matches(a))\n", + " DoPrim(op, exprs.map(parseExpr), consts.map(_.toInt))\n", + " case ref => Reference(ref)\n", + "}\n", + "\n", + "// ---- Step 3: statements, ports, modules ----\n", + "val portRegex = \"\"\"^(input|output) (\\S+) : (.*)$\"\"\".r\n", + "val regRegex = \"\"\"^reg (\\S+) : (.*)$\"\"\".r\n", + "val regResetRegex = \"\"\"^regreset (\\S+) : (.*)$\"\"\".r\n", + "val wireRegex = \"\"\"^wire (\\S+) : (.*)$\"\"\".r\n", + "val nodeRegex = \"\"\"^node (\\S+) = (.*)$\"\"\".r\n", + "val instRegex = \"\"\"^inst (\\S+) of (\\S+)$\"\"\".r\n", + "val connectRegex = \"\"\"^connect (.*)$\"\"\".r\n", + "val whenRegex = \"\"\"^when (.*) :$\"\"\".r\n", + "val moduleRegex = \"\"\"^(public )?module (\\S+) :$\"\"\".r\n", + "\n", + "// An `else :` node belongs to the `when` right before it, so we convert siblings together.\n", + "def toStatements(nodes: Seq[Node]): Seq[Statement] =\n", + " nodes.foldLeft(Vector.empty[Statement]) {\n", + " case (done, Node(Line(_, \"else :\", _), kids)) =>\n", + " done.last match {\n", + " case w: Conditionally => done.init :+ w.copy(alt = toStatements(kids))\n", + " case _ => sys.error(\"`else` without `when`\")\n", + " }\n", + " case (done, node) => done :+ toStatement(node)\n", + " }\n", + "\n", + "def toStatement(node: Node): Statement = {\n", + " val Node(Line(_, text, info), kids) = node\n", + " text match {\n", + " case regResetRegex(name, rest) =>\n", + " val Seq(tpe, clock, reset, init) = splitTop(rest)\n", + " DefRegisterReset(info, name, tpe, parseExpr(clock), parseExpr(reset), parseExpr(init))\n", + " case regRegex(name, rest) =>\n", + " val Seq(tpe, clock) = splitTop(rest)\n", + " DefRegister(info, name, tpe, parseExpr(clock))\n", + " case wireRegex(name, tpe) => DefWire(info, name, tpe)\n", + " case nodeRegex(name, value) => DefNode(info, name, parseExpr(value))\n", + " case instRegex(name, module) => DefInstance(info, name, module)\n", + " case connectRegex(args) =>\n", + " val Seq(loc, expr) = splitTop(args)\n", + " Connect(info, parseExpr(loc), parseExpr(expr))\n", + " case whenRegex(pred) => Conditionally(info, parseExpr(pred), toStatements(kids), Nil)\n", + " case other => OtherStmt(info, other, toStatements(kids))\n", + " }\n", + "}\n", + "\n", + "def toModule(node: Node): DefModule = {\n", + " val Node(Line(_, text, info), kids) = node\n", + " val moduleRegex(pub, name) = text\n", + " val (portNodes, bodyNodes) = kids.partition(k => portRegex.matches(k.line.text))\n", + " val ports = portNodes.map { k =>\n", + " val portRegex(dir, pname, tpe) = k.line.text\n", + " Port(k.line.info, dir, pname, tpe)\n", + " }\n", + " DefModule(info, name, pub != null, ports, toStatements(bodyNodes))\n", + "}\n", + "\n", + "// ---- Step 4: the whole file ----\n", + "def parseCircuit(firrtl: String): Circuit = {\n", + " // Skip the \"FIRRTL version\" line and the multi-line annotation block `circuit Top :%[[ ... ]]`.\n", + " val lines = firrtl.linesIterator.dropWhile(!_.startsWith(\"circuit \")).toList\n", + " val header = lines.head\n", + " val afterAnnos = if (header.endsWith(\"%[[\")) lines.tail.dropWhile(_.trim != \"]]\").tail else lines.tail\n", + " val main = header.stripPrefix(\"circuit \").takeWhile(c => c != ' ' && c != ':')\n", + " val (children, _) = toTree(toLines(afterAnnos.mkString(\"\\n\")), 2)\n", + " // `layer` and `extmodule` declarations are not analyzed: we keep their text to serialize it back.\n", + " def rawLines(n: Node, depth: Int): Seq[String] =\n", + " ((\" \" * depth) + n.line.text + (if (n.line.info.isEmpty) \"\" else s\" @[${n.line.info}]\")) +: n.kids.flatMap(rawLines(_, depth + 1))\n", + " val (modules, others) = children.partition(n => moduleRegex.matches(n.line.text))\n", + " Circuit(main, modules.map(toModule), others.flatMap(rawLines(_, 1)))\n", + "}\n", + "\n", + "def serialize(e: Expression): String = e match {\n", + " case Reference(name) => name\n", + " case Literal(tpe, value) => s\"$tpe($value)\"\n", + " case DoPrim(op, args, consts) => (args.map(serialize) ++ consts).mkString(s\"$op(\", \", \", \")\")\n", + "}\n", + "\n", + "def serialize(s: Statement, indent: Int): Seq[String] = {\n", + " val pad = \" \" * indent\n", + " val loc = (i: String) => if (i.isEmpty) \"\" else s\" @[$i]\"\n", + " s match {\n", + " case DefRegister(i, n, t, c) => Seq(s\"${pad}reg $n : $t, ${serialize(c)}${loc(i)}\")\n", + " case DefRegisterReset(i, n, t, c, r, v) => Seq(s\"${pad}regreset $n : $t, ${serialize(c)}, ${serialize(r)}, ${serialize(v)}${loc(i)}\")\n", + " case DefWire(i, n, t) => Seq(s\"${pad}wire $n : $t${loc(i)}\")\n", + " case DefNode(i, n, v) => Seq(s\"${pad}node $n = ${serialize(v)}${loc(i)}\")\n", + " case DefInstance(i, n, m) => Seq(s\"${pad}inst $n of $m${loc(i)}\")\n", + " case Connect(i, l, r) => Seq(s\"${pad}connect ${serialize(l)}, ${serialize(r)}${loc(i)}\")\n", + " case Conditionally(i, p, c, a) =>\n", + " Seq(s\"${pad}when ${serialize(p)} :${loc(i)}\") ++ c.flatMap(serialize(_, indent + 1)) ++\n", + " (if (a.isEmpty) Nil else Seq(s\"${pad}else :\") ++ a.flatMap(serialize(_, indent + 1)))\n", + " case OtherStmt(i, t, kids) => Seq(s\"$pad$t${loc(i)}\") ++ kids.flatMap(serialize(_, indent + 1))\n", + " }\n", + "}\n", + "\n", + "def serialize(m: DefModule): Seq[String] = {\n", + " val loc = (i: String) => if (i.isEmpty) \"\" else s\" @[$i]\"\n", + " Seq(s\" ${if (m.isPublic) \"public \" else \"\"}module ${m.name} :${loc(m.info)}\") ++\n", + " m.ports.map(p => s\" ${p.direction} ${p.name} : ${p.tpe}${loc(p.info)}\") ++ Seq(\"\") ++\n", + " m.body.flatMap(serialize(_, 2)) ++ Seq(\"\")\n", + "}\n", + "\n", + "def serialize(c: Circuit): String =\n", + " (Seq(s\"circuit ${c.main} :\") ++ c.otherDecls ++ c.modules.flatMap(serialize)).mkString(\"\\n\")\n", + "\n", + "// Pretty-prints any of our case classes as an indented tree: one node per line.\n", + "def showTree(x: Any, depth: Int = 0, label: String = \"\"): String = {\n", + " val pad = \" \" * depth + (if (label.isEmpty) \"\" else label + \": \")\n", + " def isNode(v: Any): Boolean = v match {\n", + " case _: Expression | _: Statement | _: Port | _: DefModule | _: Circuit => true\n", + " case _ => false\n", + " }\n", + " x match {\n", + " case p: Product if isNode(p) =>\n", + " val fields = p.productElementNames.toList.zip(p.productIterator.toList)\n", + " val (kids, atoms) = fields.partition {\n", + " case (_, v: Seq[_]) => v.nonEmpty && v.forall(isNode)\n", + " case (_, v) => isNode(v)\n", + " }\n", + " val head = pad + p.productPrefix + atoms.filter(_._2 != \"\").map { case (n, v) => s\" $n=$v\" }.mkString\n", + " (head +: kids.flatMap {\n", + " case (n, v: Seq[_]) => v.map(showTree(_, depth + 1, n))\n", + " case (n, v) => Seq(showTree(v, depth + 1, n))\n", + " }).mkString(\"\\n\")\n", + " case other => pad + other.toString\n", + " }\n", + "}\n", + "\n", + "// All statements of a body, including the ones nested inside `when`s (pre-order).\n", + "def allStatements(body: Seq[Statement]): Seq[Statement] = body.flatMap {\n", + " case w: Conditionally => w +: (allStatements(w.conseq) ++ allStatements(w.alt))\n", + " case o: OtherStmt => o +: allStatements(o.children)\n", + " case s => Seq(s)\n", + "}\n", + "\n", + "// The Expressions that appear directly in a statement (not the sub-expressions).\n", + "def directExprs(s: Statement): Seq[Expression] = s match {\n", + " case r: DefRegister => Seq(r.clock)\n", + " case r: DefRegisterReset => Seq(r.clock, r.reset, r.init)\n", + " case n: DefNode => Seq(n.value)\n", + " case c: Connect => Seq(c.loc, c.expr)\n", + " case w: Conditionally => Seq(w.pred)\n", + " case _ => Nil\n", + "}\n", + "\n", + "// An Expression and all its sub-expressions (pre-order).\n", + "def allExprs(e: Expression): Seq[Expression] = e match {\n", + " case DoPrim(_, args, _) => e +: args.flatMap(allExprs)\n", + " case leaf => Seq(e)\n", + "}" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will also use a small design, with two modules, where `Top` instantiates `Counter`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class Counter(width: Int) extends Module {\n", + " val io = IO(new Bundle {\n", + " val en = Input(Bool())\n", + " val count = Output(UInt(width.W))\n", + " })\n", + " val c = RegInit(0.U(width.W))\n", + " when (io.en) { c := c + 1.U }\n", + " io.count := c\n", + "}\n", + "\n", + "class Top extends Module {\n", + " val io = IO(new Bundle {\n", + " val en = Input(Bool())\n", + " val out = Output(UInt(8.W))\n", + " })\n", + " val fast = Module(new Counter(8))\n", + " val slow = Module(new Counter(4))\n", + " val last = Reg(Bool())\n", + " fast.io.en := io.en\n", + " last := fast.io.count(7)\n", + " slow.io.en := last\n", + " io.out := fast.io.count + slow.io.count\n", + "}\n", + "\n", + "lazy val topAST = parseCircuit(getFirrtl(new Top))\n", + "println(topAST.main + \": \" + topAST.modules.map(_.name))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Collecting information\n", + "Most analyses are *read-only*: they walk the tree and fill some data structure. We have seen in Module 4.2 how to collect the registers of a module. Two more examples follow.\n", + "\n", + "#### Find all connections to a signal\n", + "Which statements drive `io.out` in `Top`? In FIRRTL, signals are referenced by *text*, for example `io.out`, `fast.io.count` or `v[1]`; it is up to us to decide what we consider \"the same signal\". Here, we compare the whole text of the reference. Note how we have to look inside the `when` bodies as well, which `allStatements` does for us:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def connectsTo(m: DefModule, signal: String): Seq[Connect] =\n", + " allStatements(m.body).collect { case c @ Connect(_, Reference(`signal`), _) => c }\n", + "\n", + "lazy val topModule = topAST.modules.find(_.name == \"Top\").get\n", + "connectsTo(topModule, \"io.out\").foreach(c => println(serialize(c, 0).head))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The back-quotes in `Reference(`signal`)` make Scala compare with the *value* of `signal`, instead of introducing a new variable called `signal` in the pattern.\n", + "\n", + "Sometimes we want the opposite: a signal's **fan-out**, i.e., all the places where it is *read*. A read is a `Reference` somewhere in an expression, except the left-hand side of a connect:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def readsOf(m: DefModule, signal: String): Seq[String] =\n", + " allStatements(m.body).flatMap {\n", + " case Connect(_, _, rhs) => allExprs(rhs).collect { case Reference(`signal`) => serialize(rhs) }\n", + " case DefNode(_, name, value) => allExprs(value).collect { case Reference(`signal`) => s\"node $name = ${serialize(value)}\" }\n", + " // clocks, resets, `when` predicates...\n", + " case s => directExprs(s).flatMap(e => allExprs(e).collect { case Reference(`signal`) => serialize(e) })\n", + " }\n", + "\n", + "println(readsOf(topModule, \"fast.io.count\"))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Be careful: the Chisel expression `fast.io.count(7)` appears in the FIRRTL as `bits(fast.io.count, 7, 7)` inside of a node, which is why it is found among the `DefNode`s. The Chisel source and the FIRRTL differ, and it is the FIRRTL that you analyze here.\n", + "\n", + "#### The module-instance graph\n", + "Which module instantiates which? Every `DefInstance` in a module body is an edge of the hierarchy. Because FIRRTL modules are never nested, this is a graph over module names (and, since a module cannot instantiate itself, directly or not, a DAG):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "// module name -> names of the modules it instantiates (with repetition: an instance for each element)\n", + "def instanceGraph(c: Circuit): Map[String, Seq[String]] =\n", + " c.modules.map { m =>\n", + " m.name -> allStatements(m.body).collect { case DefInstance(_, _, module) => module }\n", + " }.toMap\n", + "\n", + "println(instanceGraph(topAST))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "A tree view makes the hierarchy easier to see. Recursion on the graph visits `Top`, then each of the modules that it instantiates:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def printHierarchy(c: Circuit): Unit = {\n", + " val instNames = c.modules.map(m => m.name -> allStatements(m.body).collect { case d: DefInstance => d }).toMap\n", + " def go(module: String, instName: String, depth: Int): Unit = {\n", + " println(\" \" * depth + s\"$instName : $module\")\n", + " // An `extmodule` (BlackBox) is not parsed: it is a leaf\n", + " instNames.getOrElse(module, Nil).foreach(i => go(i.module, i.name, depth + 1))\n", + " }\n", + " go(c.main, c.main, 0)\n", + "}\n", + "printHierarchy(topAST)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And since the graph tells who uses whom, modules that nobody instantiates (other than the top) are dead code:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "val used = instanceGraph(topAST).values.flatten.toSet + topAST.main\n", + "println(\"unused modules: \" + topAST.modules.map(_.name).filterNot(used))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Looking at Chisel instead of FIRRTL\n", + "For *generator-time* introspection, there is `DataMirror`, which we met in Module 4.1. For example, inside of `Top`, we could have checked the port list of a child before wiring it up. The FIRRTL view is better when you need the whole design after elaboration; the Chisel view is better when you need to *act* during elaboration. Use whichever fits your problem." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class CheckedTop extends Module {\n", + " val io = IO(new Bundle { val out = Output(UInt(8.W)) })\n", + " val child = Module(new Counter(8))\n", + " child.io.en := true.B\n", + " io.out := child.io.count\n", + " // Which ports of `child` have a width of exactly 8 bits?\n", + " val wide = DataMirror.fullModulePorts(child).filter { case (_, d) => d.widthOption.contains(8) }\n", + " println(\"8-bit ports of child: \" + wide.map(_._1))\n", + "}\n", + "getFirrtl(new CheckedTop)\n", + "()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ "### Adding statements\n", - "Suppose we want to write a pass that splits nested DoPrim expressions, thus transforming this:\n", + "Suppose we want to write a pass that splits nested `DoPrim` expressions, thus transforming this:\n", "```\n", - "circuit Top:\n", + "circuit Top :\n", " module Top :\n", - " input x: UInt<3>\n", - " input y: UInt<3>\n", - " input z: UInt<3>\n", - " output o: UInt<3>\n", - " o <= add(x, add(y, z))\n", + " input x : UInt<3>\n", + " input y : UInt<3>\n", + " input z : UInt<3>\n", + " output o : UInt<5>\n", + " connect o, add(x, add(y, z))\n", "```\n", "into this:\n", "```\n", - "circuit Top:\n", + "circuit Top :\n", " module Top :\n", - " input x: UInt<3>\n", - " input y: UInt<3>\n", - " input z: UInt<3>\n", - " output o: UInt<3>\n", - " node GEN_1 = add(y, z)\n", - " o <= add(x, GEN_1)\n", + " input x : UInt<3>\n", + " input y : UInt<3>\n", + " input z : UInt<3>\n", + " output o : UInt<5>\n", + " node _GEN_0 = add(y, z)\n", + " connect o, add(x, _GEN_0)\n", "```\n", + "Our parser reads FIRRTL from any source, also from a string that we write by hand, so we can test on exactly this:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "val nested = \"\"\"circuit Top :\n", + " module Top :\n", + " input x : UInt<3>\n", + " input y : UInt<3>\n", + " input z : UInt<3>\n", + " output o : UInt<5>\n", + " connect o, add(x, add(y, z))\n", + "\"\"\"\n", + "println(showTree(parseCircuit(nested).modules.head.body.head))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We first need to traverse the AST to every `Statement` and `Expression`. Then, when we see a `DoPrim` *inside of another expression*, we need to add a new `DefNode` before the statement, and insert a reference to that `DefNode` in place of the `DoPrim`. The code below implements this (and preserves the source locator). It uses a *post-order* traversal of expressions: children are split first, so the innermost operation gets the first new name.\n", + "\n", + "Note that a statement is replaced by several statements: for this reason, the function that we map over the statements returns a `Seq[Statement]` and we use `flatMap`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "object Splitter {\n", + " /** Run splitM on every module */\n", + " def run(c: Circuit): Circuit = c.mapModule(splitM)\n", "\n", - "We first need to traverse the AST to every Statement and Expression. Then, when we see a DoPrim, we need to add a new DefNode to the module's body and insert a reference to that DefNode in place of the DoPrim. The code below implements this (and preserves the Info token). Note that `Namespace` is a utility function located in [Namespace.scala](https://github.com/ucb-bar/firrtl/blob/master/src/main/scala/firrtl/Namespace.scala).\n", - "\n", - "```scala\n", - "object Splitter extends Pass {\n", - " def name = \"Splitter!\"\n", - " /** Run splitM on every module **/\n", - " def run(c: Circuit): Circuit = c.copy(modules = c.modules map(splitM(_)))\n", - "\n", - " /** Run splitS on the body of every module **/\n", - " def splitM(m: DefModule): DefModule = m map splitS(Namespace(m))\n", - "\n", - " /** Run splitE on all children Expressions.\n", - " * If stmts contain extra statements, return a Block containing them and \n", - " * the new statement; otherwise, return the new statement. */\n", - " def splitS(namespace: Namespace)(s: Statement): Statement = {\n", - " val block = mutable.ArrayBuffer[Statement]()\n", - " s match {\n", - " case s: HasInfo => \n", - " val newStmt = s map splitE(block, namespace, s.info)\n", - " block.length match {\n", - " case 0 => newStmt\n", - " case _ => Block(block.toSeq :+ newStmt)\n", - " }\n", - " case s => s map splitS(namespace)\n", - " }\n", - "\n", - " /** Run splitE on all children expressions.\n", - " * If e is a DoPrim, add a new DefNode to block and return reference to\n", - " * the DefNode; otherwise return e.*/\n", - " def splitE(block: mutable.ArrayBuffer[Statement], namespace: Namespace, \n", - " info: Info)(e: Expression): Expression = e map splitE(block, namespace, info) match {\n", - " case e: DoPrim =>\n", - " val newName = namespace.newTemp\n", - " block += DefNode(info, newName, e)\n", - " Ref(newName, e.tpe)\n", - " case _ => e\n", + " /** Run splitS on the body of the module, using a fresh name generator */\n", + " def splitM(m: DefModule): DefModule = {\n", + " var counter = 0\n", + " // Skip the names that the module already uses\n", + " val used = serialize(m).flatMap(\"[A-Za-z_][A-Za-z0-9_]*\".r.findAllIn(_)).toSet\n", + " def newName(): String = {\n", + " while (used.contains(s\"_GEN_$counter\")) counter += 1\n", + " val n = s\"_GEN_$counter\"; counter += 1; n\n", + " }\n", + " m.copy(body = splitBody(m.body, newName))\n", " }\n", + "\n", + " def splitBody(body: Seq[Statement], newName: () => String): Seq[Statement] = body.flatMap { s =>\n", + " // Statements nested in a `when` are handled recursively\n", + " val s1 = s match {\n", + " case w: Conditionally => w.copy(conseq = splitBody(w.conseq, newName), alt = splitBody(w.alt, newName))\n", + " case other => other\n", + " }\n", + " val newNodes = scala.collection.mutable.ArrayBuffer[Statement]()\n", + " // The top expression of a node or a connect may stay a DoPrim: we split only the nested ones.\n", + " val s2 = s1.mapExpr(e => e.mapExpr(splitE(newNodes, newName, s1.info)))\n", + " newNodes.toSeq :+ s2\n", + " }\n", + "\n", + " /** Post-order: split the children first. If e is a DoPrim, add a new DefNode and return a reference to it. */\n", + " def splitE(nodes: scala.collection.mutable.ArrayBuffer[Statement], newName: () => String, info: String)(e: Expression): Expression =\n", + " e.mapExpr(splitE(nodes, newName, info)) match {\n", + " case p: DoPrim =>\n", + " val name = newName()\n", + " nodes += DefNode(info, name, p)\n", + " Reference(name)\n", + " case other => other\n", + " }\n", "}\n", - "```\n", + "\n", + "println(serialize(Splitter.run(parseCircuit(nested))))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ "### Deleting statements\n", - "Suppose we want to write a pass that inlined all DefNodes whose value is a literal, thus transforming this:\n", + "Suppose we want to write a pass that inlines all `DefNode`s whose value is a literal, thus transforming this:\n", "```\n", - "circuit Top:\n", - " module Top :\n", - " input x: UInt<3>\n", - " output o: UInt<4>\n", - " node y = UInt(1)\n", - " o <= add(x, y)\n", + " node y = UInt<3>(0h1)\n", + " connect o, add(x, y)\n", "```\n", "into this:\n", "```\n", - "circuit Top:\n", - " module Top :\n", - " input x: UInt<3>\n", - " output y: UInt<4>\n", - " o <= add(x, UInt(1))\n", + " connect o, add(x, UInt<3>(0h1))\n", "```\n", + "We first need to traverse the AST to every `Statement` and `Expression`. Then, when we see a `DefNode` pointing to a `Literal`, we store it into a hash map and delete the `DefNode` from the module. Then, whenever we see a reference to the deleted node, we insert the corresponding literal." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "object Inliner {\n", + " /** Run inlineM on every module */\n", + " def run(c: Circuit): Circuit = c.mapModule(inlineM)\n", "\n", - "We first need to traverse the AST to every Statement and Expression. Then, when we see a DefNode pointing to a Literal, we need to store it into a hashmap and return an EmptyStmt (thus deleting that DefNode). Then, whenever we see a reference to the deleted DefNode, we must insert the corresponding Literal.\n", - "\n", - "```scala\n", - "object Inliner extends Pass {\n", - " def name = \"Inliner!\"\n", - " /** Run inlineM on every module **/\n", - " def run(c: Circuit): Circuit = c.copy(modules = c.modules map(inlineM(_)))\n", - "\n", - " /** Run inlineS on the body of every module **/\n", - " def inlineM(m: DefModule): DefModule = m map inlineS(mutable.HashMap[String, Expression]())\n", - "\n", - " /** Run inlineE on all children Expressions, and then run inlineS on children statements.\n", - " * If statement is a DefNode containing a literal, update values and\n", - " * return EmptyStmt; otherwise return statement. */\n", - " def inlineS(values: mutable.HashMap[String, Expression])(s: Statement): Statement =\n", - " s map inlineE(values) map inlineS(values) match {\n", - " case d: DefNode => d.value match {\n", - " case l: Literal =>\n", - " values(d.name) = l\n", - " EmptyStmt\n", - " case _ => d\n", + " def inlineM(m: DefModule): DefModule = {\n", + " val values = scala.collection.mutable.HashMap[String, Expression]()\n", + " // We do not rewrite the text of the statements we did not parse (printf, assert...), so a node\n", + " // that they mention must stay\n", + " val mentioned = allStatements(m.body).collect { case o: OtherStmt => o.text }\n", + " .flatMap(\"[A-Za-z_][A-Za-z0-9_]*\".r.findAllIn(_)).toSet\n", + " m.copy(body = inlineBody(m.body, values, mentioned))\n", + " }\n", + "\n", + " /** Statements are processed in order, so a node is known before the statements that use it. A DefNode\n", + " * containing a literal is recorded and dropped (returned as an empty Seq), the others are kept. */\n", + " def inlineBody(body: Seq[Statement], values: scala.collection.mutable.HashMap[String, Expression],\n", + " mentioned: Set[String]): Seq[Statement] =\n", + " body.flatMap { s =>\n", + " s.mapExpr(inlineE(values)) match {\n", + " case DefNode(_, name, l: Literal) if !mentioned(name) =>\n", + " values(name) = l\n", + " Nil // deleted\n", + " case w: Conditionally =>\n", + " Seq(w.copy(conseq = inlineBody(w.conseq, values, mentioned), alt = inlineBody(w.alt, values, mentioned)))\n", + " case other => Seq(other)\n", " }\n", - " case o => o \n", " }\n", "\n", - " /** If e is a reference whose name is contained in values, \n", - " * return values(e.name); otherwise run inlineE on all \n", - " * children expressions.*/\n", - " def inlineE(values: mutable.HashMap[String, Expression])(e: Expression): Expression = e match {\n", - " case e: Ref if values.contains(e.name) => values(e.name)\n", - " case _ => e map inlineE(values)\n", + " /** If e is a reference whose name is contained in values, return values(name);\n", + " * otherwise run inlineE on all children expressions. */\n", + " def inlineE(values: scala.collection.mutable.HashMap[String, Expression])(e: Expression): Expression = e match {\n", + " case Reference(name) if values.contains(name) => values(name)\n", + " case _ => e.mapExpr(inlineE(values))\n", " }\n", "}\n", - "```\n", "\n", - "### Add a Primop\n", - "Would this be useful? Let [@azidar](https://github.com/azidar) know by submitting an issue to [the firrtl repo](https://github.com/freechipsproject/firrtl)!\n", + "val withLiteral = \"\"\"circuit Top :\n", + " module Top :\n", + " input x : UInt<3>\n", + " output o : UInt<4>\n", + " node y = UInt<3>(0h1)\n", + " connect o, add(x, y)\n", + "\"\"\"\n", + "println(serialize(Inliner.run(parseCircuit(withLiteral))))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The last two sections are, in the Scala FIRRTL compiler of earlier Chisel versions, the \"Add a primop\" and the \"Swap a statement\" idioms. With our small AST they are also simple: adding a primop is a new case in `parseExpr` (it already accepts any lower-case name), and swapping a statement is a `map` that returns a different statement.\n", "\n", - "### Swap a statement\n", - "Would this be useful? Let [@azidar](https://github.com/azidar) know by submitting an issue to [the firrtl repo](https://github.com/freechipsproject/firrtl)!\n" + "**Exercise: Find dead nodes**
\n", + "A `node` that is never read is dead code. `firtool` removes it anyway, but in a generator it is often a sign of a bug. Write `unusedNodes`, which returns the names of all the `DefNode`s of a module whose names are never used in any expression of the module (including `when` conditions and the right-hand sides of connects)." ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], - "source": [] + "source": [ + "def unusedNodes(m: DefModule): Seq[String] = ???\n", + "\n", + "val deadCode = \"\"\"circuit Top :\n", + " module Top :\n", + " input x : UInt<3>\n", + " output o : UInt<4>\n", + " node a = add(x, x)\n", + " node b = not(x)\n", + " node c = and(a, a)\n", + " connect o, a\n", + "\"\"\"\n", + "println(unusedNodes(parseCircuit(deadCode).modules.head))\n", + "assert(unusedNodes(parseCircuit(deadCode).modules.head) == Seq(\"b\", \"c\"))\n", + "println(\"SUCCESS!!\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + "\n", + "
\n", + "
\n",
+    "def unusedNodes(m: DefModule): Seq[String] = {\n",
+    "  val stmts = allStatements(m.body)\n",
+    "  val used = stmts.flatMap(directExprs).flatMap(allExprs).collect { case Reference(name) => name }.toSet\n",
+    "  stmts.collect { case DefNode(_, name, _) if !used(name) => name }\n",
+    "}\n",
+    "
" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Notice that `c` is dead too, but it is `a` that is used by it as well as by the connect. A transitive version (remove `c`, and then see what else became dead) is a nice extension if you want to practice.\n", + "\n", + "That's it for this section! In the next section, we will write a complete analysis of a design." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# You're done!\n", + "\n", + "[Return to the top.](#top)" + ] } ], "metadata": { @@ -159,12 +767,11 @@ "codemirror_mode": "text/x-scala", "file_extension": ".scala", "mimetype": "text/x-scala", - "name": "scala211", + "name": "scala", "nbconvert_exporter": "script", - "pygments_lexer": "scala", - "version": "2.11.11" + "version": "2.13.18" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 1 } diff --git a/4.4_firrtl_add_ops_per_module.ipynb b/4.4_firrtl_add_ops_per_module.ipynb index acfbd0d..956311b 100644 --- a/4.4_firrtl_add_ops_per_module.ipynb +++ b/4.4_firrtl_add_ops_per_module.ipynb @@ -1,6 +1,7 @@ { "cells": [ { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -8,104 +9,386 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "# Module 4.4: A FIRRTL Transform Example\n", - "\n", "**Prev: [Common Pass Idioms](4.3_firrtl_common_idioms.ipynb)**
\n", "\n", - "This AnalyzeCircuit Transform walks a `firrtl.ir.Circuit`, and records the number of add ops it finds, per module.\n", + "This AnalyzeCircuit transform walks a `Circuit` (our AST from Modules 4.2 and 4.3), and records the number of add ops it finds, per module. Then we extend it to count the adders of a whole hierarchy.\n", "\n", "## Setup\n", "\n", - "Please run the following:" + "Please run the following. The second cell contains the IR, the parser, `serialize` and the traversal helpers that we wrote in Modules 4.2 and 4.3, once more, since notebooks do not share definitions. Nothing is new there." ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ - "val path = System.getProperty(\"user.dir\") + \"/source/load-ivy.sc\"\n", - "interp.load.module(ammonite.ops.Path(java.nio.file.FileSystems.getDefault().getPath(path)))" + "interp.load.module(os.pwd / \"source\" / \"load-ivy.sc\")" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ - "// Compiler Infrastructure\n", + "// The FIRRTL library of Modules 4.2: IR, parser, serializer and traversal helpers.\n", + "// Nothing new here: just run this cell.\n", + "// ---- Expressions ----\n", + "sealed trait Expression {\n", + " // Apply f to the child Expressions of this node (like firrtl's `map`).\n", + " def mapExpr(f: Expression => Expression): Expression = this match {\n", + " case DoPrim(op, args, consts) => DoPrim(op, args.map(f), consts)\n", + " case leaf => leaf\n", + " }\n", + "}\n", + "case class Reference(name: String) extends Expression // x, io.in[0], m0.io.out\n", + "case class Literal(tpe: String, value: String) extends Expression // UInt<3>(0h5)\n", + "case class DoPrim(op: String, args: Seq[Expression], consts: Seq[Int]) extends Expression // add(a, b), bits(a, 3, 0), mux(c, a, b)\n", + "\n", + "// ---- Statements ----\n", + "sealed trait Statement {\n", + " def info: String // source locator, e.g. \"cmd3.sc 6:19\" (\"\" if there is none)\n", + "\n", + " // Apply f to the Expressions directly contained in this statement.\n", + " def mapExpr(f: Expression => Expression): Statement = this match {\n", + " case s: DefRegister => s.copy(clock = f(s.clock))\n", + " case s: DefRegisterReset => s.copy(clock = f(s.clock), reset = f(s.reset), init = f(s.init))\n", + " case s: DefNode => s.copy(value = f(s.value))\n", + " case s: Connect => s.copy(loc = f(s.loc), expr = f(s.expr))\n", + " case s: Conditionally => s.copy(pred = f(s.pred))\n", + " case other => other\n", + " }\n", + " // Apply f to the child Statements of this statement.\n", + " def mapStmt(f: Statement => Statement): Statement = this match {\n", + " case s: Conditionally => s.copy(conseq = s.conseq.map(f), alt = s.alt.map(f))\n", + " case s: OtherStmt => s.copy(children = s.children.map(f))\n", + " case other => other\n", + " }\n", + "}\n", + "sealed trait IsRegister extends Statement { def name: String; def tpe: String }\n", + "case class DefRegister(info: String, name: String, tpe: String, clock: Expression) extends IsRegister\n", + "case class DefRegisterReset(info: String, name: String, tpe: String, clock: Expression,\n", + " reset: Expression, init: Expression) extends IsRegister\n", + "case class DefWire(info: String, name: String, tpe: String) extends Statement\n", + "case class DefNode(info: String, name: String, value: Expression) extends Statement\n", + "case class DefInstance(info: String, name: String, module: String) extends Statement\n", + "case class Connect(info: String, loc: Expression, expr: Expression) extends Statement\n", + "case class Conditionally(info: String, pred: Expression, conseq: Seq[Statement], alt: Seq[Statement]) extends Statement\n", + "// Everything we do not model in detail (printf, assert, mem, ...). Keeps the raw text.\n", + "case class OtherStmt(info: String, text: String, children: Seq[Statement]) extends Statement\n", + "\n", + "// ---- Modules and circuits ----\n", + "case class Port(info: String, direction: String, name: String, tpe: String)\n", + "case class DefModule(info: String, name: String, isPublic: Boolean, ports: Seq[Port], body: Seq[Statement]) {\n", + " def mapStmt(f: Statement => Statement): DefModule = copy(body = body.map(f))\n", + "}\n", + "case class Circuit(main: String, modules: Seq[DefModule], otherDecls: Seq[String] = Nil) {\n", + " def mapModule(f: DefModule => DefModule): Circuit = copy(modules = modules.map(f))\n", + "}\n", + "\n", + "// ---- Step 1: split into lines and recover the nesting from the indentation ----\n", + "case class Line(indent: Int, text: String, info: String)\n", + "case class Node(line: Line, kids: Seq[Node])\n", + "\n", + "val infoRegex = \"\"\"^(.*?)\\s*@\\[(.*)\\]\\s*$\"\"\".r\n", + "\n", + "def toLines(text: String): List[Line] =\n", + " text.linesIterator.filter(_.trim.nonEmpty).map { raw =>\n", + " val indent = raw.length - raw.dropWhile(_ == ' ').length\n", + " raw.trim match {\n", + " case infoRegex(code, info) => Line(indent, code, info)\n", + " case code => Line(indent, code, \"\")\n", + " }\n", + " }.toList\n", + "\n", + "// Parses the sibling lines at `indent`; a more indented line starts the children of the line before it.\n", + "def toTree(lines: List[Line], indent: Int): (Vector[Node], List[Line]) = {\n", + " var rest = lines\n", + " val siblings = Vector.newBuilder[Node]\n", + " while (rest.nonEmpty && rest.head.indent == indent) {\n", + " val line = rest.head\n", + " val (kids, remaining) =\n", + " if (rest.tail.nonEmpty && rest.tail.head.indent > indent) toTree(rest.tail, rest.tail.head.indent)\n", + " else (Vector.empty[Node], rest.tail)\n", + " siblings += Node(line, kids)\n", + " rest = remaining\n", + " }\n", + " (siblings.result(), rest)\n", + "}\n", + "\n", + "// ---- Step 2: expressions ----\n", + "// Split on commas that are not nested inside (), [], {} or <>.\n", + "def splitTop(s: String): List[String] = {\n", + " var depth = 0\n", + " val parts = scala.collection.mutable.ListBuffer[String]()\n", + " val cur = new StringBuilder\n", + " for (c <- s) c match {\n", + " case '(' | '[' | '{' | '<' => depth += 1; cur += c\n", + " case ')' | ']' | '}' | '>' => depth -= 1; cur += c\n", + " case ',' if depth == 0 => parts += cur.toString.trim; cur.clear()\n", + " case _ => cur += c\n", + " }\n", + " if (cur.nonEmpty) parts += cur.toString.trim\n", + " parts.toList\n", + "}\n", "\n", - "// Firrtl IR classes\n", + "val literalRegex = \"\"\"^((?:UInt|SInt)(?:<\\d+>)?)\\((.*)\\)$\"\"\".r\n", + "val primRegex = \"\"\"^([A-Za-z0-9_]+)\\((.*)\\)$\"\"\".r\n", + "val intRegex = \"\"\"^-?\\d+$\"\"\".r\n", "\n", - "// Map functions\n", + "def parseExpr(text: String): Expression = text.trim match {\n", + " case literalRegex(tpe, value) => Literal(tpe, value)\n", + " case primRegex(op, args) =>\n", + " val (consts, exprs) = splitTop(args).partition(a => intRegex.matches(a))\n", + " DoPrim(op, exprs.map(parseExpr), consts.map(_.toInt))\n", + " case ref => Reference(ref)\n", + "}\n", + "\n", + "// ---- Step 3: statements, ports, modules ----\n", + "val portRegex = \"\"\"^(input|output) (\\S+) : (.*)$\"\"\".r\n", + "val regRegex = \"\"\"^reg (\\S+) : (.*)$\"\"\".r\n", + "val regResetRegex = \"\"\"^regreset (\\S+) : (.*)$\"\"\".r\n", + "val wireRegex = \"\"\"^wire (\\S+) : (.*)$\"\"\".r\n", + "val nodeRegex = \"\"\"^node (\\S+) = (.*)$\"\"\".r\n", + "val instRegex = \"\"\"^inst (\\S+) of (\\S+)$\"\"\".r\n", + "val connectRegex = \"\"\"^connect (.*)$\"\"\".r\n", + "val whenRegex = \"\"\"^when (.*) :$\"\"\".r\n", + "val moduleRegex = \"\"\"^(public )?module (\\S+) :$\"\"\".r\n", + "\n", + "// An `else :` node belongs to the `when` right before it, so we convert siblings together.\n", + "def toStatements(nodes: Seq[Node]): Seq[Statement] =\n", + " nodes.foldLeft(Vector.empty[Statement]) {\n", + " case (done, Node(Line(_, \"else :\", _), kids)) =>\n", + " done.last match {\n", + " case w: Conditionally => done.init :+ w.copy(alt = toStatements(kids))\n", + " case _ => sys.error(\"`else` without `when`\")\n", + " }\n", + " case (done, node) => done :+ toStatement(node)\n", + " }\n", + "\n", + "def toStatement(node: Node): Statement = {\n", + " val Node(Line(_, text, info), kids) = node\n", + " text match {\n", + " case regResetRegex(name, rest) =>\n", + " val Seq(tpe, clock, reset, init) = splitTop(rest)\n", + " DefRegisterReset(info, name, tpe, parseExpr(clock), parseExpr(reset), parseExpr(init))\n", + " case regRegex(name, rest) =>\n", + " val Seq(tpe, clock) = splitTop(rest)\n", + " DefRegister(info, name, tpe, parseExpr(clock))\n", + " case wireRegex(name, tpe) => DefWire(info, name, tpe)\n", + " case nodeRegex(name, value) => DefNode(info, name, parseExpr(value))\n", + " case instRegex(name, module) => DefInstance(info, name, module)\n", + " case connectRegex(args) =>\n", + " val Seq(loc, expr) = splitTop(args)\n", + " Connect(info, parseExpr(loc), parseExpr(expr))\n", + " case whenRegex(pred) => Conditionally(info, parseExpr(pred), toStatements(kids), Nil)\n", + " case other => OtherStmt(info, other, toStatements(kids))\n", + " }\n", + "}\n", + "\n", + "def toModule(node: Node): DefModule = {\n", + " val Node(Line(_, text, info), kids) = node\n", + " val moduleRegex(pub, name) = text\n", + " val (portNodes, bodyNodes) = kids.partition(k => portRegex.matches(k.line.text))\n", + " val ports = portNodes.map { k =>\n", + " val portRegex(dir, pname, tpe) = k.line.text\n", + " Port(k.line.info, dir, pname, tpe)\n", + " }\n", + " DefModule(info, name, pub != null, ports, toStatements(bodyNodes))\n", + "}\n", + "\n", + "// ---- Step 4: the whole file ----\n", + "def parseCircuit(firrtl: String): Circuit = {\n", + " // Skip the \"FIRRTL version\" line and the multi-line annotation block `circuit Top :%[[ ... ]]`.\n", + " val lines = firrtl.linesIterator.dropWhile(!_.startsWith(\"circuit \")).toList\n", + " val header = lines.head\n", + " val afterAnnos = if (header.endsWith(\"%[[\")) lines.tail.dropWhile(_.trim != \"]]\").tail else lines.tail\n", + " val main = header.stripPrefix(\"circuit \").takeWhile(c => c != ' ' && c != ':')\n", + " val (children, _) = toTree(toLines(afterAnnos.mkString(\"\\n\")), 2)\n", + " // `layer` and `extmodule` declarations are not analyzed: we keep their text to serialize it back.\n", + " def rawLines(n: Node, depth: Int): Seq[String] =\n", + " ((\" \" * depth) + n.line.text + (if (n.line.info.isEmpty) \"\" else s\" @[${n.line.info}]\")) +: n.kids.flatMap(rawLines(_, depth + 1))\n", + " val (modules, others) = children.partition(n => moduleRegex.matches(n.line.text))\n", + " Circuit(main, modules.map(toModule), others.flatMap(rawLines(_, 1)))\n", + "}\n", + "\n", + "def serialize(e: Expression): String = e match {\n", + " case Reference(name) => name\n", + " case Literal(tpe, value) => s\"$tpe($value)\"\n", + " case DoPrim(op, args, consts) => (args.map(serialize) ++ consts).mkString(s\"$op(\", \", \", \")\")\n", + "}\n", + "\n", + "def serialize(s: Statement, indent: Int): Seq[String] = {\n", + " val pad = \" \" * indent\n", + " val loc = (i: String) => if (i.isEmpty) \"\" else s\" @[$i]\"\n", + " s match {\n", + " case DefRegister(i, n, t, c) => Seq(s\"${pad}reg $n : $t, ${serialize(c)}${loc(i)}\")\n", + " case DefRegisterReset(i, n, t, c, r, v) => Seq(s\"${pad}regreset $n : $t, ${serialize(c)}, ${serialize(r)}, ${serialize(v)}${loc(i)}\")\n", + " case DefWire(i, n, t) => Seq(s\"${pad}wire $n : $t${loc(i)}\")\n", + " case DefNode(i, n, v) => Seq(s\"${pad}node $n = ${serialize(v)}${loc(i)}\")\n", + " case DefInstance(i, n, m) => Seq(s\"${pad}inst $n of $m${loc(i)}\")\n", + " case Connect(i, l, r) => Seq(s\"${pad}connect ${serialize(l)}, ${serialize(r)}${loc(i)}\")\n", + " case Conditionally(i, p, c, a) =>\n", + " Seq(s\"${pad}when ${serialize(p)} :${loc(i)}\") ++ c.flatMap(serialize(_, indent + 1)) ++\n", + " (if (a.isEmpty) Nil else Seq(s\"${pad}else :\") ++ a.flatMap(serialize(_, indent + 1)))\n", + " case OtherStmt(i, t, kids) => Seq(s\"$pad$t${loc(i)}\") ++ kids.flatMap(serialize(_, indent + 1))\n", + " }\n", + "}\n", + "\n", + "def serialize(m: DefModule): Seq[String] = {\n", + " val loc = (i: String) => if (i.isEmpty) \"\" else s\" @[$i]\"\n", + " Seq(s\" ${if (m.isPublic) \"public \" else \"\"}module ${m.name} :${loc(m.info)}\") ++\n", + " m.ports.map(p => s\" ${p.direction} ${p.name} : ${p.tpe}${loc(p.info)}\") ++ Seq(\"\") ++\n", + " m.body.flatMap(serialize(_, 2)) ++ Seq(\"\")\n", + "}\n", + "\n", + "def serialize(c: Circuit): String =\n", + " (Seq(s\"circuit ${c.main} :\") ++ c.otherDecls ++ c.modules.flatMap(serialize)).mkString(\"\\n\")\n", + "\n", + "// Pretty-prints any of our case classes as an indented tree: one node per line.\n", + "def showTree(x: Any, depth: Int = 0, label: String = \"\"): String = {\n", + " val pad = \" \" * depth + (if (label.isEmpty) \"\" else label + \": \")\n", + " def isNode(v: Any): Boolean = v match {\n", + " case _: Expression | _: Statement | _: Port | _: DefModule | _: Circuit => true\n", + " case _ => false\n", + " }\n", + " x match {\n", + " case p: Product if isNode(p) =>\n", + " val fields = p.productElementNames.toList.zip(p.productIterator.toList)\n", + " val (kids, atoms) = fields.partition {\n", + " case (_, v: Seq[_]) => v.nonEmpty && v.forall(isNode)\n", + " case (_, v) => isNode(v)\n", + " }\n", + " val head = pad + p.productPrefix + atoms.filter(_._2 != \"\").map { case (n, v) => s\" $n=$v\" }.mkString\n", + " (head +: kids.flatMap {\n", + " case (n, v: Seq[_]) => v.map(showTree(_, depth + 1, n))\n", + " case (n, v) => Seq(showTree(v, depth + 1, n))\n", + " }).mkString(\"\\n\")\n", + " case other => pad + other.toString\n", + " }\n", + "}\n", "\n", + "// All statements of a body, including the ones nested inside `when`s (pre-order).\n", + "def allStatements(body: Seq[Statement]): Seq[Statement] = body.flatMap {\n", + " case w: Conditionally => w +: (allStatements(w.conseq) ++ allStatements(w.alt))\n", + " case o: OtherStmt => o +: allStatements(o.children)\n", + " case s => Seq(s)\n", + "}\n", + "\n", + "// The Expressions that appear directly in a statement (not the sub-expressions).\n", + "def directExprs(s: Statement): Seq[Expression] = s match {\n", + " case r: DefRegister => Seq(r.clock)\n", + " case r: DefRegisterReset => Seq(r.clock, r.reset, r.init)\n", + " case n: DefNode => Seq(n.value)\n", + " case c: Connect => Seq(c.loc, c.expr)\n", + " case w: Conditionally => Seq(w.pred)\n", + " case _ => Nil\n", + "}\n", + "\n", + "// An Expression and all its sub-expressions (pre-order).\n", + "def allExprs(e: Expression): Seq[Expression] = e match {\n", + " case DoPrim(_, args, _) => e +: args.flatMap(allExprs)\n", + " case leaf => Seq(e)\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ "// Scala's mutable collections\n", - "import scala.collection.mutable\n", - "\n" + "import scala.collection.mutable" ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "## Counting Adders Per Module\n", "\n", - "As described, earlier, a Firrtl circuit is represented using a tree representation:\n", - " - A Firrtl `Circuit` contains a sequence of `DefModule`s.\n", - " - A `DefModule` contains a sequence of `Port`s, and maybe a `Statement`.\n", + "As described earlier, a FIRRTL circuit is represented using a tree representation:\n", + " - A FIRRTL `Circuit` contains a sequence of `DefModule`s.\n", + " - A `DefModule` contains a sequence of `Port`s, and a sequence of `Statement`s.\n", " - A `Statement` can contain other `Statement`s, or `Expression`s.\n", - " - A `Expression` can contain other `Expression`s.\n", + " - An `Expression` can contain other `Expression`s.\n", "\n", - "To visit all Firrtl IR nodes in a circuit, we write functions that recursively walk down this tree. To record statistics, we will pass along a `Ledger` class and use it when we come across an add op:" + "To visit all FIRRTL IR nodes in a circuit, we write functions that recursively walk down this tree. To record statistics, we will pass along a `Ledger` class and use it when we come across an add op:" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ "class Ledger {\n", - " import firrtl.Utils\n", " private var moduleName: Option[String] = None\n", - " private val modules = mutable.Set[String]()\n", + " private val modules = mutable.LinkedHashSet[String]()\n", " private val moduleAddMap = mutable.Map[String, Int]()\n", " def foundAdd(): Unit = moduleName match {\n", " case None => sys.error(\"Module name not defined in Ledger!\")\n", " case Some(name) => moduleAddMap(name) = moduleAddMap.getOrElse(name, 0) + 1\n", " }\n", " def getModuleName: String = moduleName match {\n", - " case None => Utils.error(\"Module name not defined in Ledger!\")\n", + " case None => sys.error(\"Module name not defined in Ledger!\")\n", " case Some(name) => name\n", " }\n", " def setModuleName(myName: String): Unit = {\n", " modules += myName\n", " moduleName = Some(myName)\n", " }\n", + " def addCount(name: String): Int = moduleAddMap.getOrElse(name, 0)\n", " def serialize: String = {\n", " modules map { myName =>\n", - " s\"$myName => ${moduleAddMap.getOrElse(myName, 0)} add ops!\"\n", + " s\"$myName => ${addCount(myName)} add ops!\"\n", " } mkString \"\\n\"\n", " }\n", "}" ] }, { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### The transform framework\n", + "In the Scala FIRRTL compiler, an analysis or a transformation was a class that extended `firrtl.Transform`, got a `CircuitState` and returned a (possibly modified) one. The compiler ran the transforms that it was given between its parser and its emitter. That compiler is gone, but the idea is so simple that we can recreate it in a few lines for our AST: a `Transform` is just a function from `Circuit` to `Circuit`, and `compile` parses the text, runs the transforms in order and serializes the result." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "trait Transform {\n", + " def name: String = getClass.getSimpleName\n", + " def execute(c: Circuit): Circuit\n", + "}\n", + "\n", + "def compileFIRRTL(firrtl: String, transforms: Seq[Transform]): String = {\n", + " val parsed = parseCircuit(firrtl)\n", + " val result = transforms.foldLeft(parsed) { (circuit, t) => t.execute(circuit) }\n", + " serialize(result)\n", + "}" + ] + }, + { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "Now, let's define a FIRRTL Transform that walks the circuit and updates our `Ledger` whenever it comes across an adder (`DoPrim` with op argument `Add`). Don't worry about `inputForm` or `outputForm` for now.\n", + "Now, let's define a Transform that walks the circuit and updates our `Ledger` whenever it comes across an adder (a `DoPrim` with op `add`).\n", "\n", "Take some time to understand how `walkModule`, `walkStatement`, and `walkExpression` enable traversing all `DefModule`, `Statement`, and `Expression` nodes in the FIRRTL AST.\n", "\n", @@ -118,29 +401,13 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ - "class AnalyzeCircuit extends firrtl.Transform {\n", - " import firrtl._\n", - " import firrtl.ir._\n", - " import firrtl.Mappers._\n", - " import firrtl.Parser._\n", - " import firrtl.annotations._\n", - " import firrtl.PrimOps._\n", - " \n", - " // Requires the [[Circuit]] form to be \"low\"\n", - " def inputForm = LowForm\n", - " // Indicates the output [[Circuit]] form to be \"low\"\n", - " def outputForm = LowForm\n", - "\n", - " // Called by [[Compiler]] to run your pass. [[CircuitState]] contains\n", - " // the circuit and its form, as well as other related data.\n", - " def execute(state: CircuitState): CircuitState = {\n", + "class AnalyzeCircuit extends Transform {\n", + " // Called by `compileFIRRTL` to run your pass. It returns the circuit, here unchanged.\n", + " def execute(circuit: Circuit): Circuit = {\n", " val ledger = new Ledger()\n", - " val circuit = state.circuit\n", "\n", " // Execute the function walkModule(ledger) on every [[DefModule]] in\n", " // circuit, returning a new [[Circuit]] with new [[Seq]] of [[DefModule]].\n", @@ -149,13 +416,13 @@ " // - \"infix notation\" - fancy function calling syntax\n", " // - \"map\" - classic functional programming concept\n", " // - discard the returned new [[Circuit]] because circuit is unmodified\n", - " circuit map walkModule(ledger)\n", + " circuit mapModule walkModule(ledger)\n", "\n", " // Print our ledger\n", " println(ledger.serialize)\n", "\n", - " // Return an unchanged [[CircuitState]]\n", - " state\n", + " // Return an unchanged circuit\n", + " circuit\n", " }\n", "\n", " // Deeply visits every [[Statement]] in m.\n", @@ -164,47 +431,40 @@ " ledger.setModuleName(m.name)\n", "\n", " // Execute the function walkStatement(ledger) on every [[Statement]] in m.\n", - " // - return the new [[DefModule]] (in this case, its identical to m)\n", - " // - if m does not contain [[Statement]], map returns m.\n", - " m map walkStatement(ledger)\n", + " // - return the new [[DefModule]] (unmodified)\n", + " m mapStmt walkStatement(ledger)\n", " }\n", "\n", " // Deeply visits every [[Statement]] and [[Expression]] in s.\n", " def walkStatement(ledger: Ledger)(s: Statement): Statement = {\n", - "\n", " // Execute the function walkExpression(ledger) on every [[Expression]] in s.\n", - " // - discard the new [[Statement]] (in this case, its identical to s)\n", - " // - if s does not contain [[Expression]], map returns s.\n", - " s map walkExpression(ledger)\n", + " // - discard the new [[Statement]] (unmodified)\n", + " s mapExpr walkExpression(ledger)\n", "\n", " // Execute the function walkStatement(ledger) on every [[Statement]] in s.\n", - " // - return the new [[Statement]] (in this case, its identical to s)\n", - " // - if s does not contain [[Statement]], map returns s.\n", - " s map walkStatement(ledger)\n", + " // - return the new [[Statement]] (unmodified)\n", + " s mapStmt walkStatement(ledger)\n", " }\n", "\n", " // Deeply visits every [[Expression]] in e.\n", - " // - \"post-order traversal\" - handle e's children [[Expression]] before e\n", + " // - \"post-order traversal\" - handle e's children [[Expression]] before e itself\n", " def walkExpression(ledger: Ledger)(e: Expression): Expression = {\n", - "\n", " // Execute the function walkExpression(ledger) on every [[Expression]] in e.\n", - " // - return the new [[Expression]] (in this case, its identical to e)\n", - " // - if s does not contain [[Expression]], map returns e.\n", - " val visited = e map walkExpression(ledger)\n", + " // - return the new [[Expression]] (unmodified)\n", + " val visited = e mapExpr walkExpression(ledger)\n", "\n", " visited match {\n", - " // If e is an adder, increment our ledger and return e.\n", - " case DoPrim(Add, _, _, _) =>\n", - " ledger.foundAdd\n", - " e\n", - " // If e is not an adder, return e.\n", - " case notadd => notadd\n", + " // If e is an adder, increment our ledger\n", + " case DoPrim(\"add\", _, _) => ledger.foundAdd()\n", + " case _ =>\n", " }\n", + " visited\n", " }\n", "}" ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -216,14 +476,11 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ - "// Chisel stuff\n", "import chisel3._\n", - "import chisel3.Input // Technicality: avoid a conflict with _root_.almond.input.Input\n", + "import chisel3.Input // avoid a name clash with almond.input.Input\n", "import chisel3.util._" ] }, @@ -243,60 +500,185 @@ ] }, { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, let's elaborate it into FIRRTL text. (As we have seen, this is the CHIRRTL that Chisel emits; the rest of the flow is `firtool`'s job.)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "lazy val firrtlSerialization = getFirrtl(new AddMe(8, 4))" + ] + }, + { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "Next, let's elaborate it into FIRRTL AST syntax." + "Finally, let's run our FIRRTL text through `compileFIRRTL`, including our custom transform. Note that it prints out the number of add ops it found! There are 8 inputs, so we expect 7 adders (`+&` is a plain `add`, whereas `+` would be an `add` followed by a `tail`; both contain exactly one `add`)." ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ - "val firrtlSerialization = (new chisel3.stage.ChiselStage).emitChirrtl(new AddMe(8, 4))\n" + "println(compileFIRRTL(firrtlSerialization, Seq(new AnalyzeCircuit())))" ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "Finally, let's compile our FIRRTL into Verilog, but include our custom transform into the compilation. Note that it prints out the number of add ops it found!\n", + "The result is again FIRRTL text: our transform did not change the circuit, so you get your circuit back. We could now hand this text to `firtool` on the command line (`firtool --format=fir file.fir`) to continue the flow with the normal compiler. Chisel's own `ChiselStage` always starts from a Chisel generator, not from text, which is why `compileFIRRTL` in this notebook stops here.\n", "\n", - "**Note** (January 2021): The following line may be broken due to a [bug](https://github.com/freechipsproject/chisel-bootcamp/issues/129)." + "## Counting Across a Hierarchy\n", + "Real designs consist of several modules. The ledger reports the number of adders *in the definition* of each module. Let's try on the design `Top` from Module 4.1, which instantiates `AddMe(2, 4)` and `AddMe(3, 4)`:" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [ - "val verilog = compileFIRRTL(firrtlSerialization, new firrtl.VerilogCompiler(), Seq(new AnalyzeCircuit()))" + "class Top extends Module {\n", + " val io = IO(new Bundle {\n", + " val a = Input(UInt(4.W))\n", + " val b = Input(UInt(4.W))\n", + " val y = Output(UInt(5.W))\n", + " val z = Output(UInt(5.W))\n", + " })\n", + " val m0 = Module(new AddMe(2, 4))\n", + " val m1 = Module(new AddMe(3, 4))\n", + " m0.io.in(0) := io.a\n", + " m0.io.in(1) := io.b\n", + " m1.io.in(0) := io.a\n", + " m1.io.in(1) := io.b\n", + " m1.io.in(2) := m0.io.out\n", + " io.y := m0.io.out\n", + " io.z := m1.io.out\n", + "}\n", + "\n", + "lazy val topFirrtl = getFirrtl(new Top)\n", + "compileFIRRTL(topFirrtl, Seq(new AnalyzeCircuit()))\n", + "()" ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "The `compileFIRRTL` function is defined only in this tutorial - in a future section, we will describe how the process of inserting customTransforms.\n", + "Each module is reported separately: `Top` contains no adder itself, it only *instantiates* modules that do. If we want to know the total number of adders in the elaborated design, we have to follow the instances. Using the instance graph of Module 4.3, the total of a module is its own count plus the totals of the modules it instantiates, *once for every instance*. Two instances of the same module count twice.\n", "\n", - "That's it for this section!" + "For the direct counts we do not need the ledger: with the collection-style traversal from Module 4.2 they are a few lines. Here is a more general version, which counts *any* primitive operation:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def opCounts(m: DefModule): Map[String, Int] =\n", + " allStatements(m.body).flatMap(directExprs).flatMap(allExprs).collect { case DoPrim(op, _, _) => op }\n", + " .groupBy(identity).map { case (op, l) => op -> l.size }\n", + "\n", + "lazy val topCircuit = parseCircuit(topFirrtl)\n", + "for (m <- topCircuit.modules) println(s\"${m.name}: ${opCounts(m)}\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Exercise: Total number of ops**
\n", + "Write `totalOps`, which returns, for the module `name` of `circuit`, the number of occurrences of the primitive operation `op` in that module *and all modules below it*, counting each instance. You will need `DefInstance` statements (found with `allStatements`) and `opCounts`. For `Top`, we expect 1 + 2 = 3 adds in total." ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], - "source": [] + "source": [ + "def totalOps(circuit: Circuit, name: String, op: String): Int = ???\n", + "\n", + "assert(totalOps(topCircuit, \"AddMe\", \"add\") == 1)\n", + "assert(totalOps(topCircuit, \"AddMe_1\", \"add\") == 2)\n", + "assert(totalOps(topCircuit, \"Top\", \"add\") == 3)\n", + "\n", + "// Two instances of the same module count twice\n", + "class Twice extends Module {\n", + " val io = IO(new Bundle { val in = Input(UInt(4.W)); val out = Output(UInt(5.W)) })\n", + " val a = Module(new AddMe(2, 4))\n", + " val b = Module(new AddMe(2, 4))\n", + " a.io.in(0) := io.in; a.io.in(1) := io.in\n", + " b.io.in(0) := a.io.out; b.io.in(1) := io.in\n", + " io.out := b.io.out\n", + "}\n", + "lazy val twiceCircuit = parseCircuit(getFirrtl(new Twice))\n", + "println(twiceCircuit.modules.map(_.name))\n", + "assert(totalOps(twiceCircuit, \"Twice\", \"add\") == 2)\n", + "println(\"SUCCESS!!\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "
\n", + "\n", + "\n", + "
\n", + "
\n",
+    "def totalOps(circuit: Circuit, name: String, op: String): Int = {\n",
+    "  // An `extmodule` (BlackBox) is not parsed, so it is not found here: it has no operations to count\n",
+    "  circuit.modules.find(_.name == name).fold(0) { m =>\n",
+    "    val below = allStatements(m.body).collect { case DefInstance(_, _, module) => totalOps(circuit, module, op) }\n",
+    "    opCounts(m).getOrElse(op, 0) + below.sum\n",
+    "  }\n",
+    "}\n",
+    "
" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Both instances in `Twice` are of `AddMe(2, 4)`, which is a single module definition, so the module list shows only one `AddMe`, and the total counts it twice.\n", + "\n", + "### What next?\n", + "In Chisel 7, there is no place to plug such a transform into the compilation flow itself, because that flow is implemented by `firtool`. The practical options are:\n", + "* **Analyze** the FIRRTL text as we did here, e.g. in a build tool or a test that checks properties of the generated design (the number of registers, the absence of a certain operator, the hierarchy, ...).\n", + "* **Do it while elaborating**, in Chisel, with `chisel3.reflect.DataMirror` and the `Definition`/`Instance` APIs of `chisel3.experimental.hierarchy`; this can inspect and change the design before the FIRRTL is emitted.\n", + "* **Customize `firtool`** with its command line options (`firtoolOpts` of `ChiselStage.emitSystemVerilog`), and the annotations and layers that Chisel supports, to steer the compiler.\n", + "* To implement a new compiler pass for real, write it in C++ as a CIRCT pass.\n", + "\n", + "That's it for this section!" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# You're done!\n", + "\n", + "[Return to the top.](#top)" + ] } ], "metadata": { @@ -309,12 +691,11 @@ "codemirror_mode": "text/x-scala", "file_extension": ".scala", "mimetype": "text/x-scala", - "name": "scala211", + "name": "scala", "nbconvert_exporter": "script", - "pygments_lexer": "scala", - "version": "2.11.11" + "version": "2.13.18" } }, "nbformat": 4, - "nbformat_minor": 2 -} \ No newline at end of file + "nbformat_minor": 1 +} diff --git a/Dockerfile b/Dockerfile index 75c0080..4cab347 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,26 @@ # First stage : setup the system and environment FROM alpine:3.22 AS base -# Install OpenJDK 11 and runtime dependencies +# Install OpenJDK 17 and runtime dependencies. +# Chisel 7 simulates with ChiselSim, which drives Verilator, so a C++ toolchain +# (g++, make, musl-dev) and Perl (Verilator is a Perl script) are runtime dependencies. RUN \ apk add --no-cache \ - openjdk11-jre-headless \ + openjdk17-jre-headless \ graphviz \ python3 \ py3-pip \ bash \ libstdc++ \ + verilator \ + verilator-dev \ + g++ \ + make \ + musl-dev \ + perl \ && \ apk add --no-cache --virtual .build-deps \ gcc \ - g++ \ - musl-dev \ linux-headers \ python3-dev \ && \ @@ -28,18 +34,23 @@ RUN \ rm -rf /usr/lib/python3.12/site-packages/pip /usr/lib/python3.12/site-packages/setuptools && \ apk del .build-deps -ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk +ENV JAVA_HOME=/usr/lib/jvm/java-17-openjdk ENV PATH=$JAVA_HOME/bin:$PATH RUN adduser -D -s /bin/bash bootcamp -ENV SCALA_VERSION=2.12.10 -ENV ALMOND_VERSION=0.9.1 +ENV SCALA_VERSION=2.13.18 +ENV ALMOND_VERSION=0.14.5 ENV COURSIER_VERSION=2.1.24 +ENV CHISEL_VERSION=7.16.0 +# Must be the firtool version that Chisel $CHISEL_VERSION was published against +ENV FIRTOOL_VERSION=1.160.0 ENV JUPYTER_CONFIG_DIR=/jupyter/config ENV JUPYTER_DATA_DIR=/jupyter/data -COPY . /chisel-bootcamp/ +# Only the Jupyter configuration is needed here; the notebooks are copied last (final stage) +# so that editing them does not invalidate the cached dependency downloads. +COPY source/custom.js source/jupyter_server_config.py source/overrides.json /chisel-bootcamp/source/ WORKDIR /chisel-bootcamp RUN mkdir -p $JUPYTER_CONFIG_DIR/custom && \ @@ -48,35 +59,22 @@ RUN mkdir -p $JUPYTER_CONFIG_DIR/custom && \ cp source/jupyter_server_config.py $JUPYTER_CONFIG_DIR/ && \ cp source/overrides.json $JUPYTER_CONFIG_DIR/lab/user-settings/@jupyterlab/apputils-extension/notification.jupyterlab-settings +# Ammonite and the build stage use this preloaded cache +ENV COURSIER_CACHE=/coursier_cache + # Second stage - download Scala requirements and the Scala kernel FROM base AS intermediate-builder ARG TARGETARCH +# Download everything into the cache that the final image uses at runtime RUN mkdir /coursier_cache -# Install glibc for ARM64 coursier binary (x86-64 uses static musl build) -RUN \ - if [ "$TARGETARCH" = "arm64" ]; then \ - apk add --no-cache wget gcompat libstdc++ && \ - wget -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub && \ - wget -q https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.35-r1/glibc-2.35-r1.apk && \ - apk --no-cache --force-overwrite add glibc-2.35-r1.apk && \ - rm glibc-2.35-r1.apk && \ - ln -s /lib/ld-musl-aarch64.so.1 /lib/ld-linux-aarch64.so.1 2>/dev/null || true; \ - fi - +# The JVM launcher of coursier works on every architecture. RUN \ - apk add --no-cache curl && \ - case "$TARGETARCH" in \ - amd64|"") \ - CS_URL="https://github.com/coursier/coursier/releases/download/v${COURSIER_VERSION}/cs-x86_64-pc-linux-static.gz" ;; \ - arm64) \ - CS_URL="https://github.com/VirtusLab/coursier-m1/releases/download/v${COURSIER_VERSION}/cs-aarch64-pc-linux.gz" ;; \ - *) \ - echo "Unsupported architecture: $TARGETARCH" && exit 1 ;; \ - esac && \ - curl -fL --retry 3 "$CS_URL" | gzip -d > coursier && \ + apk add --no-cache curl unzip && \ + curl -fL --retry 3 -o coursier \ + "https://github.com/coursier/coursier/releases/download/v${COURSIER_VERSION}/coursier" && \ chmod +x coursier && \ ./coursier \ bootstrap \ @@ -85,34 +83,59 @@ RUN \ --default=true \ -o almond && \ ./almond --install --global && \ - # Preload Chisel dependencies - coursier fetch downloads to its default cache - # which gets picked up by Ammonite at runtime - ./coursier fetch \ + # Preload Chisel dependencies into the coursier cache, which Ammonite uses at runtime + # (Ammonite also looks for the source jars, so fetch them too) + ./coursier fetch --sources --default=true \ --intransitive \ - edu.berkeley.cs:chisel3-plugin_2.12.10:3.6.1 \ + org.chipsalliance:chisel-plugin_$SCALA_VERSION:$CHISEL_VERSION \ && \ - ./coursier fetch \ - edu.berkeley.cs:chisel3_2.12:3.6.1 \ - edu.berkeley.cs:chisel-iotesters_2.12:2.5.6 \ - edu.berkeley.cs:chiseltest_2.12:0.6.1 \ - edu.berkeley.cs:dsptools_2.12:1.5.6 \ - edu.berkeley.cs:rocket-dsptools_2.12:1.2.0 \ - org.scalanlp:breeze_2.12:1.0 \ - org.scalatest:scalatest_2.12:3.2.2 \ + ./coursier fetch --sources --default=true \ + org.chipsalliance:chisel_2.13:$CHISEL_VERSION \ + org.scalatest:scalatest_2.13:3.2.19 \ && \ rm -rf almond coursier +# Chisel 7 emits SystemVerilog by running CIRCT's `firtool`, a self-contained static x86-64 +# executable (Maven Central ships it inside the llvm-firtool jar). Install it once here so +# the bootcamp works offline. There is no native Linux/Arm64 build of firtool: on Arm64 it +# runs through qemu-user (the rest of the toolchain, including Verilator, is native). +RUN \ + mkdir -p /opt/firtool/bin && \ + curl -fL --retry 3 -o firtool.jar \ + "https://repo1.maven.org/maven2/org/chipsalliance/llvm-firtool/${FIRTOOL_VERSION}/llvm-firtool-${FIRTOOL_VERSION}-linux-x64.jar" && \ + unzip -p firtool.jar org.chipsalliance/llvm-firtool/linux-x64/bin/firtool > /opt/firtool/bin/firtool.x86_64 && \ + rm firtool.jar && \ + chmod +x /opt/firtool/bin/firtool.x86_64 && \ + if [ "$TARGETARCH" = "arm64" ]; then \ + apk add --no-cache qemu-x86_64 && \ + printf '#!/bin/sh\nexec qemu-x86_64 /opt/firtool/bin/firtool.x86_64 "$@"\n' > /opt/firtool/bin/firtool && \ + chmod +x /opt/firtool/bin/firtool; \ + else \ + mv /opt/firtool/bin/firtool.x86_64 /opt/firtool/bin/firtool; \ + fi + # Last stage FROM base AS final -# Set COURSIER_CACHE environment variable so Ammonite uses our preloaded cache -ENV COURSIER_CACHE=/coursier_cache +ARG TARGETARCH + +# On Arm64 firtool (an x86-64 executable) runs through qemu-user, see above +RUN if [ "$TARGETARCH" = "arm64" ]; then apk add --no-cache qemu-x86_64; fi + + +# Tell Chisel where to find the preinstalled firtool instead of downloading one +ENV CHISEL_FIRTOOL_PATH=/opt/firtool/bin # copy the Scala requirements and kernel into the image COPY --from=intermediate-builder --chown=bootcamp:bootcamp /coursier_cache/ /coursier_cache/ COPY --from=intermediate-builder --chown=bootcamp:bootcamp /usr/local/share/jupyter/kernels/scala/ /usr/local/share/jupyter/kernels/scala/ +COPY --from=intermediate-builder /opt/firtool/ /opt/firtool/ + +COPY --chown=bootcamp:bootcamp . /chisel-bootcamp/ -RUN chown -R bootcamp:bootcamp /chisel-bootcamp /jupyter /coursier_cache +# The directory itself is root-owned (created by WORKDIR); ChiselSim writes ./build and Jupyter +# writes checkpoints there. +RUN chown -R bootcamp:bootcamp /jupyter && chown bootcamp:bootcamp /chisel-bootcamp USER bootcamp WORKDIR /chisel-bootcamp diff --git a/Install.md b/Install.md index 9616a4b..0889880 100644 --- a/Install.md +++ b/Install.md @@ -2,7 +2,7 @@ If you want to run the bootcamp locally, follow the instructions below for your platform. -Note: Java 11 or later is recommended. If you have multiple Java versions installed, select the appropriate version before running `jupyter notebook`: +Note: Java 17 or later is recommended (Java 11 also works). If you have multiple Java versions installed, select the appropriate version before running `jupyter notebook`: * On Windows: https://gist.github.com/rwunsch/d157d5fe09e9f7cdc858cec58c8462d6 * On macOS: https://stackoverflow.com/questions/21964709/how-to-set-or-change-the-default-java-jdk-version-on-os-x @@ -14,9 +14,10 @@ Make sure you have Docker [installed](https://docs.docker.com/get-docker/) on yo #### Prebuilt Multi-Architecture Image (Recommended) The `sysprog21/chisel-bootcamp` image supports both x86-64 and Arm64 architectures. It bundles: -- Ubuntu 24.04 -- [Eclipse Temurin](https://adoptium.net/temurin) 8 -- Scala 2.12.10 with Almond 0.9.1 kernel +- Alpine Linux 3.22 with OpenJDK 17 +- Scala 2.13.18 with Almond 0.14.5 kernel +- Chisel 7.16.0 and CIRCT firtool 1.160.0 (preinstalled, so the image also works offline) +- Verilator and a C++ toolchain, used by ChiselSim to simulate designs - Jupyter Lab - Graphviz for circuit visualization @@ -54,7 +55,7 @@ For this bootcamp, we'll install jupyter first and then the Scala-specific jupyt #### Jupyter First install Jupyter. -Dependencies: openssh-client, openjdk-11-jre, openjdk-11-jdk (-headless OK for both), ca-certificates-java +Dependencies: curl, openssh-client, openjdk-17-jre (-headless OK), ca-certificates-java, Verilator 5 or newer, a C++ compiler (g++ or clang++) and make Install Jupyter using pip3 (see [Jupyter installation docs](https://jupyter.org/install)): ``` @@ -81,10 +82,23 @@ Download coursier and use it to install Almond (see [Almond installation docs](h ```bash curl -L -o coursier https://github.com/coursier/coursier/releases/latest/download/coursier chmod +x coursier -./coursier launch --use-bootstrap almond -- --install ``` -This installs Almond with compatible Scala 2.12 versions. You can delete the `coursier` file afterward if desired. +Chisel 7 needs Scala 2.13, so ask for that Scala version explicitly (the notebooks use Scala 2.13.18): +```bash +./coursier bootstrap -r jitpack sh.almond:scala-kernel_2.13.18:0.14.5 --default=true -o almond +./almond --install +``` +You can delete the `coursier` and `almond` files afterward if desired. + +#### Verilator + +Chisel simulates designs with ChiselSim ([documentation](https://www.chisel-lang.org)), which runs [Verilator](https://verilator.org) (version 5 or newer) and compiles the result with a C++ compiler. +For example, on Debian/Ubuntu: `sudo apt install verilator g++ make` (Ubuntu 24.04 or newer provides Verilator 5), on macOS: `brew install verilator`. +Check with `verilator --version`. + +Chisel also needs CIRCT's `firtool` to generate SystemVerilog. You don't have to install it: Chisel downloads the matching version on first use (this needs Internet access once). +You can also point the `CHISEL_FIRTOOL_PATH` environment variable at a directory that contains `firtool` 1.160.0. #### Visualizations @@ -95,7 +109,7 @@ Now clone the bootcamp repo and install the customization script. If you already have one, append this script to it. ``` -git clone https://github.com/freechipsproject/chisel-bootcamp.git +git clone https://github.com/sysprog21/chisel-bootcamp.git cd chisel-bootcamp mkdir -p ~/.jupyter/custom cp source/custom.js ~/.jupyter/custom/custom.js @@ -115,7 +129,7 @@ These instructions cover Windows 10 and later. Running the command prompt in Adm #### Install Java -Ensure Java 11 or later is installed. Test by typing `java -version` in a command prompt. If not found, install from [Adoptium](https://adoptium.net/temurin/releases/). +Ensure Java 17 or later is installed. Test by typing `java -version` in a command prompt. If not found, install from [Adoptium](https://adoptium.net/temurin/releases/). #### Install Jupyter Jupyter recommends using the Anaconda distribution, here is the @@ -133,15 +147,18 @@ Download the latest Coursier from [GitHub releases](https://github.com/coursier/ Navigate to your download folder and run: ```bash -java -jar coursier launch almond -- --install +java -jar coursier bootstrap -r jitpack sh.almond:scala-kernel_2.13.18:0.14.5 --default=true -o almond +java -jar almond --install ``` +Chisel's simulations need Verilator, which is not available natively on Windows. Install and use the bootcamp inside [WSL 2](https://learn.microsoft.com/windows/wsl/) (follow the Linux instructions there) or use the Docker image. + #### Visualizations [Graphviz](https://graphviz.org/download/) is required to show visualizations of Chisel modules, such as in the demo page. However, visualizations are optional as the other Chisel and Scala features will work without it. #### Install the chisel-bootcamp repo. -Download the [chisel-bootcamp](https://github.com/freechipsproject/chisel-bootcamp) as a zip file (or use a Windows git client) +Download the [chisel-bootcamp](https://github.com/sysprog21/chisel-bootcamp) as a zip file (or use a Windows git client) and unpack it in a directory you have access to. Ideally, you should put it in a path that has no spaces. diff --git a/README.md b/README.md index 3a51508..4a73c87 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,9 @@ It teaches you Scala along the way, and it frames the learning of Chisel on the - Why hardware designs are better expressed as generators, not instances - Basics and some advanced features of Scala, a modern programming language -- Basics and some advanced features of Chisel, a hardware description language embedded in Scala -- How to write unit tests for Chisel designs -- Basic introduction to some useful features in Chisel libraries, including [dsptools](https://github.com/ucb-bar/dsptools/) and [rocketchip](https://github.com/freechipsproject/rocket-chip). +- Basics and some advanced features of [Chisel 7](https://github.com/chipsalliance/chisel), a hardware description language embedded in Scala +- How to write unit tests for Chisel designs with ChiselSim and Verilator +- How Chisel designs are compiled to SystemVerilog with CIRCT's `firtool`, and how to read the FIRRTL in between ## Prerequisites @@ -24,7 +24,7 @@ It teaches you Scala along the way, and it frames the learning of Chisel on the ### Quick Start with Docker -The fastest way to get started is using the prebuilt Docker image that supports both x86-64 and Arm64: +The fastest way to get started is using the prebuilt Docker image that supports both x86-64 and Arm64 (there is no native Linux/Arm64 build of CIRCT's `firtool`, so on Arm64 it runs under qemu-user emulation; everything else is native): Using Docker: ```bash @@ -52,7 +52,8 @@ It teaches you enough to start writing Chisel, but many more Scala concepts are Chisel is introduced in *Module 2*, starting with a hardware example and breaking it down. The rest of *Module 2* covers combinational and sequential logic, as well as software and hardware control flow. *Module 3* teaches you how to write hardware generators in Chisel that take advantage of Scala's high-level programming language features. -By the end, you will be able to read and understand most of the [Chisel code base](https://github.com/freechipsproject/chisel3) and begin using [Rocket Chip](https://github.com/freechipsproject/rocket-chip). +*Module 4* looks under the hood: the FIRRTL that Chisel emits, how CIRCT's `firtool` turns it into SystemVerilog, and how to write small programs that analyze and transform FIRRTL. +By the end, you will be able to read and understand most of the [Chisel code base](https://github.com/chipsalliance/chisel) and begin using [Rocket Chip](https://github.com/chipsalliance/rocket-chip). This tutorial *does not* yet cover SBT, build systems, backend flows for FPGA or ASIC processes, or analog circuits. ## Motivation diff --git a/binder/apt.txt b/binder/apt.txt index 34b13eb..6b1eb63 100644 --- a/binder/apt.txt +++ b/binder/apt.txt @@ -1,2 +1,4 @@ -openjdk-8-jre-headless +openjdk-17-jre-headless graphviz +g++ +make diff --git a/binder/environment.yml b/binder/environment.yml new file mode 100644 index 0000000..1f2b5e3 --- /dev/null +++ b/binder/environment.yml @@ -0,0 +1,6 @@ +name: notebook +channels: + - conda-forge +dependencies: + # Ubuntu 22.04's apt package is Verilator 4, but ChiselSim requires 5. + - verilator >=5,<6 diff --git a/binder/postBuild b/binder/postBuild index 894e02f..a78ef14 100644 --- a/binder/postBuild +++ b/binder/postBuild @@ -1,24 +1,28 @@ #!/bin/bash +set -e -SCALA_VERSION=2.12.10 -ALMOND_VERSION=0.9.1 +SCALA_VERSION=2.13.18 +ALMOND_VERSION=0.14.5 +COURSIER_VERSION=2.1.24 -# Install coursier -curl -Lo coursier https://git.io/coursier-cli +# Chisel 7 simulates with ChiselSim, which needs Verilator 5 or newer +command -v verilator >/dev/null || { echo "ERROR: verilator not found" >&2; exit 1; } +VERILATOR=$(verilator --version) +echo "$VERILATOR" +[ "$(echo "$VERILATOR" | sed -E 's/^Verilator ([0-9]+).*/\1/')" -ge 5 ] || { echo "ERROR: Verilator 5 or newer is required" >&2; exit 1; } + +# Install coursier (the JVM launcher works on every platform) +curl -fLo coursier "https://github.com/coursier/coursier/releases/download/v${COURSIER_VERSION}/coursier" chmod +x coursier # Install almond ./coursier bootstrap \ -r jitpack \ - -i user -I user:sh.almond:scala-kernel-api_$SCALA_VERSION:$ALMOND_VERSION \ sh.almond:scala-kernel_$SCALA_VERSION:$ALMOND_VERSION \ - --sources --default=true \ + --default=true \ -o almond ./almond --install -# Install required Jupyter/JupyterLab extensions -jupyter labextension install @jupyterlab/plotly-extension - # Install custom Javascript for solutions mkdir -p ~/.jupyter/custom cp source/custom.js ~/.jupyter/custom/custom.js diff --git a/runtest.py b/runtest.py index 88dd4be..a3fff11 100755 --- a/runtest.py +++ b/runtest.py @@ -1,60 +1,100 @@ #!/usr/bin/env python3 +"""Run the bootcamp notebooks in the Scala (Almond) kernel and check their errors. + +Many cells are exercises (they contain `???`) or demonstrate an error on purpose, so a +notebook is not expected to run without errors. Each notebook below lists the errors that +are expected, in order; the run fails if the notebook reports anything else. + +Usage: runtest.py [notebook_name.ipynb] [notebook_name_2.ipynb] [...] +""" import itertools +import json import os -import sys +import queue import subprocess -import tempfile +import sys +import time -from typing import Any, Dict, List +from typing import Dict, List, Tuple -import nbformat +from jupyter_client.manager import start_new_kernel +ROOT = os.path.dirname(os.path.abspath(__file__)) +# The first cell downloads and compiles dependencies, each `simulate` runs Verilator. +CELL_TIMEOUT = int(os.environ.get("BOOTCAMP_CELL_TIMEOUT", "600")) -def _notebook_run(path): - """Execute a notebook via nbconvert and collect output. - :returns (parsed nb object, execution errors) - """ - dirname, __ = os.path.split(path) - if len(dirname) > 0: - os.chdir(dirname) - with tempfile.NamedTemporaryFile(suffix=".ipynb", mode='w+') as fout: - args = ["jupyter-nbconvert", "--to", "notebook", "--execute", - "--allow-errors", - "--ExecutePreprocessor.timeout=60", - "--output", fout.name, path] - subprocess.check_call(args) +NOT_IMPLEMENTED = "scala.NotImplementedError" +COMPILATION_FAILED = "Compilation Failed" - fout.seek(0) - nb = nbformat.read(fout, nbformat.current_nbformat) - errors = [output for cell in nb.cells if "outputs" in cell - for output in cell["outputs"] \ - if output.output_type == "error"] +def _notebook_run(path: str) -> List[Tuple[int, str]]: + """Execute every code cell of a notebook in a fresh kernel. - return nb, errors + :returns the errors as a list of (cell index, ": "). + """ + with open(path, encoding="utf-8") as f: + nb = json.load(f) + + # The notebooks load `source/load-ivy.sc` relative to the working directory. + os.chdir(ROOT) + # The kernel logs the stack trace of every error to its stderr; they are checked below. + km, kc = start_new_kernel( + kernel_name="scala", startup_timeout=300, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + errors: List[Tuple[int, str]] = [] + try: + for index, cell in enumerate(nb["cells"]): + if cell["cell_type"] != "code": + continue + source = "".join(cell["source"]) + if not source.strip(): + continue + msg_id = kc.execute(source) + timed_out = False + deadline = time.monotonic() + CELL_TIMEOUT # for the whole cell, however chatty it is + while True: + try: + msg = kc.get_iopub_msg(timeout=max(0.0, deadline - time.monotonic())) + except queue.Empty: + errors.append((index, "Timeout: cell did not finish")) + km.interrupt_kernel() + timed_out = True + break + if msg["parent_header"].get("msg_id") != msg_id: + continue + if msg["msg_type"] == "error": + content = msg["content"] + errors.append((index, "{}: {}".format(content["ename"], content["evalue"]))) + elif msg["msg_type"] == "status" and msg["content"]["execution_state"] == "idle": + break + if timed_out: # the kernel state can no longer be trusted: don't wait for the next cells + break + finally: + kc.stop_channels() + km.shutdown_kernel(now=True) + return errors -def check_errors(file_name, expected: List[str], actual: List[Any]) -> bool: - """When errors occur they are due to a mismatch in the errors that occurred at runtime - and the expected error that are defined in notebooks - This produces huge output but the relevant information is between bars of '=' at the end. - Look at what was expected and either fix the error or add the text of the error that occurred - :note Errors shown may have hidden escape sequence for colors etc. When pasting be careful about this. +def check_errors(file_name: str, expected: List[str], actual: List[Tuple[int, str]]) -> bool: + """When errors occur they are due to a mismatch between the errors that occurred at + runtime and the errors that are expected, which are defined below. + Look at what was expected and either fix the notebook or add the text of the error that + occurred. :param file_name: name of file where errors occurred :param expected: errors that were expected - :param actual: errors that occurred. - :return: + :param actual: errors that occurred: (cell index, error). + :return: True if the actual errors are exactly the expected ones """ - actual_tracebacks: List[str] = list(map(lambda x: str(x['traceback'][0][:100]), actual)) - return_value = True - for i, (e, a) in enumerate(itertools.zip_longest(expected, actual_tracebacks, fillvalue="-- No Error --")): - if e not in a: + for i, (e, a) in enumerate(itertools.zip_longest(expected, actual, fillvalue=None)): + if e is None or a is None or e not in a[1]: if return_value: print("=" * 100, file=sys.stderr) print(f"Errors detected in {file_name}", file=sys.stderr) - print(f"No match for {i}-th expected error '{e}' got '{a[:100]}'", file=sys.stderr) + got = "-- No Error --" if a is None else f"cell {a[0]}: {a[1][:100]!r}" + print(f"No match for {i}-th expected error {e if e else '-- No Error --'!r}, got {got}", + file=sys.stderr) return_value = False if not return_value: print("=" * 100, file=sys.stderr) @@ -62,58 +102,52 @@ def check_errors(file_name, expected: List[str], actual: List[Any]) -> bool: notebooks: Dict[str, List[str]] = { - # This is the list of pages to try and rum along with each pages expected errors. - + # This is the list of pages to try and run along with each page's expected errors. + # (a substring of ": " per error, in order of occurrence) + "0_demo.ipynb": [], "1_intro_to_scala.ipynb": [], - "2.1_first_module.ipynb": ["chisel3.internal.ChiselException: Exception thrown when elaborating ChiselGeneratorAnnotation"], - "2.2_comb_logic.ipynb": ['Compilation Failed'] + - ['chisel3.internal.ChiselException: Exception thrown when elaborating ChiselGeneratorAnnotation'] + - ['chisel3.internal.ChiselException: Exception thrown when elaborating ChiselGeneratorAnnotation'] + - ['chisel3.internal.ChiselException: Exception thrown when elaborating ChiselGeneratorAnnotation'], - "2.3_control_flow.ipynb": ['scala.NotImplementedError'] * 2 + - ['Compilation Failed'] + - ['scala.NotImplementedError'] + - ['Compilation Failed'], - "2.4_sequential_logic.ipynb": ['chisel3.internal.ChiselException: Exception thrown when elaborating ChiselGeneratorAnnotation'] * 2, - "2.5_exercise.ipynb": ['chisel3.internal.ChiselException: Exception thrown when elaborating ChiselGeneratorAnnotation'] * 3 + - ['Compilation Failed'], - "2.6_chiseltest.ipynb": [], - "3.1_parameters.ipynb": ['java.util.NoSuchElementException'], - "3.2_collections.ipynb": [ - 'chisel3.internal.ChiselException: Exception thrown when elaborating ChiselGeneratorAnnotation'], + "2.1_first_module.ipynb": [NOT_IMPLEMENTED], + "2.2_comb_logic.ipynb": [COMPILATION_FAILED] + [NOT_IMPLEMENTED] * 3, + "2.3_control_flow.ipynb": [NOT_IMPLEMENTED] * 2 + [COMPILATION_FAILED] + [NOT_IMPLEMENTED] + [COMPILATION_FAILED], + "2.4_sequential_logic.ipynb": [NOT_IMPLEMENTED] * 2, + "2.5_exercise.ipynb": [NOT_IMPLEMENTED] * 3, + "2.6_chiselsim.ipynb": [], + "3.1_parameters.ipynb": ["java.util.NoSuchElementException"], + "3.2_collections.ipynb": [NOT_IMPLEMENTED], "3.2_interlude.ipynb": [], - "3.3_higher-order_functions.ipynb": ['scala.NotImplementedError'] + - ['java.lang.UnsupportedOperationException'] + - ['scala.NotImplementedError'] * 2 + - ['chisel3.internal.ChiselException: Exception thrown when elaborating ChiselGeneratorAnnotation'], - "3.4_functional_programming.ipynb": ['scala.NotImplementedError'] + - ['Compilation Failed'] + - ['scala.NotImplementedError'] + - ['Compilation Failed'], - "3.5_object_oriented_programming.ipynb": ['Compilation Failed'], - "3.6_types.ipynb": ['chisel3.internal.ChiselException: Connection between sink'] + - ['Failed to elaborate Chisel circuit'] + + "3.3_higher-order_functions.ipynb": [NOT_IMPLEMENTED] + + ["java.lang.UnsupportedOperationException"] + + [NOT_IMPLEMENTED] * 3, + "3.4_functional_programming.ipynb": [NOT_IMPLEMENTED, COMPILATION_FAILED, NOT_IMPLEMENTED, COMPILATION_FAILED], + "3.5_object_oriented_programming.ipynb": [], + "3.6_types.ipynb": ["ChiselException: Connection between sink"] + + ["I give up!"] + ['expected ")"'] + - ['scala.MatchError: ChiselExecutionFailure'] + - ['Compilation Failed'] * 5, - "4.1_firrtl_ast.ipynb": [], - "4.2_firrtl_ast_traversal.ipynb": [], - "4.3_firrtl_common_idioms.ipynb": [], - "4.4_firrtl_add_ops_per_module.ipynb": ['FirrtlInternalException'], # bug 129 + [NOT_IMPLEMENTED] * 2 + + [COMPILATION_FAILED] * 4, + "4.1_firrtl_ast.ipynb": [NOT_IMPLEMENTED], + "4.2_firrtl_ast_traversal.ipynb": [NOT_IMPLEMENTED], + "4.3_firrtl_common_idioms.ipynb": [NOT_IMPLEMENTED], + "4.4_firrtl_add_ops_per_module.ipynb": [NOT_IMPLEMENTED], } if __name__ == "__main__": - notebooks_to_run: List[str] = [] if len(sys.argv) > 1: - if sys.argv[1] == "--help": - print("Usage: {} [notebook_name.ipynb] [notebook_name_2.ipynb] [...]".format(sys.argv[0])) - print("By default, check all notebooks if notebooks are not specified.") + if sys.argv[1] in ("--help", "-h"): + print(__doc__) sys.exit(0) - else: - notebooks_to_run = sys.argv[1:] + notebooks_to_run = sys.argv[1:] else: notebooks_to_run = sorted(notebooks) # all notebooks + failed = False + notebooks_to_run = [os.path.basename(n) for n in notebooks_to_run] + unknown = [n for n in notebooks_to_run if n not in notebooks] + if unknown: + print("Unknown notebook(s): " + ", ".join(unknown), file=sys.stderr) + sys.exit(2) for n in notebooks_to_run: - expected = notebooks[n] - nb, errors = _notebook_run(n) - assert check_errors(n, expected, errors) + print(f"Running {n} ...", flush=True) + actual = _notebook_run(os.path.join(ROOT, n)) + if not check_errors(n, notebooks[n], actual): + failed = True + sys.exit(1 if failed else 0) diff --git a/source/load-ivy.sc b/source/load-ivy.sc index d60795c..0a73c93 100644 --- a/source/load-ivy.sc +++ b/source/load-ivy.sc @@ -1,324 +1,256 @@ -interp.repositories() ::: List( - coursierapi.MavenRepository.of("https://oss.sonatype.org/content/repositories/snapshots") -) +// Chisel 7 requires Scala 2.13 (the Almond kernel is 2.13.18) and the compiler plugin. +// The plugin must be loaded first, with its full Scala version (":::"). +import $plugin.$ivy.`org.chipsalliance:::chisel-plugin:7.16.0` -@ - -// Chisel 3.6+ requires the compiler plugin -import $plugin.$ivy.`edu.berkeley.cs:::chisel3-plugin:3.6.+` - -interp.configureCompiler(x => x.settings.source.value = scala.tools.nsc.settings.ScalaVersion("2.12.10")) +// * Anonymous `new Bundle { ... }` port lists use structural types, which Scala 2.13 +// only accepts with the reflectiveCalls language feature. +// * Chisel 7's dependencies put Scala 3 (TASTy) classes on the classpath; without +// -Ytasty-reader, e.g. `import scala.util.Random` fails with "Missing dependency ... TASTy". +interp.configureCompiler(_.settings.processArguments(List("-language:reflectiveCalls", "-Ytasty-reader"), processAll = true)) // Uncomment and change to use proxy // System.setProperty("https.proxyHost", "proxy.example.com") // System.setProperty("https.proxyPort", "3128") -import $ivy.`edu.berkeley.cs::chisel3:3.6.+` -import $ivy.`edu.berkeley.cs::chisel-iotesters:2.5.+` -import $ivy.`edu.berkeley.cs::chiseltest:0.6.+` -import $ivy.`edu.berkeley.cs::dsptools:1.5.+` -import $ivy.`org.scalanlp::breeze:1.0` -import $ivy.`edu.berkeley.cs::rocket-dsptools:1.2.0` - -// firrtl-diagrammer removed due to json4s compatibility issues with Chisel 3.6 -// Visualization functions below provide alternative output - -import $ivy.`org.scalatest::scalatest:3.2.2` - -// Convenience function to invoke Chisel and grab emitted Verilog. -// Note: emitVerilog has json4s compatibility issues, using FIRRTL as fallback -def getVerilog(dut: => chisel3.Module): String = { - import chisel3.stage.ChiselStage - try { - (new ChiselStage).emitVerilog(dut) - } catch { - case e: NoSuchMethodError if e.getMessage.contains("json4s") => - println("Warning: Verilog generation has json4s compatibility issues, using FIRRTL") - (new ChiselStage).emitChirrtl(dut) - case e: Exception => - throw e - } -} - -// Convenience function to invoke Chisel and grab emitted FIRRTL. -def getFirrtl(dut: => chisel3.Module): String = { - import chisel3.stage.ChiselStage - (new ChiselStage).emitChirrtl(dut) -} +import $ivy.`org.chipsalliance::chisel:7.16.0` +import $ivy.`org.scalatest::scalatest:3.2.19` -def compileFIRRTL( - inputFirrtl: String, - compiler: firrtl.Compiler, - customTransforms: Seq[firrtl.Transform] = Seq.empty, - infoMode: firrtl.Parser.InfoMode = firrtl.Parser.IgnoreInfo, - annotations: firrtl.AnnotationSeq = firrtl.AnnotationSeq(Seq.empty) -): String = { - import firrtl.{Compiler, AnnotationSeq, CircuitState, ChirrtlForm, FIRRTLException} - import firrtl.Parser._ - import scala.io.Source - import scala.util.control.ControlThrowable - import firrtl.passes._ - val outputBuffer = new java.io.CharArrayWriter - try { - //val parsedInput = firrtl.Parser.parse(Source.fromFile(input).getLines(), infoMode) - val parsedInput = firrtl.Parser.parse(inputFirrtl.split("\n").toIterator, infoMode) - compiler.compile( - CircuitState(parsedInput, ChirrtlForm, annotations), - outputBuffer, - customTransforms) - } +// Chisel 7 emits SystemVerilog through CIRCT's `firtool`. Chisel downloads a matching +// firtool automatically (the `llvm-firtool` artifact) on first use. Simulation with +// ChiselSim (chisel3.simulator) additionally needs Verilator 5.x and a C++ toolchain. +// Almond 0.14.5 cannot report an exception whose stack trace Ammonite has trimmed to "..." +// frames: the cell then never finishes. That happens to errors raised while Chisel +// elaborates a module (e.g. a `???` exercise inside `new MyModule`). `scrub` drops the +// (uninteresting) stack traces of such exceptions so that the error message is shown. +def scrub[T](body: => T): T = + try body catch { - // Rethrow the exceptions which are expected or due to the runtime environment (out of memory, stack overflow) - case p: ControlThrowable => throw p - case p: PassException => throw p - case p: FIRRTLException => throw p - // Treat remaining exceptions as internal errors. - case e: Exception => firrtl.Utils.throwInternalError(exception = Some(e)) + // not the fatal ones (as `scala.util.control.NonFatal`, which this file cannot import before + // `-Ytasty-reader` is set): `???` is a NotImplementedError, an `Error`, so it must be caught + case e: Throwable if !(e.isInstanceOf[VirtualMachineError] || e.isInstanceOf[LinkageError] || + e.isInstanceOf[InterruptedException]) => + var t: Throwable = e + while (t != null) { t.setStackTrace(Array.empty); t = t.getCause } + throw e } - val outputString = outputBuffer.toString - outputString -} +// Convenience function to invoke Chisel and grab emitted SystemVerilog. +def getVerilog(dut: => chisel3.RawModule): String = + scrub(circt.stage.ChiselStage.emitSystemVerilog(dut)) -def stringifyAST(firrtlAST: firrtl.ir.Circuit): String = { - var ntabs = 0 - val buf = new StringBuilder - val string = firrtlAST.toString - string.zipWithIndex.foreach { case (c, idx) => - c match { - case ' ' => - case '(' => - ntabs += 1 - buf ++= "(\n" + "| " * ntabs - case ')' => - ntabs -= 1 - buf ++= "\n" + "| " * ntabs + ")" - case ','=> buf ++= ",\n" + "| " * ntabs - case c if idx > 0 && string(idx-1)==')' => - buf ++= "\n" + "| " * ntabs + c - case c => buf += c +// Convenience function to invoke Chisel and grab emitted FIRRTL. +def getFirrtl(dut: => chisel3.RawModule): String = + scrub(circt.stage.ChiselStage.emitCHIRRTL(dut)) + +// ChiselSim's `EphemeralSimulator.simulate`: elaborates `module`, compiles it with Verilator and +// runs `body` against it. Two differences, so that the notebooks stay short: +// * errors go through `scrub`; +// * a `Module` is reset for one clock cycle before `body` runs, like chiseltest's `test` +// did. (ChiselSim itself leaves the reset to you: until it is asserted, registers hold +// random values.) +// The `poke`/`peek`/`expect` methods come from `import chisel3.simulator.PeekPokeAPI._`. +def simulate[T <: chisel3.RawModule](module: => T)(body: T => Unit): Unit = { + import chisel3.fromBooleanToLiteral + import chisel3.simulator.PeekPokeAPI._ + scrub(chisel3.simulator.EphemeralSimulator.simulate(module) { dut => + dut match { + case m: chisel3.Module => + m.reset.poke(true.B) + m.clock.step(1) + m.reset.poke(false.B) + case _ => } - } - buf.toString + body(dut) + }) } -// Visualization functions - graphical SVG generation using graphviz -def visualize(gen: () => chisel3.RawModule): Unit = { - import chisel3._ - import chisel3.stage.ChiselGeneratorAnnotation - import chisel3.stage.phases.{Elaborate, Convert} - import firrtl.stage.FirrtlCircuitAnnotation - import sys.process._ - import java.io.{File, PrintWriter} - import almond.interpreter.api.DisplayData - import almond.api.helpers.Display - - // Step 1: Elaborate and convert to FIRRTL - val elaboratePhase = new Elaborate - val elaborated = elaboratePhase.transform(Seq(ChiselGeneratorAnnotation(gen))) - - val convertPhase = new Convert - val converted = convertPhase.transform(elaborated) - - val firrtlCircuit = converted.collectFirst { - case FirrtlCircuitAnnotation(cir) => cir - }.get - - val firrtlString = firrtlCircuit.serialize - - // Helper function to sanitize node names for DOT format - def sanitizeName(name: String): String = { - // Replace array indices with underscores and remove other invalid chars - name.replaceAll("\\[([0-9]+)\\]", "_$1") - .replaceAll("[^a-zA-Z0-9_]", "_") +// Visualization: turn the FIRRTL emitted by Chisel into a Graphviz graph. +// FIRRTL (spec 4.x, as emitted by Chisel 7) is line-oriented text, so a few regular +// expressions are enough to recover ports, registers, wires, nodes and connections. +def firrtlToDot(firrtlString: String): String = { + val lines = firrtlString.split("\n").map(_.replaceAll("@\\[.*\\]", "")) + + def sanitize(name: String): String = name.replaceAll("\\[([A-Za-z0-9_.]+)\\]", "_$1").replaceAll("[^a-zA-Z0-9_]", "_") + // A (possibly sub-field / sub-index) reference, e.g. `io.in` or `zs[1]`. + // Names directly followed by `(` are primitive operations (`add(a, b)`), not signals. + val ident = "[A-Za-z_][A-Za-z0-9_]*(?:\\.[A-Za-z_][A-Za-z0-9_]*|\\[[A-Za-z_][A-Za-z0-9_.]*\\]|\\[[0-9]+\\])*(?![A-Za-z0-9_(])".r + val literal = "(?:UInt|SInt)(?:<[0-9]+>)?\\([^)]*\\)".r // e.g. UInt<1>(0h1) + val primOp = "^([a-z0-9_]+)\\(.*".r // e.g. add(a, b) + + val declPort = "^(input|output) ([A-Za-z0-9_]+) : (.*)".r + val declReg = "^(?:reg|regreset) ([A-Za-z0-9_]+) :.*".r + val declNode = "^node ([A-Za-z0-9_]+) = (.*)".r + val connect = "^connect ([^,]+), (.*)".r + + val inputs = scala.collection.mutable.LinkedHashSet[String]() + val outputs = scala.collection.mutable.LinkedHashSet[String]() + val regs = scala.collection.mutable.LinkedHashSet[String]() + val edges = scala.collection.mutable.LinkedHashSet[(String, String)]() + val opLabel = scala.collection.mutable.LinkedHashMap[String, String]() + val circuit = "^circuit ([A-Za-z0-9_]+) :.*".r + val module = "^(?:public )?(?:module|extmodule) ([A-Za-z0-9_]+) :.*".r + val top = lines.iterator.map(_.trim).collectFirst { case circuit(n) => n } + .getOrElse(throw new IllegalArgumentException("FIRRTL circuit name is missing")) + val controls = scala.collection.mutable.LinkedHashSet[(String, String)]() + val instance = "^inst ([A-Za-z0-9_]+) of ([A-Za-z0-9_]+).*".r + val moduleLines = scala.collection.mutable.LinkedHashMap[String, scala.collection.mutable.ArrayBuffer[String]]() + var currentModule: Option[String] = None + lines.foreach { raw => + raw.trim match { + case module(name) => + currentModule = Some(name) + moduleLines.getOrElseUpdate(name, scala.collection.mutable.ArrayBuffer[String]()) + case _ => currentModule.foreach(n => moduleLines(n) += raw) } + } - // Parse FIRRTL to extract structure - val lines = firrtlString.split("\n") - val moduleName = lines.find(_.trim.startsWith("module ")).map(_.trim.split(" ")(1).replace(":", "")).getOrElse("Module") - - // Collect all referenced nodes from connections (this includes array elements) - val referencedNodes = scala.collection.mutable.Set[String]() - lines.filter(_.contains("<=")).foreach { line => - val parts = line.trim.split("<=").map(_.trim) - if (parts.length == 2) { - val targetRaw = parts(0).split("\\.")(0).split("\\(")(0).trim - val sourceRaw = parts(1).split("\\.")(0).split("\\(")(0).split(" ")(0).trim - referencedNodes += sanitizeName(targetRaw) - referencedNodes += sanitizeName(sourceRaw) + // Bundle-typed ports look like `output io : { flip in : UInt<8>, bus : { out : UInt<8> } }`. + // A field is an input if an even number of `flip`s (its own and its parents') applies to it. + val portToken = "\\{|\\}|(flip )?([A-Za-z0-9_]+) :".r + def addPort(dir: String, name: String, tpe: String): Unit = { + def add(path: String, isInput: Boolean): Unit = (if (isInput) inputs else outputs) += sanitize(path) + if (!tpe.startsWith("{")) add(name, dir == "input") + else { + var scopes = List((name, dir == "input")) + var pending = Option.empty[(String, Boolean)] // the last field, until we know if it is a leaf + def flush(): Unit = { pending.foreach(p => add(p._1, p._2)); pending = None } + portToken.findAllMatchIn(tpe).foreach { m => + m.matched match { + case "{" => pending.foreach(p => scopes ::= p); pending = None + case "}" => flush(); scopes = scopes.tail + case _ => + flush() + val (path, isInput) = scopes.head + pending = Some((path + "_" + m.group(2), isInput ^ (m.group(1) != null))) + } } } + } - // Extract declared nodes (base names without array indices) - val inputs = lines.filter(_.trim.startsWith("input ")).map { l => - val name = l.trim.split(" ")(1).split(":")(0).trim - sanitizeName(name) - }.toSet - - val outputs = lines.filter(_.trim.startsWith("output ")).map { l => - val name = l.trim.split(" ")(1).split(":")(0).trim - sanitizeName(name) - }.toSet - - val regs = lines.filter(_.trim.startsWith("reg ")).map { l => - val name = l.trim.split(" ")(1).split(":")(0).trim - sanitizeName(name) - }.toSet - - val wires = lines.filter(_.trim.startsWith("wire ")).map { l => - val name = l.trim.split(" ")(1).split(":")(0).trim - sanitizeName(name) - }.toSet - - // Categorize all referenced nodes - val inputNodes = referencedNodes.filter { node => - inputs.contains(node) || inputs.exists(i => node.startsWith(i + "_")) - } - val outputNodes = referencedNodes.filter { node => - outputs.contains(node) || outputs.exists(o => node.startsWith(o + "_")) - } - val regNodes = referencedNodes.filter { node => - regs.contains(node) || regs.exists(r => node.startsWith(r + "_")) - } - val wireNodes = referencedNodes.filter { node => - wires.contains(node) || wires.exists(w => node.startsWith(w + "_")) - } -- inputNodes -- outputNodes -- regNodes - - // Generate GraphViz DOT - val dot = new StringBuilder - dot ++= "digraph circuit {\n" - dot ++= " rankdir=LR;\n" - dot ++= " node [shape=box, style=rounded];\n\n" - - // Input nodes - if (inputNodes.nonEmpty) { - dot ++= " subgraph cluster_inputs {\n" - dot ++= " label=\"Inputs\";\n" - dot ++= " style=filled; color=lightblue;\n" - inputNodes.foreach(i => dot ++= s" $i [shape=circle, fillcolor=lightgreen, style=filled];\n") - dot ++= " }\n\n" - } - - // Output nodes - if (outputNodes.nonEmpty) { - dot ++= " subgraph cluster_outputs {\n" - dot ++= " label=\"Outputs\";\n" - dot ++= " style=filled; color=lightblue;\n" - outputNodes.foreach(o => dot ++= s" $o [shape=doublecircle, fillcolor=lightcoral, style=filled];\n") - dot ++= " }\n\n" - } - - // Register nodes - if (regNodes.nonEmpty) { - dot ++= " subgraph cluster_regs {\n" - dot ++= " label=\"Registers\";\n" - dot ++= " style=filled; color=lightyellow;\n" - regNodes.foreach { r => - dot ++= s" $r [shape=box, fillcolor=yellow, style=filled];\n" + val dynamic = "\\[([A-Za-z_][A-Za-z0-9_.]*)\\]".r + val index = "\\[([A-Za-z_][A-Za-z0-9_.]*|[0-9]+)\\]".r + val fixedSelections = scala.collection.mutable.LinkedHashSet[(String, String)]() + val dynamicReads = scala.collection.mutable.LinkedHashSet[String]() + val dynamicWrites = scala.collection.mutable.LinkedHashSet[String]() + def visit(moduleName: String, prefix: String, ancestors: Set[String]): Unit = { + if (ancestors(moduleName)) throw new IllegalArgumentException("Recursive FIRRTL module: " + moduleName) + val body = moduleLines.getOrElse(moduleName, + throw new IllegalArgumentException("FIRRTL module definition is missing: " + moduleName)) + def scoped(name: String): String = sanitize(prefix + name) + def reference(ref: String, write: Boolean = false): String = { + val selected = scoped(ref) + index.findAllMatchIn(ref).foreach { m => + val base = scoped(ref.substring(0, m.start)) + if (m.group(1).forall(_.isDigit)) fixedSelections += (base -> selected) + else if (write) dynamicWrites += base + else dynamicReads += base } - dot ++= " }\n\n" - } - - // Wire nodes (internal signals) - if (wireNodes.nonEmpty) { - wireNodes.foreach { w => - dot ++= s" $w [shape=box, style=filled, fillcolor=lightgray];\n" + dynamic.findAllMatchIn(ref).foreach { m => + val base = ref.substring(0, m.start) + if (write) edges += (selected -> scoped(base)) + else edges += (scoped(base) -> selected) + edges += (scoped(m.group(1)) -> selected) } - dot ++= "\n" + selected } - - // Parse connections from FIRRTL - lines.filter(l => l.contains("<=") && !l.trim.startsWith("reset")).foreach { line => - val parts = line.trim.split("<=").map(_.trim) - if (parts.length == 2) { - // Extract base name, handling array indices and field accesses - val targetRaw = parts(0).split("\\.")(0).split("\\(")(0).trim - val sourceRaw = parts(1).split("\\.")(0).split("\\(")(0).split(" ")(0).trim - - // Sanitize array indices in both target and source - val target = sanitizeName(targetRaw) - val source = sanitizeName(sourceRaw) - - // Skip constants, temporaries, self-loops, and invalid nodes - if (!sourceRaw.startsWith("UInt") && !sourceRaw.contains("\"") && - !sourceRaw.startsWith("_") && source != target && - referencedNodes.contains(source) && referencedNodes.contains(target)) { - dot ++= s" $source -> $target;\n" + def sources(expr: String): Seq[String] = + ident.findAllIn(literal.replaceAllIn(expr, "")).filterNot(Set("clock", "reset")).map(ref => reference(ref)).toSeq + + var conditions = List.empty[(Int, Seq[String])] + var previousConditions = Map.empty[Int, Seq[String]] + // `expr` drives `target`, as does every enclosing `when` condition (the dashed edges) + def drive(expr: String, target: String): Unit = { + sources(expr).foreach(s => edges += (s -> target)) + conditions.flatMap(_._2).foreach(s => controls += (s -> target)) + } + body.foreach { raw => + val line = raw.trim + val indent = raw.takeWhile(_ == ' ').length + if (line.nonEmpty) { + while (conditions.nonEmpty && conditions.head._1 >= indent) { + val (level, condition) = conditions.head + previousConditions += level -> condition + conditions = conditions.tail + } + if (line.startsWith("when ") && line.endsWith(" :")) { + val condition = sources(line.stripPrefix("when ").stripSuffix(" :")) + conditions = (indent -> condition) :: conditions + previousConditions -= indent + } else if (line == "else :") { + conditions = (indent -> previousConditions.getOrElse(indent, Seq.empty)) :: conditions + } else line match { + case instance(n, child) => visit(child, prefix + n + ".", ancestors + moduleName) + case declPort(dir, n, tpe) => + if (prefix.isEmpty && n != "clock" && n != "reset") addPort(dir, n, tpe) + case declReg(n) => regs += scoped(n) + case declNode(n, expr) => + expr match { case primOp(op) => opLabel(scoped(n)) = op; case _ => } + drive(expr, scoped(n)) + case connect(dst, src) => + val target = reference(dst.trim, write = true) + drive(src, target) + case _ => } } } + } + visit(top, "", Set.empty) + fixedSelections.foreach { case (base, selected) => + if (dynamicWrites(base)) edges += (base -> selected) + if (dynamicReads(base)) edges += (selected -> base) + } - dot ++= "}\n" - - // Write DOT file and generate SVG - val dotFile = File.createTempFile("circuit", ".dot") - val svgFile = File.createTempFile("circuit", ".svg") - val pw = new PrintWriter(dotFile) - val dotContent = dot.toString - pw.write(dotContent) - pw.close() - - // Generate SVG using graphviz - val result = s"dot -Tsvg ${dotFile.getAbsolutePath} -o ${svgFile.getAbsolutePath}".! - - if (result != 0) { - // Print DOT content for debugging - println("=== Generated DOT (debug) ===") - println(dotContent) - println("=== End DOT ===") - } + // Sub-fields and indices (`zs[1]`) are sanitized into `zs_1`, so they are matched with + // their declaration by prefix. + def declaredAs(node: String, set: scala.collection.Set[String]): Boolean = + set.exists(d => node == d || node.startsWith(d + "_")) + + // Ports and registers are drawn even when only a constant drives them (so they have no edge) + val nodes = ((edges ++ controls).flatMap { case (a, b) => Seq(a, b) }.toSeq ++ inputs ++ outputs ++ regs).distinct + val dot = new StringBuilder + dot ++= s"digraph circuit {\n label=\"Module: $top\";\n rankdir=LR;\n node [shape=box, style=\"rounded,filled\", fillcolor=lightgray];\n" + nodes.foreach { n => + val attrs = + if (declaredAs(n, inputs)) "shape=circle, fillcolor=lightgreen" + else if (declaredAs(n, outputs)) "shape=doublecircle, fillcolor=lightcoral" + else if (declaredAs(n, regs)) "shape=box, fillcolor=gold" + else if (opLabel.contains(n)) s"""shape=ellipse, fillcolor=white, label="${opLabel(n)}${if (n.startsWith("_")) "" else "\\n" + n}"""" + else "shape=box, fillcolor=lightgray" + dot ++= s" $n [$attrs];\n" + } + def emit(es: Iterable[(String, String)], style: String): Unit = + es.foreach { case (a, b) => if (a != b) dot ++= s" $a -> $b$style;\n" } + emit(edges, "") + emit(controls, " [style=dashed, label=\"condition\"]") + dot ++= "}\n" + dot.toString +} - if (result == 0 && svgFile.exists()) { - val svgContent = scala.io.Source.fromFile(svgFile).mkString - Display.html(svgContent) - } else { - println("=== Module FIRRTL (Intermediate Representation) ===") +// Draw the circuit with Graphviz (`dot`). Falls back to printing the FIRRTL without Graphviz. +def visualize(gen: () => chisel3.RawModule): Unit = { + import almond.api.helpers.Display + + val firrtlString = getFirrtl(gen()) + val svg = + try Some(os.proc("dot", "-Tsvg").call(stdin = firrtlToDot(firrtlString), check = false)).filter(_.exitCode == 0).map(_.out.text()) + catch { case _: java.io.IOException => None } + svg.filter(_.nonEmpty) match { + case Some(html) => Display.html(html) + case None => + println("Graphviz (`dot`) is not available; showing the FIRRTL instead.") println(firrtlString) - println("\nNote: Graphviz not available. Install with: apt-get install graphviz") - } - - // Cleanup - dotFile.delete() - svgFile.delete() + } } def visualizeHierarchy(gen: () => chisel3.RawModule): Unit = { - import chisel3._ - import chisel3.stage.ChiselGeneratorAnnotation - import chisel3.stage.phases.{Elaborate, Convert} - import chisel3.stage.ChiselCircuitAnnotation - import firrtl.stage.FirrtlCircuitAnnotation - - println("=== Module Hierarchy ===") - - // Elaborate and convert - val elaboratePhase = new Elaborate - val elaborated = elaboratePhase.transform(Seq(ChiselGeneratorAnnotation(gen))) - - val convertPhase = new Convert - val converted = convertPhase.transform(elaborated) - - val firrtlCircuit = converted.collectFirst { - case FirrtlCircuitAnnotation(cir) => cir - }.get - - val firrtlString = firrtlCircuit.serialize - - // Print just the module hierarchy - val hierarchyLines = firrtlString.split("\n").filter(line => - line.trim.startsWith("module ") || - line.trim.startsWith("inst ") || - line.trim.matches("^\\s+inst .*") - ) - - if (hierarchyLines.nonEmpty) { - hierarchyLines.foreach(println) - } else { - println("(No submodule instances found)") - } - - println("\n=== Full FIRRTL ===") - println(firrtlString) + val firrtlString = getFirrtl(gen()) + println("=== Module Hierarchy ===") + val hierarchy = firrtlString.split("\n").filter { l => + val t = l.trim + t.startsWith("module ") || t.startsWith("public module ") || t.startsWith("inst ") + } + if (hierarchy.nonEmpty) hierarchy.foreach(println) else println("(No submodule instances found)") + println("\n=== Full FIRRTL ===") + println(firrtlString) } - diff --git a/tests/test_visualizer.py b/tests/test_visualizer.py new file mode 100644 index 0000000..9b63357 --- /dev/null +++ b/tests/test_visualizer.py @@ -0,0 +1,92 @@ +"""Run from the repository root: python3 tests/test_visualizer.py.""" +from jupyter_client.manager import start_new_kernel +km,kc=start_new_kernel(kernel_name='scala',startup_timeout=300) +def run(code): + mid=kc.execute(code) + while True: + m=kc.get_iopub_msg(timeout=180) + if m['parent_header'].get('msg_id') != mid: continue + if m['msg_type']=='stream': print(m['content']['text'], end='') + if m['msg_type']=='error': raise RuntimeError(m['content']) + if m['msg_type']=='status' and m['content']['execution_state']=='idle': break +try: + run('interp.load.module(os.pwd / "source" / "load-ivy.sc")') + run('''val graph = firrtlToDot("""circuit Top : + module Child : + input other : UInt<1> + connect io_out, other + public module Top : + input sel : UInt<1> + input nested : UInt<1> + input data : UInt<1> + output out : UInt<1> + output elseout : UInt<1> + when sel : + when nested : + connect out, data + else : + connect elseout, UInt<1>(0) +""") +assert(graph.contains("sel -> out [style=dashed")) +assert(graph.contains("nested -> out [style=dashed")) +assert(graph.contains("sel -> elseout [style=dashed")) +assert(graph.contains("data -> out")) +assert(!graph.contains("other")) +assert(graph.contains("Module: Top"))''') + run('''val constOnly = firrtlToDot("""circuit K : + public module K : + input clock : Clock + output k : UInt<1> + connect k, UInt<1>(1) +""") +assert(constOnly.contains("k [")) +assert(!constOnly.contains("clock ["))''') + run('''val hierarchy = firrtlToDot("""circuit Top : + module Child : + input in : UInt<8> + output out : UInt<8> + connect out, in + public module Top : + input data : UInt<8> + output result : UInt<8> + inst a of Child + inst b of Child + connect a.in, data + connect b.in, a.out + connect result, b.out +""") +assert(hierarchy.contains("data -> a_in")) +assert(hierarchy.contains("a_in -> a_out")) +assert(hierarchy.contains("a_out -> b_in")) +assert(hierarchy.contains("b_in -> b_out")) +assert(hierarchy.contains("b_out -> result"))''') + run('''val indexed = firrtlToDot("""circuit VecTest : + public module VecTest : + wire values : UInt<8>[4] + input index : UInt<2> + output result : UInt<8> + connect result, values[index] + wire storage : UInt<8>[4] + connect storage[index], result + connect result, storage[0] + connect values[1], result +""") +assert(indexed.contains("values -> values_index")) +assert(indexed.contains("index -> values_index")) +assert(indexed.contains("values_index -> result")) +assert(indexed.contains("result -> storage_index")) +assert(indexed.contains("index -> storage_index")) +assert(indexed.contains("storage_index -> storage"))''') + run('''assert(indexed.contains("storage -> storage_0")) +assert(indexed.contains("storage_0 -> result")) +assert(indexed.contains("values_1 -> values"))''') + run('''val nested = firrtlToDot("""circuit N : + public module N : + output io : { flip bus : { data : UInt<8> }, out : UInt<8> } + connect io.out, io.bus.data +""") +assert(nested.contains("io_bus_data [shape=circle")) +assert(nested.contains("io_out [shape=doublecircle"))''') + print('visualizer regression checks passed') +finally: + kc.stop_channels(); km.shutdown_kernel(now=True)