diff --git a/.gitignore b/.gitignore index 2ce44cf..2e38409 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,9 @@ .bender +.github modelsim/vsim test/BUILD **.log **.txt **__pycache** verif/python/generated/** -install \ No newline at end of file +install diff --git a/Bender.yml b/Bender.yml index a8d3a46..645c466 100644 --- a/Bender.yml +++ b/Bender.yml @@ -3,11 +3,12 @@ package: authors: - "Francesco Conti " - "Sergio Mazzola " + - "Cyrill Durrer " dependencies: hwpe-stream: { git: "https://github.com/pulp-platform/hwpe-stream.git", rev: 40ab0fe1433dc080a49aa1926ce09df9ab3f5fb5 } # branch: prasadar/multi-precision hci: { git: "https://github.com/pulp-platform/hci.git", rev: 9844a893c34612ac66697e8dcd60214296f2634c } # branch: prasadar/multi-precision - hwpe-ctrl: { git: "https://github.com/pulp-platform/hwpe-ctrl.git", rev: 4977b6cf42b96f3fefd1973281d5c4b8259c50d5 } # branch: master + hwpe-ctrl: { git: "https://github.com/pulp-platform/hwpe-ctrl.git", rev: 4977b6cf42b96f3fefd1973281d5c4b8259c50d5 } # version: 2.0 } sources: diff --git a/CONFIG_USAGE.md b/CONFIG_USAGE.md new file mode 100644 index 0000000..f5a191f --- /dev/null +++ b/CONFIG_USAGE.md @@ -0,0 +1,394 @@ +# Configuration Examples and Usage Guide + +NOTE: OUTDATED TESTING METHODOLOGY! +This document describes how to use the flexible configuration system for the datamover HWPE project. + +## 🚀 Quick Commands + +```bash +# Show configuration help +make help + +# Validate current config +make validate-config + +# Run with preset +make sim CONFIG_PRESET=small-tensor + +# Override specific parameter +make sim CONFIG_PRESET=transpose-test TRANSP_MODE=4 + +# Test all presets +make test-all-presets + +# Test transpose modes +make test-transpose-modes + +# Test configuration combinations (grid) +make test-transpose-grid +make test-cim-grid +``` + +## Quick Start + +### Using Configuration Presets + +Run simulations with predefined configurations: + +```bash +# Small tensor for quick testing +make sim CONFIG_PRESET=small-tensor + +# Medium tensor for moderate testing +make sim CONFIG_PRESET=medium-tensor + +# Large tensor for stress testing +make sim CONFIG_PRESET=large-tensor + +# Transpose-focused testing +make sim CONFIG_PRESET=transpose-test + +# Rectangular tensor testing +make sim CONFIG_PRESET=rect-wide +make sim CONFIG_PRESET=rect-tall +make sim CONFIG_PRESET=rect-narrow +make sim CONFIG_PRESET=rect-elongated + +# Copy mode testing +make sim CONFIG_PRESET=copy-small +make sim CONFIG_PRESET=copy-medium + +# CIM mode testing +make sim CONFIG_PRESET=cim-small +make sim CONFIG_PRESET=cim-medium +make sim CONFIG_PRESET=cim-large +``` + +### Command Line Overrides + +Override specific parameters while keeping preset base: + +```bash +# Use small-tensor preset but change transpose mode +make sim CONFIG_PRESET=small-tensor TRANSP_MODE=2 + +# Use medium-tensor preset but different element width +make sim CONFIG_PRESET=medium-tensor ELEM_WIDTH=16 + +# Override tensor dimensions +make sim CONFIG_PRESET=transpose-test TENSOR_SIZE_M=64 TENSOR_SIZE_N=64 +``` + +### Custom Configuration + +Use completely custom parameters: + +```bash +# Custom configuration via command line +make sim CONFIG_PRESET=custom BANDWIDTH=512 ELEM_WIDTH=16 TENSOR_SIZE_M=128 TENSOR_SIZE_N=128 + +# Or edit config.mk for persistent custom settings +make sim CONFIG_PRESET=custom +``` + +## Configuration Hierarchy + +Parameters are resolved in this order (highest priority first): + +1. **Command line arguments**: `make sim TRANSP_MODE=2` +2. **Preset definitions**: Values from config_presets.mk +3. **Default values**: Fallback values in config.mk + +## Available Presets + +| Preset | Description | Tensor Size | Memory | Mode | +|--------|-------------|-------------|--------|------| +| `small-tensor` | Quick testing | 4x4 | 2KB | Transpose | +| `medium-tensor` | Moderate testing | 64x64 | 16KB | Transpose | +| `large-tensor` | Stress testing | 448x448 | 512KB | Transpose | +| `transpose-test` | Transpose focus | 32x32 | 64KB | Transpose | +| `rect-wide` | Wide rectangle | 64x256 | 128KB | Transpose | +| `rect-tall` | Tall rectangle | 256x64 | 128KB | Transpose | +| `rect-narrow` | Narrow rectangle | 16x128 | 32KB | Transpose | +| `rect-elongated` | Elongated rectangle | 128x32 | 64KB | Transpose | +| `copy-small` | Copy mode testing | 4x4 | 2KB | Copy | +| `copy-medium` | Copy mode testing | 64x64 | 16KB | Copy | +| `cim-small` | CIM mode testing | 32x128 | 32KB | CIM | +| `cim-medium` | CIM mode testing | 64x256 | 64KB | CIM | +| `cim-large` | CIM mode testing | 128x256 | 256KB | CIM | +| `custom` | User-defined | Variable | Variable | Variable | + +## � Datamover Modes + +The datamover supports three main operation modes: + +### Copy Mode (DATAMOVER_MODE=0) +- **Purpose**: Direct memory-to-memory copy operations +- **Use case**: Basic data movement without transformation +- **Presets**: `copy-small`, `copy-medium` +- **Example**: `make sim CONFIG_PRESET=copy-small` + +### Transpose Mode (DATAMOVER_MODE=1) +- **Purpose**: Tensor transposition during data movement +- **Use case**: Data layout transformations for optimized access patterns +- **Transpose elements**: 1, 2, or 4 elements per cycle (`TRANSP_MODE`) +- **Presets**: `small-tensor`, `medium-tensor`, `large-tensor`, `transpose-test`, `rect-*` +- **Example**: `make sim CONFIG_PRESET=transpose-test TRANSP_MODE=2` + +### CIM Mode (DATAMOVER_MODE=2) +- **Purpose**: Compute-In-Memory data layout conversion +- **Use case**: Converting row-major data to CIM accelerator layouts +- **CIM layout**: Row-major -> CIM-layout (`CIM_MODE=0`) or CIM-layout -> row-major (`CIM_MODE=1`) +- **Dimensions**: Configurable `ROW_TILE_SIZE` +- **Presets**: `cim-small`, `cim-medium`, `cim-large` +- **Example**: `make sim CONFIG_PRESET=cim-medium` + +## Key Parameters + +| Parameter | Values | Description | +|-----------|---------|-------------| +| `DATAMOVER_MODE` | 0,1,2 | Operation mode (0=Copy, 1=Transpose, 2=CIM layout conversion, 3=CIM transpose, 4=unfold, 5=fold) | +| `TRANSP_MODE` | 0,1,2,4 | Transpose elements per cycle | +| `CIM_MODE` | 0,1 | CIM layout (0=row-major->CIM-Layout, 1=CIM-Layout->row_major) | +| `ROW_TILE_SIZE` | 32,64,... | Row tile size in elements | +| `ELEM_WIDTH` | 8 | Element width in bits | +| `BANDWIDTH` | 64,128,256,512,1024 | Memory bandwidth in bits | +| `TENSOR_SIZE_M` | Any | Tensor height in elements | +| `TENSOR_SIZE_N` | Any | Tensor width in elements | +| `MEMORY_SIZE` | Any | Available memory in words | +| `WORD_WIDTH` | 16,32,64 | Word width in bits (typically 32) | + +### Testbench Stimulus Parameters + +| Parameter | Description | +|-----------|-------------| +| `STIM_READ_BASE_ADDR` / `STIM_WRITE_BASE_ADDR` | Start address for read/write bursts (element-addressed) | +| `STIM_*_D0_LENGTH` / `STIM_*_D0_STRIDE` | Inner dimension length and stride (elements) | +| `STIM_*_D1_LENGTH` / `STIM_*_D1_STRIDE` | Second dimension length and stride | +| `STIM_*_D2_LENGTH` / `STIM_*_D2_STRIDE` | Optional third dimension length/stride (0 when unused) | +| `STIM_*_D3_LENGTH` / `STIM_*_D3_STRIDE` | Optional fourth dimension length/stride (0 when unused) | +| `STIM_*_D4_STRIDE` | Optional stride for a fifth dimension (0 when unused) | +| `STIM_*_TOT_LENGTH` | Total number of accesses for the transfer | +| `STIM_*_DIM_ENABLE` | 4-bit mask enabling address-generation dimensions (d1..d4) -> d0 always active | +| `STIM_MEM_SIZE` | Number of words in the testbench memory | +| `STIM_TRANSP_MODE` | Transpose mode encoded for the datamover: 1,2 or 4 elements | + +## Built-in Test Targets + +The system provides several built-in test targets for comprehensive testing: + +```bash +# Test all available presets with detailed reporting (continues through failures) +make test-all-presets + +# Test all transpose modes with transpose-test preset +make test-transpose-modes + +# Test configuration parameter combinations (bandwidth/transpose/word width grid) +make test-transpose-grid + +# Test CIM configuration parameter combinations (bandwidth/CIM dimensions/word width grid) +make test-cim-grid + +# Validate configuration without running simulation +make validate-config CONFIG_PRESET= +``` + +## Creating New Presets + +Add new presets to `config_presets.mk`: + +```makefile +ifeq ($(CONFIG_PRESET),my-test) + BANDWIDTH = 1024 + WORD_WIDTH = 32 + ELEM_WIDTH = 16 + MEMORY_SIZE = 32768 + TRANSP_MODE = 4 + TENSOR_SIZE_M = 64 + TENSOR_SIZE_N = 32 + CONFIG_DESC = "Custom test for specific use case" +endif +``` + +## Advanced Usage + +### Environment-based Configuration + +Set up your shell environment: + +```bash +# Set default preset for your session +export CONFIG_PRESET=transpose-test + +# Override specific parameters +export TRANSP_MODE=4 +export ELEM_WIDTH=16 + +# Run simulation with environment settings +make sim +``` + +### Makefile Integration + +The system includes the following built-in test targets: + +```makefile +# Built-in targets (already available) +test-all-presets: + # Tests all 8 configuration presets (stops on first failure) + $(MAKE) sim CONFIG_PRESET=small-tensor + $(MAKE) sim CONFIG_PRESET=medium-tensor + $(MAKE) sim CONFIG_PRESET=large-tensor + $(MAKE) sim CONFIG_PRESET=transpose-test + $(MAKE) sim CONFIG_PRESET=rect-wide + $(MAKE) sim CONFIG_PRESET=rect-tall + $(MAKE) sim CONFIG_PRESET=rect-narrow + $(MAKE) sim CONFIG_PRESET=rect-elongated + +test-all-presets: + # Tests all presets with detailed reporting (continues through failures) + # Provides summary of which tests passed/failed + +test-transpose-modes: + # Tests all transpose modes (0,1,2,4) + $(MAKE) sim CONFIG_PRESET=transpose-test TRANSP_MODE=0 + $(MAKE) sim CONFIG_PRESET=transpose-test TRANSP_MODE=1 + $(MAKE) sim CONFIG_PRESET=transpose-test TRANSP_MODE=2 + $(MAKE) sim CONFIG_PRESET=transpose-test TRANSP_MODE=4 +``` + +For additional custom test suites, you can create your own targets: + +```makefile +# Add to your Makefile for custom test suites +test-custom-suite: + @echo "Testing custom configuration suite..." + $(MAKE) sim CONFIG_PRESET=small-tensor TRANSP_MODE=2 + $(MAKE) sim CONFIG_PRESET=medium-tensor ELEM_WIDTH=16 + $(MAKE) sim CONFIG_PRESET=custom BANDWIDTH=1024 TENSOR_SIZE_M=64 TENSOR_SIZE_N=64 +``` + +### Configuration Validation + +The system includes automatic validation: + +- TRANSP_MODE must be 0, 1, 2, or 4 +- Tensor dimensions must fit in available memory +- Bandwidth and element width must be compatible + +### Debug Configuration + +To see all computed values and configuration info: + +```bash +# Show configuration details +make help + +# Validate configuration with detailed output +make validate-config CONFIG_PRESET=small-tensor + +# Enable debug output in config.mk (uncomment $(info) lines) +make sim CONFIG_PRESET=small-tensor | grep -E "(BANDWIDTH|TENSOR|STIM)" +``` + +## 💡 Usage Patterns + +```bash +# Development cycle +make sim CONFIG_PRESET=small-tensor # Quick check +make sim CONFIG_PRESET=transpose-test # Feature test +make sim CONFIG_PRESET=large-tensor # Stress test + +# Research/tuning +make sim CONFIG_PRESET=custom BANDWIDTH=1024 ELEM_WIDTH=32 + +# Validation and help +make validate-config CONFIG_PRESET=transpose-test +make help +``` + +## Tips and Best Practices + +1. **Start Small**: Use `small-tensor` for initial testing, then scale up +2. **Test Transpose**: Use `transpose-test` preset for transpose functionality +3. **Rectangular Testing**: Use rectangular presets (`rect-wide`, `rect-tall`, etc.) for non-square matrices +4. **Parameter Validation**: Always run `make validate-config` to check computed values +5. **Documentation**: Document custom presets with clear descriptions +6. **Memory Requirements**: Ensure tensor dimensions fit within available memory +7. **Bandwidth Alignment**: Both tensor dimensions should be ≥ BANDWIDTH/ELEM_WIDTH + +## Example Workflows + +### Development Workflow +```bash +# Quick functionality check +make sim CONFIG_PRESET=small-tensor + +# Detailed transpose testing +make sim CONFIG_PRESET=transpose-test + +# Stress testing with large matrices +make sim CONFIG_PRESET=large-tensor +``` + +### CI/Testing Workflow +```bash +# Comprehensive test suite with detailed reporting +make test-all-presets + +# Test specific functionality +make test-transpose-modes + +# Comprehensive parameter grid testing +make test-transpose-grid +make test-cim-grid +``` + +### Custom Research Configuration +```bash +# Specific research parameters +make sim CONFIG_PRESET=custom \ + BANDWIDTH=1024 \ + ELEM_WIDTH=32 \ + TENSOR_SIZE_M=256 \ + TENSOR_SIZE_N=128 \ + TRANSP_MODE=4 +``` + +## 🔍 System Files + +The configuration system consists of these key files: + +- **`config.mk`** - Main configuration with default values and computed parameters +- **`config_presets.mk`** - Preset definitions for common test scenarios +- **`Makefile`** - Enhanced targets including test-all-presets and validation +- **`verif/python/validate_config.py`** - Python validation script for parameter checking +- **`CONFIG_USAGE.md`** - This comprehensive documentation file + +## Troubleshooting + +### Common Issues + +1. **Invalid TRANSP_MODE**: Must be 0, 1, 2, or 4 + ```bash + make validate-config CONFIG_PRESET=my-preset # Check for errors + ``` + +2. **Memory insufficient**: Tensor too large for available memory + ```bash + # Reduce tensor size or increase MEMORY_SIZE + make sim CONFIG_PRESET=small-tensor # Use smaller preset + ``` + +3. **Bandwidth alignment warnings**: Tensor dimensions not aligned to bandwidth + ```bash + # Adjust tensor dimensions to be multiples of BANDWIDTH/ELEM_WIDTH + make sim TENSOR_SIZE_M=32 TENSOR_SIZE_N=32 # Use aligned dimensions + ``` + +4. **Configuration not taking effect**: Check parameter precedence + ```bash + # Command line overrides presets + make sim CONFIG_PRESET=small-tensor TRANSP_MODE=2 # Override works + ``` diff --git a/Makefile b/Makefile index e0e3f32..83b85db 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,9 @@ # Licensed under the Apache License, Version 2.0, see LICENSE for details. # SPDX-License-Identifier: Apache-2.0 +# Standalone testing setup OUTDATED! +# ToDo: Implement SW-based testing with a CPU core + include config.mk SHELL = /usr/bin/env bash @@ -19,7 +22,7 @@ SYNTH_PATH = synopsys BENDER_TARGETS = -t rtl -t test -t datamover_test -GUI ?= 1 +GUI ?= 0 target ?= sim_tb_datamover_top_wrap VLOG_FLAGS += -svinputport=compat @@ -38,26 +41,274 @@ TESTBENCH_DEFINES += -DSTIM_READ_D0_STRIDE=${STIM_READ_D0_STRIDE} TESTBENCH_DEFINES += -DSTIM_READ_D0_LENGTH=${STIM_READ_D0_LENGTH} TESTBENCH_DEFINES += -DSTIM_READ_D1_STRIDE=${STIM_READ_D1_STRIDE} TESTBENCH_DEFINES += -DSTIM_READ_D1_LENGTH=${STIM_READ_D1_LENGTH} +TESTBENCH_DEFINES += -DSTIM_READ_D2_STRIDE=${STIM_READ_D2_STRIDE} +TESTBENCH_DEFINES += -DSTIM_READ_D2_LENGTH=${STIM_READ_D2_LENGTH} +TESTBENCH_DEFINES += -DSTIM_READ_D3_STRIDE=${STIM_READ_D3_STRIDE} +TESTBENCH_DEFINES += -DSTIM_READ_D3_LENGTH=${STIM_READ_D3_LENGTH} +TESTBENCH_DEFINES += -DSTIM_READ_D4_STRIDE=${STIM_READ_D4_STRIDE} TESTBENCH_DEFINES += -DSTIM_READ_TOT_LENGTH=${STIM_READ_TOT_LENGTH} +TESTBENCH_DEFINES += -DSTIM_READ_DIM_ENABLE=${STIM_READ_DIM_ENABLE} TESTBENCH_DEFINES += -DSTIM_WRITE_BASE_ADDR=${STIM_WRITE_BASE_ADDR} TESTBENCH_DEFINES += -DSTIM_WRITE_D0_STRIDE=${STIM_WRITE_D0_STRIDE} TESTBENCH_DEFINES += -DSTIM_WRITE_D0_LENGTH=${STIM_WRITE_D0_LENGTH} TESTBENCH_DEFINES += -DSTIM_WRITE_D1_STRIDE=${STIM_WRITE_D1_STRIDE} TESTBENCH_DEFINES += -DSTIM_WRITE_D1_LENGTH=${STIM_WRITE_D1_LENGTH} +TESTBENCH_DEFINES += -DSTIM_WRITE_D2_STRIDE=${STIM_WRITE_D2_STRIDE} +TESTBENCH_DEFINES += -DSTIM_WRITE_D2_LENGTH=${STIM_WRITE_D2_LENGTH} +TESTBENCH_DEFINES += -DSTIM_WRITE_D3_STRIDE=${STIM_WRITE_D3_STRIDE} +TESTBENCH_DEFINES += -DSTIM_WRITE_D3_LENGTH=${STIM_WRITE_D3_LENGTH} +TESTBENCH_DEFINES += -DSTIM_WRITE_D4_STRIDE=${STIM_WRITE_D4_STRIDE} TESTBENCH_DEFINES += -DSTIM_WRITE_TOT_LENGTH=${STIM_WRITE_TOT_LENGTH} +TESTBENCH_DEFINES += -DSTIM_WRITE_DIM_ENABLE=${STIM_WRITE_DIM_ENABLE} TESTBENCH_DEFINES += -DSTIM_MEM_SIZE=${STIM_MEM_SIZE} TESTBENCH_DEFINES += -DSTIM_TRANSP_MODE=${STIM_TRANSP_MODE} +# TESTBENCH_DEFINES += -DSTIM_TRANSP_LEN=${STIM_TRANSP_LEN} + +TESTBENCH_DEFINES += -DSTIM_TENSOR_SIZE_M=${STIM_TENSOR_SIZE_M} +TESTBENCH_DEFINES += -DSTIM_TENSOR_SIZE_N=${STIM_TENSOR_SIZE_N} + +TESTBENCH_DEFINES += -DSTIM_NUM_CHANNELS=${STIM_NUM_CHANNELS} +TESTBENCH_DEFINES += -DSTIM_TOTAL_ELEMENTS=${STIM_TOTAL_ELEMENTS} TESTBENCH_DEFINES += -DBANDWIDTH=${BANDWIDTH} TESTBENCH_DEFINES += -DNUM_ELEM_WORD=${NUM_ELEM_WORD} TESTBENCH_DEFINES += -DELEM_WIDTH=${ELEM_WIDTH} +TESTBENCH_DEFINES += -DMISALIGNED_ACCESSES=${MISALIGNED_ACCESSES} # .PHONY: clean-sim sim-script sim synopsys-script -all: testvector sim +all: sim + +# Configuration help target +help: + @echo "==========================================" + @echo "Datamover Configuration System" + @echo "==========================================" + @echo "" + @echo "Available presets:" + @echo " small-tensor : 4x4 tensor (transpose)" + @echo " medium-tensor : 64x64 tensor (transpose)" + @echo " large-tensor : 448x448 tensor (transpose)" + @echo " transpose-test : 32x32 tensor (transpose)" + @echo " rect-wide : 64x256 wide rectangular tensor (transpose)" + @echo " rect-tall : 256x64 tall rectangular tensor (transpose)" + @echo " rect-narrow : 16x128 narrow rectangular tensor (transpose)" + @echo " rect-elongated : 128x32 elongated rectangular tensor (transpose)" + @echo " copy-small : 4x4 tensor (copy mode)" + @echo " copy-medium : 64x64 tensor (copy mode)" + @echo " cim-small : 32x128 tensor (CIM mode)" + @echo " cim-medium : 64x256 tensor (CIM mode)" + @echo " cim-large : 128x256 tensor (CIM mode)" + @echo " custom : User-defined (config.mk default)" + @echo "" + @echo "Usage examples:" + @echo " make sim CONFIG_PRESET=small-tensor" + @echo " make sim CONFIG_PRESET=transpose-test TRANSP_MODE=2" + @echo " make sim TENSOR_SIZE_M=64 TENSOR_SIZE_N=32" + @echo "" + @echo "Test targets:" + @echo " make test-all-presets : Test all presets (detailed reporting)" + @echo " make test-transpose-modes : Test all transpose modes" + @echo " make test-transpose-grid : Test all bandwidth/transpose/word width combinations" + @echo " make test-cim-grid : Test all bandwidth/CIM dimension/word width combinations" + @echo "" + @echo "For detailed documentation, see CONFIG_USAGE.md" + @echo "==========================================" + +# Test multiple configurations +test-all-presets: + @echo "Testing all configuration presets..." + @failed_tests=""; \ + for preset in small-tensor medium-tensor large-tensor transpose-test rect-wide rect-tall rect-narrow rect-elongated copy-small copy-medium cim-small cim-medium cim-large; do \ + echo "=== Testing CONFIG_PRESET=$$preset ==="; \ + if $(MAKE) sim CONFIG_PRESET=$$preset; then \ + echo "✓ $$preset: PASSED"; \ + else \ + echo "✗ $$preset: FAILED"; \ + failed_tests="$$failed_tests $$preset"; \ + fi; \ + done; \ + if [ -n "$$failed_tests" ]; then \ + echo ""; \ + echo "====== SUMMARY: The following presets FAILED:$$failed_tests ======"; \ + exit 1; \ + else \ + echo ""; \ + echo "====== SUMMARY: All presets PASSED! ======"; \ + fi + +test-transpose-modes: + @echo "Testing all transpose modes..." + @failed_tests=""; \ + for mode in 1 2 4; do \ + echo "=== Testing TRANSP_MODE=$$mode ==="; \ + if $(MAKE) sim CONFIG_PRESET=transpose-test DATAMOVER_MODE=1 TRANSP_MODE=$$mode; then \ + echo "✓ TRANSP_MODE=$$mode: PASSED"; \ + else \ + echo "✗ TRANSP_MODE=$$mode: FAILED"; \ + failed_tests="$$failed_tests $$mode"; \ + fi; \ + done; \ + if [ -n "$$failed_tests" ]; then \ + echo ""; \ + echo "====== SUMMARY: The following transpose modes FAILED:$$failed_tests ======"; \ + exit 1; \ + else \ + echo ""; \ + echo "====== SUMMARY: All transpose modes PASSED! ======"; \ + fi + + +test-transpose-grid: + @echo "Testing configuration parameter combinations (grid)..." + @failed_tests=""; \ + total_tests=0; \ + passed_tests=0; \ + for bandwidth in 256 512; do \ + for transp_mode in 1 2 4; do \ + for word_width in 16 32 64; do \ + total_tests=$$((total_tests + 1)); \ + echo "=== Testing BANDWIDTH=$$bandwidth TRANSP_MODE=$$transp_mode WORD_WIDTH=$$word_width ==="; \ + if $(MAKE) sim CONFIG_PRESET=medium-tensor BANDWIDTH=$$bandwidth DATAMOVER_MODE=1 TRANSP_MODE=$$transp_mode WORD_WIDTH=$$word_width; then \ + echo "✓ BANDWIDTH=$$bandwidth TRANSP_MODE=$$transp_mode WORD_WIDTH=$$word_width: PASSED"; \ + passed_tests=$$((passed_tests + 1)); \ + else \ + echo "✗ BANDWIDTH=$$bandwidth TRANSP_MODE=$$transp_mode WORD_WIDTH=$$word_width: FAILED"; \ + failed_tests="$$failed_tests BANDWIDTH=$$bandwidth/TRANSP_MODE=$$transp_mode/WORD_WIDTH=$$word_width"; \ + fi; \ + done; \ + done; \ + done; \ + echo ""; \ + echo "====== TEST SUMMARY ======"; \ + echo "Total tests: $$total_tests"; \ + echo "Passed: $$passed_tests"; \ + echo "Failed: $$((total_tests - passed_tests))"; \ + if [ -n "$$failed_tests" ]; then \ + echo ""; \ + echo "FAILED combinations:$$failed_tests"; \ + exit 1; \ + else \ + echo ""; \ + echo "====== SUMMARY: All bandwidth/transpose/word width combinations PASSED! ======"; \ + fi + +test-transpose-grid-misaligned: + @echo "Testing configuration parameter combinations (grid)..." + @failed_tests=""; \ + total_tests=0; \ + passed_tests=0; \ + for bandwidth in 160 288; do \ + for transp_mode in 1 2 4; do \ + for word_width in 32; do \ + total_tests=$$((total_tests + 1)); \ + echo "=== Testing BANDWIDTH=$$bandwidth MISALIGNED_ACCESSES=1 TRANSP_MODE=$$transp_mode WORD_WIDTH=$$word_width ==="; \ + if $(MAKE) sim CONFIG_PRESET=medium-tensor BANDWIDTH=$$bandwidth MISALIGNED_ACCESSES=1 DATAMOVER_MODE=1 TRANSP_MODE=$$transp_mode WORD_WIDTH=$$word_width; then \ + echo "✓ BANDWIDTH=$$bandwidth MISALIGNED_ACCESSES=1 TRANSP_MODE=$$transp_mode WORD_WIDTH=$$word_width: PASSED"; \ + passed_tests=$$((passed_tests + 1)); \ + else \ + echo "✗ BANDWIDTH=$$bandwidth MISALIGNED_ACCESSES=1 TRANSP_MODE=$$transp_mode WORD_WIDTH=$$word_width: FAILED"; \ + failed_tests="$$failed_tests BANDWIDTH=$$bandwidth/MISALIGNED_ACCESSES=1/TRANSP_MODE=$$transp_mode/WORD_WIDTH=$$word_width"; \ + fi; \ + done; \ + done; \ + done; \ + for bandwidth in 320 576; do \ + for transp_mode in 1 2 4; do \ + for word_width in 64; do \ + total_tests=$$((total_tests + 1)); \ + echo "=== Testing BANDWIDTH=$$bandwidth MISALIGNED_ACCESSES=1 TRANSP_MODE=$$transp_mode WORD_WIDTH=$$word_width ==="; \ + if $(MAKE) sim CONFIG_PRESET=medium-tensor BANDWIDTH=$$bandwidth MISALIGNED_ACCESSES=1 DATAMOVER_MODE=1 TRANSP_MODE=$$transp_mode WORD_WIDTH=$$word_width; then \ + echo "✓ BANDWIDTH=$$bandwidth MISALIGNED_ACCESSES=1 TRANSP_MODE=$$transp_mode WORD_WIDTH=$$word_width: PASSED"; \ + passed_tests=$$((passed_tests + 1)); \ + else \ + echo "✗ BANDWIDTH=$$bandwidth MISALIGNED_ACCESSES=1 TRANSP_MODE=$$transp_mode WORD_WIDTH=$$word_width: FAILED"; \ + failed_tests="$$failed_tests BANDWIDTH=$$bandwidth/MISALIGNED_ACCESSES=1/TRANSP_MODE=$$transp_mode/WORD_WIDTH=$$word_width"; \ + fi; \ + done; \ + done; \ + done; \ + echo ""; \ + echo "====== TEST SUMMARY ======"; \ + echo "Total tests: $$total_tests"; \ + echo "Passed: $$passed_tests"; \ + echo "Failed: $$((total_tests - passed_tests))"; \ + if [ -n "$$failed_tests" ]; then \ + echo ""; \ + echo "FAILED combinations:$$failed_tests"; \ + exit 1; \ + else \ + echo ""; \ + echo "====== SUMMARY: All bandwidth/transpose/word width combinations PASSED! ======"; \ + fi + +test-cim-grid: + @echo "Testing CIM configuration parameter combinations (grid)..." + @failed_tests=""; \ + total_tests=0; \ + passed_tests=0; \ + for bandwidth in 128 256; do \ + for word_width in 32 64; do \ + for row_tile_size in 32 64; do \ + total_tests=$$((total_tests + 1)); \ + echo "=== Testing BANDWIDTH=$$bandwidth WORD_WIDTH=$$word_width ROW_TILE_SIZE=$$row_tile_size ==="; \ + if $(MAKE) sim CONFIG_PRESET=cim-large DATAMOVER_MODE=2 CIM_MODE=0 BANDWIDTH=$$bandwidth WORD_WIDTH=$$word_width ROW_TILE_SIZE=$$row_tile_size; then \ + echo "✓ BANDWIDTH=$$bandwidth WORD_WIDTH=$$word_width ROW_TILE_SIZE=$$row_tile_size: PASSED"; \ + passed_tests=$$((passed_tests + 1)); \ + else \ + echo "✗ BANDWIDTH=$$bandwidth WORD_WIDTH=$$word_width ROW_TILE_SIZE=$$row_tile_size: FAILED"; \ + failed_tests="$$failed_tests BANDWIDTH=$$bandwidth/WORD_WIDTH=$$word_width/ROW_TILE_SIZE=$$row_tile_size"; \ + fi; \ + done; \ + done; \ + done; \ + for bandwidth in 512; do \ + for word_width in 32 64; do \ + for row_tile_size in 64; do \ + total_tests=$$((total_tests + 1)); \ + echo "=== Testing BANDWIDTH=$$bandwidth WORD_WIDTH=$$word_width ROW_TILE_SIZE=$$row_tile_size ==="; \ + if $(MAKE) sim CONFIG_PRESET=cim-large DATAMOVER_MODE=2 CIM_MODE=0 BANDWIDTH=$$bandwidth WORD_WIDTH=$$word_width ROW_TILE_SIZE=$$row_tile_size; then \ + echo "✓ BANDWIDTH=$$bandwidth WORD_WIDTH=$$word_width ROW_TILE_SIZE=$$row_tile_size: PASSED"; \ + passed_tests=$$((passed_tests + 1)); \ + else \ + echo "✗ BANDWIDTH=$$bandwidth WORD_WIDTH=$$word_width ROW_TILE_SIZE=$$row_tile_size: FAILED"; \ + failed_tests="$$failed_tests BANDWIDTH=$$bandwidth/WORD_WIDTH=$$word_width/ROW_TILE_SIZE=$$row_tile_size"; \ + fi; \ + done; \ + done; \ + done; \ + echo ""; \ + echo "====== CIM TEST SUMMARY ======"; \ + echo "Total tests: $$total_tests"; \ + echo "Passed: $$passed_tests"; \ + echo "Failed: $$((total_tests - passed_tests))"; \ + if [ -n "$$failed_tests" ]; then \ + echo ""; \ + echo "FAILED CIM combinations:$$failed_tests"; \ + exit 1; \ + else \ + echo ""; \ + echo "====== SUMMARY: All CIM configuration combinations PASSED! ======"; \ + fi + +# Validate current configuration +validate-config: + @echo "Validating current configuration..." + @python3 verif/python/validate_config.py \ + --bandwidth $(BANDWIDTH) \ + --word_width $(WORD_WIDTH) \ + --elem_width $(ELEM_WIDTH) \ + --memory_size $(MEMORY_SIZE) \ + --datamover_mode $(DATAMOVER_MODE) \ + --transp_mode $(TRANSP_MODE) \ + --cim_mode $(CIM_MODE) \ + --row_tile_size $(ROW_TILE_SIZE) \ + --size_m $(TENSOR_SIZE_M) \ + --size_n $(TENSOR_SIZE_N) \ + --num_channels $(NUM_CHANNELS) clean-sim: rm -rf $(SIM_PATH)/work @@ -71,30 +322,28 @@ sim-script: clean-sim mkdir -p $(SIM_PATH) $(BENDER_INSTALL_DIR)/bender script vsim $(BENDER_TARGETS) $(TESTBENCH_DEFINES) --vlog-arg="$(VLOG_FLAGS)" >> $(SIM_PATH)/compile.tcl -sim: stimuli sim-script +sim: stimuli sim-script validate-config cd modelsim && \ GUI=$(GUI) $(MAKE) $(target) buildpath=$(ROOT_DIR)/$(SIM_PATH) clean-stimuli: rm -rf $(STIMULI_DIR) -stimuli: clean-stimuli +stimuli: clean-stimuli validate-config python -m verif.python.generate_stimuli \ --mem_size $(STIM_MEM_SIZE) \ --read_base_addr $(STIM_READ_BASE_ADDR) \ - --read_d0_stride $(STIM_READ_D0_STRIDE) \ - --read_d0_length $(STIM_READ_D0_LENGTH) \ - --read_d1_stride $(STIM_READ_D1_STRIDE) \ - --read_d1_length $(STIM_READ_D1_LENGTH) \ --write_base_addr $(STIM_WRITE_BASE_ADDR) \ - --write_d0_stride $(STIM_WRITE_D0_STRIDE) \ - --write_d0_length $(STIM_WRITE_D0_LENGTH) \ - --write_d1_stride $(STIM_WRITE_D1_STRIDE) \ - --write_d1_length $(STIM_WRITE_D1_LENGTH) \ --bandwidth_bits $(BANDWIDTH) \ --num_elem_word $(NUM_ELEM_WORD) \ --elem_width $(ELEM_WIDTH) \ + --datamover_mode $(DATAMOVER_MODE) \ --transp_mode $(STIM_TRANSP_MODE) \ + --cim_mode $(CIM_MODE) \ + --row_tile_size $(ROW_TILE_SIZE) \ + --size_m $(TENSOR_SIZE_M) \ + --size_n $(TENSOR_SIZE_N) \ + --num_channels $(NUM_CHANNELS) \ --output_dir "verif/python/generated" # Bender @@ -114,4 +363,6 @@ check-bender: $(BENDER_INSTALL_DIR)/bender: mkdir -p $(BENDER_INSTALL_DIR) && cd $(BENDER_INSTALL_DIR) && \ - curl --proto '=https' --tlsv1.2 https://pulp-platform.github.io/bender/init -sSf | sh -s -- $(BENDER_VERSION) \ No newline at end of file + curl --proto '=https' --tlsv1.2 https://pulp-platform.github.io/bender/init -sSf | sh -s -- $(BENDER_VERSION) + +.PHONY: all help test-all-presets test-transpose-modes test-transpose-grid test-cim-grid validate-config clean-sim sim-script sim clean-stimuli stimuli bender check-bender diff --git a/README.md b/README.md index 1062fae..c0a592d 100644 --- a/README.md +++ b/README.md @@ -15,15 +15,20 @@ Configure the hardware and testbench parameters in `config.mk`. - `NUM_ELEM_WORD * ELEM_WIDTH` is the width of a memory bank word. `BANDWIDTH` must be divisible by such word width, as `BANDWIDTH / (NUM_ELEM_WORD * ELEM_WIDTH)` is the number of banks accessed in parallel by the HWPE in one memory access. - `NUM_ELEM_WORD` must be a power of two due to memory addressing. Currently, the configurations `NUM_ELEM_WORD` = 2,4 support the datamover's transposition mode (1 elem, 2 elems, 4 elems). `NUM_ELEM_WORD` = 1 and `NUM_ELEM_WORD` > 4 is not supported. -### Testbench parameters +### Testbench parameters - OUTDATED The following parameters are used to generate the testbench stimuli and to configure the datamover registers. `STIM_*_BASE_ADDR` = start address of the read/write access bursts (element-addressed) `STIM_*_LENGTH` = number of read/write accesses for the d0/d1 dimensions and in total (it is not an address offset!) `STIM_*_STRIDE` = stride between element across dimensions d0/d1 (element-addressed, e.g., stride d1 would be the distance in an element-addressed offset between A[row=0][col=0] and A[row=1][col=0]) +`STIM_*_D2/D3_LENGTH` = optional higher-dimension lengths when using multi-dimensional tiling (0 means unused) +`STIM_*_D2/D3_STRIDE` = stride for the higher dimensions (element-addressed) +`STIM_*_D4_STRIDE` = optional fourth-dimension stride (element-addressed) `STIM_MEM_SIZE` = number of words of the testbench memory +`STIM_*_DIM_ENABLE` = 4-bit mask enabling address generation dimensions (d0..d3) `STIM_TRANSP_MODE` = transposition mode to configure for the datamover (`3'b000` = none, `3'b001` = 1 elem, `3'b010` = 2 elem, `3'b100` = 4 elem) +`STIM_TRANSP_LEN` = transposition length (if set to 0: transp_len = BANDWIDTH_ALIGNED / ELEM_WIDTH) For the complete list of the datamover configuration registers, cf. `datamover_package.sv`. @@ -61,6 +66,45 @@ make sim ``` By default QuestaSim GUI is active. You can simulate the RTL in CLI mode with `GUI=0 make sim`. +## Testing and Validation - OUTDATED + +The datamover HWPE provides several test targets for comprehensive validation: + +### Configuration Testing +```sh +# Test all configuration presets +make test-all-presets + +# Test all transpose modes +make test-transpose-modes + +# Test configuration parameter combinations (grid testing) +make test-transpose-grid +make test-cim-grid + +# Show available configurations +make help +``` + +### Configuration Presets +The system includes predefined test configurations: +- `small-matrix`: 4×4 matrix (quick testing) +- `medium-matrix`: 64×64 matrix (moderate testing) +- `large-matrix`: 448×448 matrix (stress testing) +- `transpose-test`: 32×32 matrix (transpose focus) +- `rect-wide`: 64×256 matrix (4-element transpose) +- `rect-tall`: 256×64 matrix (2-element transpose) +- `rect-narrow`: 16×128 matrix (1-element transpose) +- `rect-elongated`: 128×32 matrix (2-element transpose) +- `copy-small`: 4×4 matrix (copy mode testing) +- `copy-medium`: 64×64 matrix (copy mode testing) +- `cim-small`: 32×128 matrix (CIM mode, 32 inner_dim, 128-bit bandwidth) +- `cim-medium`: 64×256 matrix (CIM mode, 64 inner_dim, 256-bit bandwidth) +- `cim-large`: 128×256 matrix (CIM mode, 64 inner_dim, 512-bit bandwidth) +- `custom`: User-defined (config.mk default) + +For detailed configuration documentation, see `CONFIG_USAGE.md`. + ## Test Results If the tests pass successfully, you should see the following message displayed at the end: ``` @@ -70,10 +114,12 @@ PASSED!!!! ## Contributors - Francesco Conti, University of Bologna (*f.conti@unibo.it*) - Arpan Suravi Prasad, ETH Zurich (*prasadar@iis.ee.ethz.ch*) +- Sergio Mazzola, ETH Zurich (*smazzola@iis.ee.ethz.ch*) +- Cyrill Durrer, ETH Zurich (*cdurrer@iis.ee.ethz.ch*) ## License This repository makes use of two licenses: - for all *software*: Apache License Version 2.0 - for all *hardware*: Solderpad Hardware License Version 0.51 - -For further information have a look at the license files: `LICENSE.hw`, `LICENSE.sw` \ No newline at end of file + +For further information have a look at the license files: `LICENSE.hw`, `LICENSE.sw` diff --git a/config.mk b/config.mk index 97ffd27..b2eb938 100644 --- a/config.mk +++ b/config.mk @@ -5,32 +5,273 @@ # This file contains the configuration parameter for # the standalone simulation of the datamover HWPE +# Standalone testing setup OUTDATED! +# ToDo: Implement SW-based testing with a CPU core + +# Include configuration presets (optional) +-include config_presets.mk + ####################### -# Datamover hw config # +# Datamover HW Config # ####################### -BANDWIDTH ?= 128 # in bits -NUM_ELEM_WORD ?= 4 # number of element in a memory bank word (powers of 2) -ELEM_WIDTH ?= 8 # width of an element (e.g., a byte is 8 bits) - -######################### -# Stimuli configuration # -######################### - -# Tb stimuli config -STIM_READ_BASE_ADDR ?= 0 # element-addressed -STIM_READ_D0_LENGTH ?= 4 -STIM_READ_D0_STRIDE ?= 16 # element-addressed -STIM_READ_D1_LENGTH ?= 4 -STIM_READ_D1_STRIDE ?= 32 # element-addressed -STIM_READ_TOT_LENGTH ?= 16 - -STIM_WRITE_BASE_ADDR ?= 256 # element-addressed -STIM_WRITE_D0_LENGTH ?= 4 -STIM_WRITE_D0_STRIDE ?= 64 # element-addressed -STIM_WRITE_D1_LENGTH ?= 4 -STIM_WRITE_D1_STRIDE ?= 64 # element-addressed -STIM_WRITE_TOT_LENGTH ?= 16 -STIM_MEM_SIZE ?= 65536 # in words - -STIM_TRANSP_MODE ?= 0 # 3'b000 = none, 3'b001 = 1 elem, 3'b010 = 2 elem, 3'b100 = 4 elem +# Hardware configuration (can be overridden by presets or command line) +BANDWIDTH ?= 512 # in bits, multiple of WORD_WIDTH (512) +WORD_WIDTH ?= 64 # in bits, multiple of ELEM_WIDTH (64) +ELEM_WIDTH ?= 8 # in bits (8) +MEMORY_SIZE ?= 131072 # in words +MISALIGNED_ACCESSES ?= 0 + +DATAMOVER_MODE ?= 1 # 0 = copy, 1 = transpose, 2 = CIM data layout conversion, 3 = CIM data layout transpose, 4 = unfold (MobileViT), 5 = fold (MobileViT), other values: not accepted +TRANSP_MODE ?= 1 # 1 = 1 elem, 2 = 2 elem, 4 = 4 elem, other values: not accepted +CIM_MODE ?= 0 # Data layout conversion mode: 0: row-major -> CIM-Layout, 1: reverse (CIM-Layout -> row-major) - This parameter is not passed to the HW, only used in the HAL +ROW_TILE_SIZE ?= 64 # Row tile size for CIM data layout (in elements): 64 for 64x8 CIM macro + +# Input tensor dimensions (in elements) +NUM_CHANNELS ?= 1 # Number of channels (ToDo: currently, this should be set to 1 except for unfold and fold modes) +TENSOR_SIZE_M ?= 64 # Matrix height in elements +TENSOR_SIZE_N ?= 64 # Matrix width in elements + +READ_BASE_ADDR = 0 + +# Derived constants from basic parameters +# BANDWIDTH_REDUCTION := $(shell echo $$(($(MISALIGNED_ACCESSES) * $(WORD_WIDTH)))) # in bits +# BANDWIDTH_ALIGNED := $(shell echo $$(($(BANDWIDTH) - $(BANDWIDTH_REDUCTION)))) # in bits +BANDWIDTH_ALIGNED := $(BANDWIDTH) # in bits +BANDWIDTH_ELEMS := $(shell echo $$(($(BANDWIDTH_ALIGNED) / $(ELEM_WIDTH)))) # Number of elements per bandwidth +NUM_ELEM_WORD := $(shell echo $$(($(WORD_WIDTH) / $(ELEM_WIDTH)))) # Number of elements per word +TENSOR_SIZE_TOT := $(shell echo $$(($(TENSOR_SIZE_M) * $(TENSOR_SIZE_N)))) # Total number of elements in the tensor +TOTAL_ELEMENTS := $(shell echo $$(($(NUM_CHANNELS) * $(TENSOR_SIZE_TOT)))) # Total number of elements in all channels +MATRIX_MISALIGNED := $(shell echo $$(($(TOTAL_ELEMENTS) % $(BANDWIDTH_ELEMS)))) # 1 if tensor size is not multiple of bandwidth elements +ifeq "$(strip $(MATRIX_MISALIGNED))" "0" # Matrix size aligned + TOTAL_ACCESSES := $(shell echo $$(($(TOTAL_ELEMENTS) / $(BANDWIDTH_ELEMS)))) # Total number of memory accesses (words) for the tensor (floor division) +else # Matrix size misaligned + TOTAL_ACCESSES := $(shell echo $$(($(TOTAL_ELEMENTS) / $(BANDWIDTH_ELEMS) + 1))) # Total number of memory accesses (words) for the tensor (+1 for misaligned access) +endif +WRITE_BASE_ADDR = $(shell echo $$(($(READ_BASE_ADDR) + $(TOTAL_ELEMENTS)))) # Element-addressed + +# Align tensor dimensions to bandwidth for transposition (fill elem_matrix) +TENSOR_SIZE_M_MOD := $(shell echo $$(( $(TENSOR_SIZE_M) % $(BANDWIDTH_ELEMS) ))) +TENSOR_SIZE_N_MOD := $(shell echo $$(( $(TENSOR_SIZE_N) % $(BANDWIDTH_ELEMS) ))) +ifeq ($(TENSOR_SIZE_M_MOD),0) + TENSOR_SIZE_M_ALIGNED := $(TENSOR_SIZE_M) +else + TENSOR_SIZE_M_ALIGNED := $(shell echo $$(($(TENSOR_SIZE_M) + $(BANDWIDTH_ELEMS) - $(TENSOR_SIZE_M_MOD)))) # Align M dimension to bandwidth +endif +ifeq ($(TENSOR_SIZE_N_MOD),0) + TENSOR_SIZE_N_ALIGNED := $(TENSOR_SIZE_N) +else + TENSOR_SIZE_N_ALIGNED := $(shell echo $$(($(TENSOR_SIZE_N) + $(BANDWIDTH_ELEMS) - $(TENSOR_SIZE_N_MOD)))) # Align N dimension to bandwidth +endif + +# ADDR and STRIDE are in bytes, LENGTH is in number of memory accesses (bandwidth) + +ifeq "$(strip $(DATAMOVER_MODE))" "0" # Copy mode +$(info Copy mode enabled) +STIM_READ_BASE_ADDR ?= $(READ_BASE_ADDR) # Element-addressed +STIM_READ_D0_LENGTH ?= $(TOTAL_ACCESSES) # [Nof accesses with bandwidth BW per D0-transfer ("row")] +STIM_READ_D0_STRIDE ?= $(BANDWIDTH_ELEMS) # [Elements] +STIM_READ_D1_LENGTH ?= 0 +STIM_READ_D1_STRIDE ?= 0 +STIM_READ_D2_LENGTH ?= 0 +STIM_READ_D2_STRIDE ?= 0 +STIM_READ_D3_LENGTH ?= 0 +STIM_READ_D3_STRIDE ?= 0 +STIM_READ_D4_STRIDE ?= 0 +STIM_READ_TOT_LENGTH ?= $(STIM_READ_D0_LENGTH) # [Total memory accesses] +STIM_READ_DIM_ENABLE ?= "4'b0000" + +STIM_MEM_SIZE ?= $(MEMORY_SIZE) # [Words] +STIM_TRANSP_MODE ?= 0 +# STIM_TRANSP_LEN ?= 0 +STIM_TENSOR_SIZE_M ?= $(TENSOR_SIZE_M) +STIM_TENSOR_SIZE_N ?= $(TENSOR_SIZE_N) +STIM_NUM_CHANNELS ?= 1 +STIM_TOTAL_ELEMENTS ?= $(TOTAL_ELEMENTS) + +STIM_WRITE_BASE_ADDR ?= $(WRITE_BASE_ADDR) +STIM_WRITE_D0_LENGTH ?= $(STIM_READ_D0_LENGTH) +STIM_WRITE_D0_STRIDE ?= $(STIM_READ_D0_STRIDE) +STIM_WRITE_D1_LENGTH ?= 0 +STIM_WRITE_D1_STRIDE ?= 0 +STIM_WRITE_D2_LENGTH ?= 0 +STIM_WRITE_D2_STRIDE ?= 0 +STIM_WRITE_D3_LENGTH ?= 0 +STIM_WRITE_D3_STRIDE ?= 0 +STIM_WRITE_D4_STRIDE ?= 0 +STIM_WRITE_TOT_LENGTH ?= $(STIM_READ_TOT_LENGTH) +STIM_WRITE_DIM_ENABLE ?= "4'b0000" + +else ifeq "$(strip $(DATAMOVER_MODE))" "1" # Transpose mode +$(info Transpose mode $(TRANSP_MODE) enabled) +ifneq ($(filter 1 2 4,$(strip $(TRANSP_MODE))), $(strip $(TRANSP_MODE))) + $(error Invalid TRANSP_MODE $(TRANSP_MODE): must be 1, 2, or 4) +endif + +STIM_READ_BASE_ADDR ?= $(READ_BASE_ADDR) # Element-addressed +STIM_READ_D0_LENGTH ?= $(TENSOR_SIZE_M_ALIGNED) # [Nof accesses with bandwidth BW per D0-transfer ("row")] +STIM_READ_D0_STRIDE ?= $(TENSOR_SIZE_N) # [Elements] +STIM_READ_D1_LENGTH ?= $(shell echo $$(($(TENSOR_SIZE_N_ALIGNED) / $(BANDWIDTH_ELEMS)))) # [Number of full D0-transfers ("rows")] +STIM_READ_D1_STRIDE ?= $(BANDWIDTH_ELEMS) # [Elements] -> manually compute "next row" stride +STIM_READ_D2_LENGTH ?= 0 +STIM_READ_D2_STRIDE ?= 0 +STIM_READ_D3_LENGTH ?= 0 +STIM_READ_D3_STRIDE ?= 0 +STIM_READ_D4_STRIDE ?= 0 +STIM_READ_TOT_LENGTH ?= $(shell echo $$(($(STIM_READ_D0_LENGTH) * $(STIM_READ_D1_LENGTH)))) # [Total memory accesses] +STIM_READ_DIM_ENABLE ?= "4'b0001" + +STIM_MEM_SIZE ?= $(MEMORY_SIZE) # [Words] +STIM_TRANSP_MODE ?= $(TRANSP_MODE) # 1 = 1 elem, 2 = 2 elem, 4 = 4 elem, other values: not accepted +# STIM_TRANSP_LEN ?= 0 # If 0: BANDWIDTH_ALIGNED / ELEM_WIDTH +STIM_TENSOR_SIZE_M ?= $(TENSOR_SIZE_M) # Actual (non-aligned) tensor dimensions +STIM_TENSOR_SIZE_N ?= $(TENSOR_SIZE_N) +STIM_NUM_CHANNELS ?= 1 +STIM_TOTAL_ELEMENTS ?= $(TOTAL_ELEMENTS) + +STIM_WRITE_BASE_ADDR ?= $(WRITE_BASE_ADDR) # Element-addressed +STIM_WRITE_D0_LENGTH ?= $(shell echo $$(($(BANDWIDTH_ELEMS) / $(TRANSP_MODE)))) # Transpose tile width corresponds to bandwidth +STIM_WRITE_D0_STRIDE ?= $(shell echo $$(($(TENSOR_SIZE_M) * $(TRANSP_MODE)))) # Transpose: Input tensor height corresponds to output tensor width +STIM_WRITE_D1_LENGTH ?= $(shell echo $$(($(STIM_WRITE_D0_STRIDE) / $(BANDWIDTH_ELEMS))))# Transpose: Input tensor height corresponds to output tensor width +STIM_WRITE_D1_STRIDE ?= $(BANDWIDTH_ELEMS) # Transpose tile height corresponds to bandwidth +STIM_WRITE_D2_LENGTH ?= 0 +STIM_WRITE_D2_STRIDE ?= $(shell echo $$(($(STIM_WRITE_D0_STRIDE) * $(STIM_WRITE_D0_LENGTH)))) # D2 length is controlled by total length +STIM_WRITE_D3_LENGTH ?= 0 +STIM_WRITE_D3_STRIDE ?= 0 +STIM_WRITE_D4_STRIDE ?= 0 +STIM_WRITE_TOT_LENGTH ?= $(STIM_READ_TOT_LENGTH) +STIM_WRITE_DIM_ENABLE ?= "4'b0011" + +else ifeq "$(strip $(DATAMOVER_MODE))" "2" # CIM data layout conversion mode +$(info CIM data layout conversion mode $(CIM_MODE) enabled) +ifneq ($(filter 0 1, $(strip $(CIM_MODE))), $(strip $(CIM_MODE))) + $(error "Invalid CIM_MODE $(CIM_MODE): must be 0 or 1") +endif +STIM_READ_BASE_ADDR ?= $(READ_BASE_ADDR) # Element-addressed +STIM_READ_D0_LENGTH ?= $(shell echo $$(($(ROW_TILE_SIZE) / $(BANDWIDTH_ELEMS)))) # [Nof accesses with bandwidth BW per D0-transfer ("row")] +STIM_READ_D0_STRIDE ?= $(BANDWIDTH_ELEMS) # [Elements] +STIM_READ_D1_LENGTH ?= $(TENSOR_SIZE_M) +STIM_READ_D1_STRIDE ?= $(TENSOR_SIZE_N) +STIM_READ_D2_LENGTH ?= $(shell echo $$(($(TENSOR_SIZE_N) / $(ROW_TILE_SIZE)))) # Redundant (handled by TOT_LEN) +STIM_READ_D2_STRIDE ?= $(ROW_TILE_SIZE) +STIM_READ_D3_LENGTH ?= 0 +STIM_READ_D3_STRIDE ?= 0 +STIM_READ_D4_STRIDE ?= 0 +PARTIAL_MULT = $(shell echo $$(($(STIM_READ_D0_LENGTH) * $(STIM_READ_D1_LENGTH)))) +STIM_READ_TOT_LENGTH ?= $(shell echo $$(($(PARTIAL_MULT) * $(STIM_READ_D2_LENGTH)))) # [Total memory accesses] +STIM_READ_DIM_ENABLE ?= "4'b0011" + +STIM_MEM_SIZE ?= $(MEMORY_SIZE) # [Words] +STIM_TRANSP_MODE ?= 0 +# STIM_TRANSP_LEN ?= 0 +STIM_TENSOR_SIZE_M ?= $(TENSOR_SIZE_M) +STIM_TENSOR_SIZE_N ?= $(TENSOR_SIZE_N) +STIM_NUM_CHANNELS ?= 1 +STIM_TOTAL_ELEMENTS ?= $(TOTAL_ELEMENTS) + +STIM_WRITE_BASE_ADDR ?= $(WRITE_BASE_ADDR) # Element-addressed +STIM_WRITE_D0_LENGTH ?= $(shell echo $$(($(STIM_READ_D0_LENGTH) * $(TENSOR_SIZE_M)))) +STIM_WRITE_D0_STRIDE ?= $(BANDWIDTH_ELEMS) +STIM_WRITE_D1_LENGTH ?= $(shell echo $$(($(TENSOR_SIZE_N) / $(ROW_TILE_SIZE)))) +STIM_WRITE_D1_STRIDE ?= $(shell echo $$(($(STIM_WRITE_D0_LENGTH) * $(BANDWIDTH_ELEMS)))) +STIM_WRITE_D2_LENGTH ?= 0 +STIM_WRITE_D2_STRIDE ?= 0 +STIM_WRITE_D3_LENGTH ?= 0 +STIM_WRITE_D3_STRIDE ?= 0 +STIM_WRITE_D4_STRIDE ?= 0 +STIM_WRITE_TOT_LENGTH ?= $(STIM_READ_TOT_LENGTH) # Same total length as read +STIM_WRITE_DIM_ENABLE ?= "4'b0001" + +else ifeq "$(strip $(DATAMOVER_MODE))" "3" # CIM data layout transpose mode +$(info CIM data layout transpose mode $(CIM_MODE) enabled) +ifneq ($(filter 0 1, $(strip $(CIM_MODE))), $(strip $(CIM_MODE))) + $(error "Invalid CIM_MODE $(CIM_MODE): must be 0 or 1") +endif +ifneq ($(filter 1 2 4,$(strip $(TRANSP_MODE))), $(strip $(TRANSP_MODE))) + $(error Invalid TRANSP_MODE $(TRANSP_MODE): must be 1, 2, or 4) +endif + +STIM_READ_BASE_ADDR ?= $(READ_BASE_ADDR) # Element-addressed +STIM_READ_D0_LENGTH ?= $(TENSOR_SIZE_M) +STIM_READ_D0_STRIDE ?= $(ROW_TILE_SIZE) +STIM_READ_D1_LENGTH ?= $(shell echo $$(($(ROW_TILE_SIZE) / $(BANDWIDTH_ELEMS)))) +STIM_READ_D1_STRIDE ?= $(BANDWIDTH_ELEMS) +STIM_READ_D2_LENGTH ?= 0 # Not used (controlled by TOT_LEN) +STIM_READ_D2_STRIDE ?= $(shell echo $$(($(TENSOR_SIZE_M) * $(ROW_TILE_SIZE)))) +STIM_READ_D3_LENGTH ?= 0 +STIM_READ_D3_STRIDE ?= 0 +STIM_READ_D4_STRIDE ?= 0 +STIM_READ_TOT_LENGTH ?= $(shell echo $$(($(TOTAL_ELEMENTS) / $(BANDWIDTH_ELEMS)))) +STIM_READ_DIM_ENABLE ?= "4'b0011" + +STIM_MEM_SIZE ?= $(MEMORY_SIZE) # [Words] +STIM_TRANSP_MODE ?= $(TRANSP_MODE) # transp_mode +# STIM_TRANSP_LEN ?= 0 +STIM_TENSOR_SIZE_M ?= $(TENSOR_SIZE_M) +STIM_TENSOR_SIZE_N ?= $(TENSOR_SIZE_N) +STIM_NUM_CHANNELS ?= 1 +STIM_TOTAL_ELEMENTS ?= $(TOTAL_ELEMENTS) + +STIM_WRITE_BASE_ADDR ?= $(WRITE_BASE_ADDR) # Element-addressed +STIM_WRITE_D0_LENGTH ?= $(BANDWIDTH_ELEMS) +STIM_WRITE_D0_STRIDE ?= $(ROW_TILE_SIZE) +STIM_WRITE_D1_LENGTH ?= $(shell echo $$(($(ROW_TILE_SIZE) / $(BANDWIDTH_ELEMS)))) +STIM_WRITE_D1_STRIDE ?= $(BANDWIDTH_ELEMS) +STIM_WRITE_D2_LENGTH ?= 0 +STIM_WRITE_D2_STRIDE ?= $(shell echo $$(($(BANDWIDTH_ELEMS) * $(ROW_TILE_SIZE)))) +STIM_WRITE_D3_LENGTH ?= 0 +STIM_WRITE_D3_STRIDE ?= 0 +STIM_WRITE_D4_STRIDE ?= 0 +STIM_WRITE_TOT_LENGTH ?= $(STIM_READ_TOT_LENGTH) # Same total length as read +STIM_WRITE_DIM_ENABLE ?= "4'b0011" + + +else +$(error "Invalid DATAMOVER_MODE $(DATAMOVER_MODE): must be 0 (copy), 1 (transpose), 2 (CIM data layout conversion), or 3 (CIM data layout transpose)") +endif + +# Debug: Print computed values (uncomment to see values during make) +$(info ========================================) +$(info Hardware Configuration:) +$(info BANDWIDTH: $(BANDWIDTH) bits) +$(info MISALIGNED_ACCESSES: $(MISALIGNED_ACCESSES)) +$(info BANDWIDTH_ALIGNED: $(BANDWIDTH_ALIGNED) bits) +$(info WORD_WIDTH: $(WORD_WIDTH) bits) +$(info ELEM_WIDTH: $(ELEM_WIDTH) bits) +$(info BANDWIDTH_ELEMS: $(BANDWIDTH_ELEMS)) +$(info NUM_ELEM_WORD: $(NUM_ELEM_WORD)) +$(info DATAMOVER_MODE: $(DATAMOVER_MODE)) +$(info TRANSP_MODE: $(TRANSP_MODE)) +$(info CIM_MODE: $(CIM_MODE)) +$(info ROW_TILE_SIZE: $(ROW_TILE_SIZE) elements) +$(info ) +$(info Matrix Configuration:) +$(info MATRIX_SIZE: $(TENSOR_SIZE_M) x $(TENSOR_SIZE_N)) +$(info MATRIX SIZE ALIGNED: $(TENSOR_SIZE_M_ALIGNED) x $(TENSOR_SIZE_N_ALIGNED)) +$(info MEMORY_SIZE: $(MEMORY_SIZE) words) +$(info STIM_MEM_SIZE: $(STIM_MEM_SIZE) words) +$(info ) +$(info Read Configuration:) +$(info BASE_ADDR: $(STIM_READ_BASE_ADDR)) +$(info D0_LENGTH: $(STIM_READ_D0_LENGTH), D0_STRIDE: $(STIM_READ_D0_STRIDE)) +$(info D1_LENGTH: $(STIM_READ_D1_LENGTH), D1_STRIDE: $(STIM_READ_D1_STRIDE)) +$(info D2_LENGTH: $(STIM_READ_D2_LENGTH), D2_STRIDE: $(STIM_READ_D2_STRIDE)) +$(info D3_LENGTH: $(STIM_READ_D3_LENGTH), D3_STRIDE: $(STIM_READ_D3_STRIDE)) +$(info D4_STRIDE: $(STIM_READ_D4_STRIDE)) +$(info TOT_LENGTH: $(STIM_READ_TOT_LENGTH)) +$(info DIM_ENABLE: $(STIM_READ_DIM_ENABLE)) +$(info ) +$(info Write Configuration:) +$(info BASE_ADDR: $(STIM_WRITE_BASE_ADDR)) +$(info D0_LENGTH: $(STIM_WRITE_D0_LENGTH), D0_STRIDE: $(STIM_WRITE_D0_STRIDE)) +$(info D1_LENGTH: $(STIM_WRITE_D1_LENGTH), D1_STRIDE: $(STIM_WRITE_D1_STRIDE)) +$(info D2_LENGTH: $(STIM_WRITE_D2_LENGTH), D2_STRIDE: $(STIM_WRITE_D2_STRIDE)) +$(info D3_LENGTH: $(STIM_WRITE_D3_LENGTH), D3_STRIDE: $(STIM_WRITE_D3_STRIDE)) +$(info D4_STRIDE: $(STIM_WRITE_D4_STRIDE)) +$(info TOT_LENGTH: $(STIM_WRITE_TOT_LENGTH)) +$(info DIM_ENABLE: $(STIM_WRITE_DIM_ENABLE)) +$(info ) +$(info Transpose Configuration:) +$(info TRANSP_MODE: $(STIM_TRANSP_MODE)) +# $(info TRANSP_LEN: $(STIM_TRANSP_LEN)) +$(info ========================================) diff --git a/config_presets.mk b/config_presets.mk new file mode 100644 index 0000000..971cc27 --- /dev/null +++ b/config_presets.mk @@ -0,0 +1,236 @@ +# Copyright 2023 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 + +# Configuration Presets for Datamover HWPE +# This file defines named configuration presets for common test scenarios + +######################################### +# Configuration Preset System # +######################################### + +# Available presets: +# - small-tensor : Small 4x4 tensor for quick testing (transpose) +# - medium-tensor : Medium 64x64 tensor for moderate testing (transpose) +# - large-tensor : Large 448x448 tensor for stress testing (transpose) +# - transpose-test : Optimized for transpose functionality verification +# - rect-wide : Wide rectangular tensor (64x256) (transpose) +# - rect-tall : Tall rectangular tensor (256x64) (transpose) +# - rect-narrow : Narrow rectangular tensor (16x128) (transpose) +# - rect-elongated : Elongated rectangular tensor (128x32) (transpose) +# - copy-small : Small 4x4 tensor for copy mode testing +# - copy-medium : Medium 64x64 tensor for copy mode testing +# - cim-small : CIM 32x128 tensor, ROW_TILE_SIZE=32, 128-bit bandwidth +# - cim-medium : CIM 64x256 tensor, ROW_TILE_SIZE=64, 256-bit bandwidth +# - cim-large : CIM 128x256 tensor, ROW_TILE_SIZE=64, 512-bit bandwidth +# - custom : User-defined configuration (default) + +# Select configuration preset (can be overridden via command line) +CONFIG_PRESET ?= custom + +# Preset-specific configurations +ifeq ($(CONFIG_PRESET),small-tensor) + BANDWIDTH = 32 + WORD_WIDTH = 32 + ELEM_WIDTH = 8 + MEMORY_SIZE = 512 + MISALIGNED_ACCESSES = 0 + DATAMOVER_MODE = 1 + TRANSP_MODE = 1 + CIM_MODE = 0 + ROW_TILE_SIZE = 0 + TENSOR_SIZE_M = 4 + TENSOR_SIZE_N = 4 + CONFIG_DESC = "Small 4x4 tensor, 1-element transpose" +endif + +ifeq ($(CONFIG_PRESET),medium-tensor) + BANDWIDTH = 128 + WORD_WIDTH = 32 + ELEM_WIDTH = 8 + MEMORY_SIZE = 4096 + MISALIGNED_ACCESSES = 0 + DATAMOVER_MODE = 1 + TRANSP_MODE = 1 + CIM_MODE = 0 + ROW_TILE_SIZE = 0 + TENSOR_SIZE_M = 64 + TENSOR_SIZE_N = 64 + CONFIG_DESC = "Medium 64x64 tensor, 1-element transpose" +endif + +ifeq ($(CONFIG_PRESET),large-tensor) + BANDWIDTH = 512 + WORD_WIDTH = 32 + ELEM_WIDTH = 8 + MEMORY_SIZE = 131072 + MISALIGNED_ACCESSES = 0 + DATAMOVER_MODE = 1 + TRANSP_MODE = 1 + CIM_MODE = 0 + ROW_TILE_SIZE = 0 + TENSOR_SIZE_M = 448 + TENSOR_SIZE_N = 448 + CONFIG_DESC = "Large 448x448 tensor, 1-element transpose" +endif + +ifeq ($(CONFIG_PRESET),transpose-test) + BANDWIDTH = 256 + WORD_WIDTH = 32 + ELEM_WIDTH = 8 + MEMORY_SIZE = 16384 + MISALIGNED_ACCESSES = 0 + DATAMOVER_MODE = 1 + TRANSP_MODE = 2 + CIM_MODE = 0 + ROW_TILE_SIZE = 0 + TENSOR_SIZE_M = 32 + TENSOR_SIZE_N = 32 + CONFIG_DESC = "32x32 tensor, 2-element transpose" +endif + +ifeq ($(CONFIG_PRESET),rect-wide) + BANDWIDTH = 256 + WORD_WIDTH = 16 + ELEM_WIDTH = 8 + MEMORY_SIZE = 32768 + MISALIGNED_ACCESSES = 0 + DATAMOVER_MODE = 1 + TRANSP_MODE = 4 + CIM_MODE = 0 + ROW_TILE_SIZE = 0 + TENSOR_SIZE_M = 64 + TENSOR_SIZE_N = 256 + CONFIG_DESC = "Wide rectangular tensor 64x256, 4-element transpose" +endif + +ifeq ($(CONFIG_PRESET),rect-tall) + BANDWIDTH = 256 + WORD_WIDTH = 32 + ELEM_WIDTH = 8 + MEMORY_SIZE = 32768 + MISALIGNED_ACCESSES = 0 + DATAMOVER_MODE = 1 + TRANSP_MODE = 2 + CIM_MODE = 0 + ROW_TILE_SIZE = 0 + TENSOR_SIZE_M = 256 + TENSOR_SIZE_N = 64 + CONFIG_DESC = "Tall rectangular tensor 256x64, 2-element transpose" +endif + +ifeq ($(CONFIG_PRESET),rect-narrow) + BANDWIDTH = 128 + WORD_WIDTH = 32 + ELEM_WIDTH = 8 + MEMORY_SIZE = 8192 + MISALIGNED_ACCESSES = 0 + DATAMOVER_MODE = 1 + TRANSP_MODE = 1 + CIM_MODE = 0 + ROW_TILE_SIZE = 0 + TENSOR_SIZE_M = 16 + TENSOR_SIZE_N = 128 + CONFIG_DESC = "Narrow rectangular tensor 16x128, 1-element transpose" +endif + +ifeq ($(CONFIG_PRESET),rect-elongated) + BANDWIDTH = 256 + WORD_WIDTH = 32 + ELEM_WIDTH = 8 + MEMORY_SIZE = 16384 + MISALIGNED_ACCESSES = 0 + DATAMOVER_MODE = 1 + TRANSP_MODE = 2 + CIM_MODE = 0 + ROW_TILE_SIZE = 0 + TENSOR_SIZE_M = 128 + TENSOR_SIZE_N = 32 + CONFIG_DESC = "Elongated rectangular tensor 128x32, 2-element transpose" +endif + +ifeq ($(CONFIG_PRESET),copy-small) + BANDWIDTH = 32 + WORD_WIDTH = 32 + ELEM_WIDTH = 8 + MEMORY_SIZE = 512 + MISALIGNED_ACCESSES = 0 + DATAMOVER_MODE = 0 + TRANSP_MODE = 0 + CIM_MODE = 0 + ROW_TILE_SIZE = 0 + TENSOR_SIZE_M = 4 + TENSOR_SIZE_N = 4 + CONFIG_DESC = "Small 4x4 tensor, copy mode" +endif + +ifeq ($(CONFIG_PRESET),copy-medium) + BANDWIDTH = 128 + WORD_WIDTH = 32 + ELEM_WIDTH = 8 + MEMORY_SIZE = 4096 + MISALIGNED_ACCESSES = 0 + DATAMOVER_MODE = 0 + TRANSP_MODE = 0 + CIM_MODE = 0 + ROW_TILE_SIZE = 0 + TENSOR_SIZE_M = 64 + TENSOR_SIZE_N = 64 + CONFIG_DESC = "Medium 64x64 tensor, copy mode" +endif + +ifeq ($(CONFIG_PRESET),cim-small) + BANDWIDTH = 128 + WORD_WIDTH = 32 + ELEM_WIDTH = 8 + MEMORY_SIZE = 8192 + MISALIGNED_ACCESSES = 0 + DATAMOVER_MODE = 2 + TRANSP_MODE = 0 + CIM_MODE = 0 + ROW_TILE_SIZE = 32 + TENSOR_SIZE_M = 32 + TENSOR_SIZE_N = 128 + CONFIG_DESC = "CIM 32x128 tensor, ROW_TILE_SIZE=32, 128-bit bandwidth" +endif + +ifeq ($(CONFIG_PRESET),cim-medium) + BANDWIDTH = 256 + WORD_WIDTH = 64 + ELEM_WIDTH = 8 + MEMORY_SIZE = 16384 + MISALIGNED_ACCESSES = 0 + DATAMOVER_MODE = 2 + TRANSP_MODE = 0 + CIM_MODE = 0 + ROW_TILE_SIZE = 64 + TENSOR_SIZE_M = 64 + TENSOR_SIZE_N = 256 + CONFIG_DESC = "CIM 64x256 tensor, ROW_TILE_SIZE=64, 256-bit bandwidth" +endif + +ifeq ($(CONFIG_PRESET),cim-large) + BANDWIDTH = 512 + WORD_WIDTH = 64 + ELEM_WIDTH = 8 + MEMORY_SIZE = 65536 + MISALIGNED_ACCESSES = 0 + DATAMOVER_MODE = 2 + TRANSP_MODE = 0 + CIM_MODE = 0 + ROW_TILE_SIZE = 64 + TENSOR_SIZE_M = 128 + TENSOR_SIZE_N = 256 + CONFIG_DESC = "CIM 128x256 tensor, ROW_TILE_SIZE=64, 512-bit bandwidth" +endif + +ifeq ($(CONFIG_PRESET),custom) + # Use values from config.mk or command line overrides + CONFIG_DESC = "Custom user-defined configuration (config.mk)" +endif + +# Print current configuration info +$(info ========================================) +$(info Configuration Preset (CONFIG_PRESET): $(CONFIG_PRESET)) +$(info Description: $(CONFIG_DESC)) +$(info ========================================) diff --git a/modelsim/Makefile b/modelsim/Makefile index f6ce7b9..db7baf7 100644 --- a/modelsim/Makefile +++ b/modelsim/Makefile @@ -28,4 +28,4 @@ build: cd $(buildpath) && $(VSIM) -c -do 'source compile.tcl; quit' sim_tb_datamover_top_wrap: lib build - cd $(buildpath) && $(VSIM) $(VSIM_FLAGS) -do 'source ../sim_tb_datamover_top_wrap.tcl' \ No newline at end of file + cd $(buildpath) && $(VSIM) $(VSIM_FLAGS) -do 'source ../sim_tb_datamover_top_wrap.tcl' diff --git a/rtl/datamover_engine.sv b/rtl/datamover_engine.sv index b13255e..6237efb 100644 --- a/rtl/datamover_engine.sv +++ b/rtl/datamover_engine.sv @@ -1,5 +1,5 @@ /* - * Copyright (C) 2025 ETH Zurich and University of Bologna + * Copyright (C) 2025-2026 ETH Zurich and University of Bologna * * Copyright and related rights are licensed under the Solderpad Hardware * License, Version 0.51 (the "License"); you may not use this file except in @@ -14,6 +14,7 @@ /* * Authors: Francesco Conti * Sergio Mazzola + * Cyrill Durrer */ module datamover_engine @@ -22,7 +23,7 @@ module datamover_engine import datamover_package::*; #( parameter int unsigned FIFO_DEPTH = 2, - parameter int unsigned BANDWIDTH_ALIGNED = 32, + parameter int unsigned BANDWIDTH_ALIGNED = 512, parameter int unsigned NUM_ELEM_WORD = 4, // number of elements in a bank word parameter int unsigned ELEM_WIDTH = 8, // element width (in bits) // Dependent parameters: do not modify! @@ -31,11 +32,11 @@ module datamover_engine // global signals input logic clk_i, input logic rst_ni, - input logic test_mode_i, + input logic test_mode_i, // unused // local enable & clear - input logic enable_i, + input logic enable_i, // unused input logic clear_i, - // FIXME make it ctrl + // control registers input ctrl_engine_t ctrl_i, // input data stream + handshake hwpe_stream_intf_stream.sink data_in, @@ -44,33 +45,43 @@ module datamover_engine ); // number of elements (in the full bandwidth, not a single bank word) - localparam NB_ELEMENTS = BANDWIDTH_ALIGNED / ELEM_WIDTH; + localparam int unsigned NB_ELEMENTS = BANDWIDTH_ALIGNED / ELEM_WIDTH; + localparam int unsigned NB_ELEM_LOG2 = $clog2(NB_ELEMENTS); // Type def and internal signals typedef enum logic { WRITE, READ } datamover_engine_fsm_t; - datamover_engine_fsm_t fsm_d, fsm_q; - logic clear_elem_matrix; - logic [$clog2(NB_ELEMENTS):0] cnt_q, cnt_d; - logic cnt_en; + datamover_engine_fsm_t fsm_d, fsm_q; + logic clear_elem_matrix; + logic [NB_ELEM_LOG2-1:0] cnt_q, cnt_d; + logic [17:0] tot_cnt_q, tot_cnt_d; + logic cnt_en; logic [NB_ELEMENTS-1:0][ELEM_WIDTH-1:0] data_in_unrolled; - logic data_in_valid; - logic data_in_ready; + logic data_in_valid; + logic data_in_ready; logic [NB_ELEMENTS-1:0][ELEM_WIDTH-1:0] data_out_unrolled; - logic data_out_valid; - logic data_out_ready; - + logic data_out_valid; + logic data_out_ready; + logic [NB_ELEM_LOG2-1:0] remaining_elems; + logic [17:0] total_accesses_copy_mode, total_accesses; + logic [15:0] y_elem_cnt, expanded_y_elems; + logic [9:0] y_tiles, n_tiles, n_tile_cnt; + logic [NB_ELEM_LOG2:0] leftover_rows, leftover_cols; + logic last_y_tile, last_n_tile; + + logic execution_done; + // FSM: WRITE -> READ on input handshake at end of write, READ -> WRITE on output handshake at end of read always_comb begin fsm_d = fsm_q; case (fsm_q) WRITE: begin - if ((cnt_q == ctrl_i.transp_len-ctrl_i.transp_stride) && (data_in_valid & data_in_ready)) begin + if (((cnt_q == ctrl_i.transp_len-ctrl_i.transp_stride)) && (data_in_valid & data_in_ready)) begin fsm_d = READ; end end READ: begin - if ((cnt_q == ctrl_i.transp_len-ctrl_i.transp_stride) && (data_out_valid & data_out_ready)) begin + if (((cnt_q == ctrl_i.transp_len-ctrl_i.transp_stride)) && (data_out_valid & data_out_ready)) begin fsm_d = WRITE; end end @@ -83,14 +94,14 @@ module datamover_engine always_ff @(posedge clk_i or negedge rst_ni) begin if (~rst_ni) begin fsm_q <= WRITE; - end else if (clear_i) begin + end else if (clear_i || execution_done) begin fsm_q <= WRITE; end else begin fsm_q <= fsm_d; end end - assign clear_elem_matrix = (fsm_q == READ && fsm_d == WRITE) ? 1'b1 : 1'b0; + assign clear_elem_matrix = (fsm_q == READ && fsm_d == WRITE); // internal interfaces and unrolling hwpe_stream_intf_stream #( @@ -136,7 +147,111 @@ module datamover_engine .push_i ( data_out_prefifo ), .pop_o ( data_out ) ); - assign data_out_prefifo.strb = '1; // FIXME for leftovers + + + // Partial tile / leftover elements handling + // Due to the streamer address generation, matrices need to be word-aligned in n-dimension for transposition + localparam logic [NB_ELEMENTS-1:0] STRB_ONE = {{(NB_ELEMENTS-1){1'b0}}, 1'b1}; // Necessary to force the shifting operation to the correct bitwidth (default would be only 32b) + assign remaining_elems = ctrl_i.total_elements & (NB_ELEMENTS - 1); // modulo (NB_ELEMENTS: power of two) - this signal is only used in copy mode + assign total_accesses_copy_mode = (ctrl_i.total_elements >> NB_ELEM_LOG2) + ((remaining_elems != 0) ? 1 : 0); + + // y_tiles represents the number of tiles in c-dimension for unfold/fold modes, and the number of tiles in m-dimension for all other modes + assign y_tiles = (ctrl_i.datamover_mode == DATAMOVER_UNFOLD || ctrl_i.datamover_mode == DATAMOVER_FOLD) ? + (ctrl_i.num_channels + NB_ELEMENTS - 1) >> NB_ELEM_LOG2 : + (ctrl_i.tensor_size_m + NB_ELEMENTS - 1) >> NB_ELEM_LOG2; // ceil division + assign n_tiles = (ctrl_i.tensor_size_n + NB_ELEMENTS - 1) >> NB_ELEM_LOG2; // ceil division + assign total_accesses = (ctrl_i.datamover_mode == DATAMOVER_UNFOLD || ctrl_i.datamover_mode == DATAMOVER_FOLD) ? (y_tiles * ctrl_i.tensor_size_m * n_tiles) << NB_ELEM_LOG2 : (y_tiles * n_tiles) << NB_ELEM_LOG2; // NB_ELEMENTS is a power of 2, so multiply by shifting; ToDo: remaining MUL overhead, could be pre-computed in HAL and configured in control register + assign leftover_rows = (ctrl_i.datamover_mode == DATAMOVER_UNFOLD || ctrl_i.datamover_mode == DATAMOVER_FOLD) ? ctrl_i.num_channels & (NB_ELEMENTS - 1) : ctrl_i.tensor_size_m & (NB_ELEMENTS - 1); + assign leftover_cols = ctrl_i.tensor_size_n & (NB_ELEMENTS - 1); + assign expanded_y_elems = y_tiles << NB_ELEM_LOG2; // taking into account partial tiles + assign y_elem_cnt = (expanded_y_elems == 0) ? '0 : (tot_cnt_q % expanded_y_elems); // ToDo: restructure without modulo (pre-compute in HAL?) + assign n_tile_cnt = (expanded_y_elems == 0) ? '0 : (tot_cnt_q / expanded_y_elems); // ToDo: restructure without division (pre-compute in HAL?) + assign last_y_tile = (ctrl_i.datamover_mode == DATAMOVER_UNFOLD || ctrl_i.datamover_mode == DATAMOVER_FOLD) ? + ((y_elem_cnt >> NB_ELEM_LOG2) >= (ctrl_i.num_channels >> NB_ELEM_LOG2)) : + ((y_elem_cnt >> NB_ELEM_LOG2) >= (ctrl_i.tensor_size_m >> NB_ELEM_LOG2)); + assign last_n_tile = (n_tile_cnt >= (ctrl_i.tensor_size_n >> NB_ELEM_LOG2)); + + always_comb begin + data_out_prefifo.strb = '1; + if(ctrl_i.total_elements != 0) begin + if(ctrl_i.datamover_mode == DATAMOVER_COPY) begin // Copy mode + data_out_prefifo.strb = ((tot_cnt_q >= total_accesses_copy_mode-1) && (remaining_elems != 0)) ? ((STRB_ONE << remaining_elems) - 1) : '1; + end else if(ctrl_i.datamover_mode == DATAMOVER_TRANSPOSE) begin // Transpose mode + if((last_y_tile && leftover_rows != 0) && (last_n_tile && leftover_cols != 0)) begin + if((y_elem_cnt & (NB_ELEMENTS - 1)) < leftover_cols) begin + data_out_prefifo.strb = ((STRB_ONE << leftover_rows) - 1); + end else begin + data_out_prefifo.strb = '0; + end + end else if(last_y_tile && leftover_rows != 0) begin + data_out_prefifo.strb = ((STRB_ONE << leftover_rows) - 1); + end else if(last_n_tile && leftover_cols != 0) begin + if((y_elem_cnt & (NB_ELEMENTS - 1)) < leftover_cols) begin + data_out_prefifo.strb = '1; + end else begin + data_out_prefifo.strb = '0; + end + end else begin + data_out_prefifo.strb = '1; + end + end else if(ctrl_i.datamover_mode == DATAMOVER_CIM_CONVERSION) begin // CIM layout conversion mode + if((last_y_tile && leftover_rows != 0) && (last_n_tile && leftover_cols != 0)) begin + if((y_elem_cnt & (NB_ELEMENTS - 1)) < leftover_rows) begin + data_out_prefifo.strb = ((STRB_ONE << leftover_cols) - 1); + end else begin + data_out_prefifo.strb = '0; + end + end else if(last_y_tile && leftover_rows != 0) begin + if((y_elem_cnt & (NB_ELEMENTS - 1)) < leftover_rows) begin + data_out_prefifo.strb = '1; + end else begin + data_out_prefifo.strb = '0; + end + end else if(last_n_tile && leftover_cols != 0) begin + data_out_prefifo.strb = ((STRB_ONE << leftover_cols) - 1); + end else begin + data_out_prefifo.strb = '1; + end + end else if(ctrl_i.datamover_mode == DATAMOVER_UNFOLD) begin // Unfold mode + if((last_y_tile && leftover_rows != 0) && (last_n_tile && leftover_cols != 0)) begin + if((y_elem_cnt & (NB_ELEMENTS - 1)) < leftover_cols) begin + data_out_prefifo.strb = ((STRB_ONE << leftover_rows) - 1); + end else begin + data_out_prefifo.strb = '0; + end + end else if(last_y_tile && leftover_rows != 0) begin + data_out_prefifo.strb = ((STRB_ONE << leftover_rows) - 1); + end else if(last_n_tile && leftover_cols != 0) begin + if((y_elem_cnt & (NB_ELEMENTS - 1)) < leftover_cols) begin + data_out_prefifo.strb = '1; + end else begin + data_out_prefifo.strb = '0; + end + end else begin + data_out_prefifo.strb = '1; + end + end else if(ctrl_i.datamover_mode == DATAMOVER_FOLD) begin // Fold mode (inverse of unfold: leftover_rows <-> leftover_cols roles swapped) + if((last_y_tile && leftover_rows != 0) && (last_n_tile && leftover_cols != 0)) begin + if((y_elem_cnt & (NB_ELEMENTS - 1)) < leftover_rows) begin + data_out_prefifo.strb = ((STRB_ONE << leftover_cols) - 1); + end else begin + data_out_prefifo.strb = '0; + end + end else if(last_n_tile && leftover_cols != 0) begin + data_out_prefifo.strb = ((STRB_ONE << leftover_cols) - 1); + end else if(last_y_tile && leftover_rows != 0) begin + if((y_elem_cnt & (NB_ELEMENTS - 1)) < leftover_rows) begin + data_out_prefifo.strb = '1; + end else begin + data_out_prefifo.strb = '0; + end + end else begin + data_out_prefifo.strb = '1; + end + end + end + end + assign data_out_prefifo.data = data_out_unrolled; assign data_out_prefifo.valid = data_out_valid; assign data_out_ready = data_out_prefifo.ready; @@ -146,30 +261,42 @@ module datamover_engine begin if (~rst_ni) begin cnt_q <= '0; + tot_cnt_q <= '0; end else if(clear_i) begin cnt_q <= '0; + tot_cnt_q <= '0; + end + else if(execution_done) begin + cnt_q <= '0; + tot_cnt_q <= '0; end else if(cnt_en) begin cnt_q <= cnt_d; + tot_cnt_q <= tot_cnt_d; end end - assign cnt_d = cnt_q < ctrl_i.transp_len-ctrl_i.transp_stride ? cnt_q+ctrl_i.transp_stride : '0; + assign cnt_d = (cnt_q < (ctrl_i.transp_len-ctrl_i.transp_stride)) ? cnt_q+ctrl_i.transp_stride : '0; assign cnt_en = fsm_q == WRITE ? data_in_valid & data_in_ready : data_out_valid & data_out_ready; + assign execution_done = (ctrl_i.datamover_mode == DATAMOVER_COPY) ? (total_accesses_copy_mode != 0) && (data_out_prefifo.valid & data_out_prefifo.ready) && (tot_cnt_q >= total_accesses_copy_mode - 1) : + (total_accesses != 0) && (data_out_prefifo.valid & data_out_prefifo.ready) && (tot_cnt_q >= total_accesses - 1); + + // count total number of write accesses + assign tot_cnt_d = (data_out_prefifo.valid & data_out_prefifo.ready) ? tot_cnt_q + 1 : tot_cnt_q; // "Smart shifting": this set of combinational blocks shifts data_in_unrolled // appropriately, depending on the configuration. // E.g., if you have a classical configuration with - // - NUM_ELEM_WORD = 4 and - // - ELEM_WIDTH = 8 bits, i.e. total is 32 bits per word + // - NUM_ELEM_WORD = 8 and + // - ELEM_WIDTH = 8 bits, i.e. total is 64 bits per word // the configurations are: 8b transpose, 16b transpose, 32b transpose. We assume // that transposes >= 64b can be done efficiently by Snitch processors through SSRs // and those < 8b are not interesting in our use case. - localparam MAX_SHIFTING = 4; // in "number of elements per word" + localparam MAX_SHIFTING = (NUM_ELEM_WORD > 4) ? NUM_ELEM_WORD : 4; // e.g., in a classical configuration (ELEM_WIDTH = 8), MAX_SHIFTING is // in bytes, i.e., "4" for 32b transpose (includes shifting by 0 bytes) logic [MAX_SHIFTING-1:0][NB_ELEMENTS-1:0][ELEM_WIDTH-1:0] data_in_shifted; - + for(genvar ii=0; ii * - * Copyright (C) 2019-2020 ETH Zurich, University of Bologna + * Copyright (C) 2019-2026 ETH Zurich, University of Bologna * Copyright and related rights are licensed under the Solderpad Hardware * License, Version 0.51 (the "License"); you may not use this file except in * compliance with the License. You may obtain a copy of the License at @@ -15,6 +15,7 @@ /* * Authors: Francesco Conti + * Cyrill Durrer */ package datamover_package; @@ -31,11 +32,17 @@ package datamover_package; parameter int unsigned MAX_BANDWIDTH = 512; // support maximum 512bits of bandwidth - typedef enum logic[1:0] { TRANSP_4ELEM, TRANSP_2ELEM, TRANSP_1ELEM, TRANSP_NONE } transp_mode_e; + typedef enum logic[1:0] { TRANSP_NONE, TRANSP_1ELEM, TRANSP_2ELEM, TRANSP_4ELEM } transp_mode_e; + typedef enum logic[4:0] { DATAMOVER_COPY, DATAMOVER_TRANSPOSE, DATAMOVER_CIM_CONVERSION, DATAMOVER_CIM_TRANSPOSE, DATAMOVER_UNFOLD, DATAMOVER_FOLD } datamover_mode_e; typedef struct packed { transp_mode_e transp_mode; logic [$clog2(MAX_BANDWIDTH/8):0] transp_len; - logic [2:0] transp_stride; // 1, 2, or 4 + logic [2:0] transp_stride; // 1, 2, or 4 elements + datamover_mode_e datamover_mode; // 0: copy, 1: tranpose, 2: CIM layout conversion + logic [11:0] tensor_size_m; + logic [11:0] tensor_size_n; + logic [20:0] total_elements; // num_channels * size_m * size_n (pre-computed by HAL) + logic [10:0] num_channels; // number of channels (for unfolding/folding) } ctrl_engine_t; parameter int unsigned HWPE_REGISTER_OFFS = 32'h00; // Standard HWPE register offset @@ -54,18 +61,24 @@ package datamover_package; parameter int unsigned DATAMOVER_REGISTER_CXT0_OFFS = 32'h80; // Context 0 offset parameter int unsigned DATAMOVER_REGISTER_CXT1_OFFS = 32'h120; // Context 1 offset - // Job-specific registers - parameter int unsigned DATAMOVER_REG_IN_PTR = 32'h00; // Input pointer - parameter int unsigned DATAMOVER_REG_OUT_PTR = 32'h04; // Output pointer - parameter int unsigned DATAMOVER_REG_LEN0 = 32'h08; // [31:24] in_d1_len[7:0]; [23:12] in_d0_len; [11:0] tot_len - parameter int unsigned DATAMOVER_REG_LEN1 = 32'h0C; // [27:24] in_d1_len[11:8]; [23:12] out_d1_len; [11:0] out_d0_len - parameter int unsigned DATAMOVER_REG_IN_D0_STRIDE = 32'h10; // Input dimension 0 stride - parameter int unsigned DATAMOVER_REG_IN_D1_STRIDE = 32'h14; // Input dimension 1 stride - parameter int unsigned DATAMOVER_REG_IN_D2_STRIDE = 32'h18; // Input dimension 2 stride - parameter int unsigned DATAMOVER_REG_OUT_D0_STRIDE = 32'h1C; // Output dimension 0 stride - parameter int unsigned DATAMOVER_REG_OUT_D1_STRIDE = 32'h20; // Output dimension 1 stride - parameter int unsigned DATAMOVER_REG_OUT_D2_STRIDE = 32'h24; // Output dimension 2 stride - parameter int unsigned DATAMOVER_REG_TRANSP_MODE = 32'h28; // Transposition mode (LSB: 000=none, 001=1 elem, 010=2 elem, 100=4 elem) - // Leftover: [31:16], if 0 then no leftover + // Job-specific registers (packed into 32-bit words to speed up configuration and save memory) + parameter int unsigned DATAMOVER_REG_IN_PTR = 32'h00; // Input pointer + parameter int unsigned DATAMOVER_REG_OUT_PTR = 32'h04; // Output pointer + parameter int unsigned DATAMOVER_REG_TOT_LEN = 32'h08; // Total length in number of accesses (BW) + parameter int unsigned DATAMOVER_REG_IN_D0 = 32'h0C; // [31:16] in_d0_stride; [15:0] in_d0_len + parameter int unsigned DATAMOVER_REG_IN_D1 = 32'h10; // [31:16] in_d1_stride; [15:0] in_d1_len + parameter int unsigned DATAMOVER_REG_IN_D2 = 32'h14; // [31:16] in_d2_stride; [15:0] in_d2_len + parameter int unsigned DATAMOVER_REG_IN_D3 = 32'h18; // [31:16] in_d3_stride; [15:0] in_d3_len + parameter int unsigned DATAMOVER_REG_OUT_D0 = 32'h1C; // [31:16] out_d0_stride; [15:0] out_d0_len + parameter int unsigned DATAMOVER_REG_OUT_D1 = 32'h20; // [31:16] out_d1_stride; [15:0] out_d1_len + parameter int unsigned DATAMOVER_REG_OUT_D2 = 32'h24; // [31:16] out_d2_stride; [15:0] out_d2_len + parameter int unsigned DATAMOVER_REG_OUT_D3 = 32'h28; // [31:16] out_d3_stride; [15:0] out_d3_len + parameter int unsigned DATAMOVER_REG_IN_OUT_D4_STRIDE = 32'h2C; // [31:16] out_d4_stride; [15:0] in_d4_stride (d4_len unnecessary due to tot_len) + parameter int unsigned DATAMOVER_REG_MATRIX_DIM = 32'h30; // [31:16] tensor_size_n; [15:0] tensor_size_m + parameter int unsigned DATAMOVER_REG_CHANNELS = 32'h34; // [31:11] total_elements = num_channels * size_m * size_n (pre-compute to save HW resources); [10:0] num_channels (for unfolding/folding) + parameter int unsigned DATAMOVER_REG_CTRL_ENGINE = 32'h38; // [15:12] write_dim_en; [11:8] read_dim_en; [7:3] datamover_mode; [2:0] transp_mode (LSB: 000=none, 001=1 elem, 010=2 elem, 100=4 elem) + + // Note: increase N_IO_REGS in datamover_top.sv when adding new registers here! + endpackage diff --git a/rtl/datamover_streamer.sv b/rtl/datamover_streamer.sv index e93842e..43f6887 100644 --- a/rtl/datamover_streamer.sv +++ b/rtl/datamover_streamer.sv @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020 ETH Zurich and University of Bologna + * Copyright (C) 2020-2026 ETH Zurich and University of Bologna * * Copyright and related rights are licensed under the Solderpad Hardware * License, Version 0.51 (the "License"); you may not use this file except in @@ -14,6 +14,7 @@ /* * Authors: Francesco Conti * Sergio Mazzola + * Cyrill Durrer */ `include "hci_helpers.svh" @@ -49,6 +50,7 @@ module datamover_streamer ); localparam int unsigned BW = `HCI_SIZE_GET_BW(tcdm); + localparam int unsigned AW = `HCI_SIZE_GET_AW(tcdm); localparam int unsigned UW = `HCI_SIZE_GET_UW(tcdm); localparam int unsigned IW = `HCI_SIZE_GET_IW(tcdm); localparam int unsigned EW = `HCI_SIZE_GET_EW(tcdm); @@ -61,10 +63,15 @@ module datamover_streamer hci_core_intf #( .DW ( BANDWIDTH ), .BW ( BW ), + .AW ( AW ), .UW ( UW ), .IW ( IW ), .EW ( EW ), .EHW ( EHW) + // `ifndef SYNTHESIS + // ,.WAIVE_RQ4_ASSERT ( 1'b1 ) // ToDo: make sure that these waives are not hiding real issues in the design + // ,.WAIVE_RQ3_ASSERT ( 1'b1 ) + // `endif ) virt_tcdm [1:0] ( .clk ( clk_i ) ); @@ -74,6 +81,7 @@ module datamover_streamer hci_core_intf #( .DW ( BANDWIDTH ), .BW ( BW ), + .AW ( AW ), .UW ( UW ), .IW ( IW ), .EW ( EW ), @@ -81,7 +89,7 @@ module datamover_streamer ) tcdm_prefifo ( .clk ( clk_i ) ); - + // "Virtual" TCDM interface, used to embody data after the TCDM FIFO // (if present) but before the load filter. Notice this is technically // an array of interfaces, with one single instance inside. This is @@ -89,10 +97,11 @@ module datamover_streamer hci_core_intf #( .DW ( BANDWIDTH ), .BW ( BW ), - .UW ( UW ), - .IW ( IW ), - .EW ( EW ), - .EHW ( EHW ) + .AW ( AW ), + .UW ( UW ), + .IW ( IW ), + .EW ( EW ), + .EHW ( EHW) ) tcdm_prefilter [0:0] ( .clk ( clk_i ) ); @@ -104,6 +113,7 @@ module datamover_streamer .ELEMENT_WIDTH ( ELEM_WIDTH ), // e.g., 8 bits per element .ELEMENTS_PER_BANK ( NUM_ELEM_WORD ), // number of elements in one memory bank word .MISALIGNED_ACCESSES ( MISALIGNED_ACCESSES ), + .DIM_ENABLE_1H ( 4'b1111 ), .`HCI_SIZE_PARAM(tcdm) ( `HCI_SIZE_PARAM(tcdm) ) ) i_source ( .clk_i ( clk_i ), @@ -124,6 +134,7 @@ module datamover_streamer .ELEMENT_WIDTH ( ELEM_WIDTH ), // e.g., 8 bits per element .ELEMENTS_PER_BANK ( NUM_ELEM_WORD ), // number of elements in one memory bank word .MISALIGNED_ACCESSES ( MISALIGNED_ACCESSES ), + .DIM_ENABLE_1H ( 4'b1111 ), .`HCI_SIZE_PARAM(tcdm) ( `HCI_SIZE_PARAM(tcdm) ) ) i_sink ( .clk_i ( clk_i ), @@ -149,6 +160,7 @@ module datamover_streamer hci_core_load_store_mixer #( .DW ( BANDWIDTH ), .BW ( BW ), + .AW ( AW ), .UW ( UW ), .EW ( EW ) ) i_ld_st_mux_static ( @@ -160,7 +172,7 @@ module datamover_streamer .out ( tcdm_prefifo ) ); - // The HCI core FIFO the request path from the response path, easing + // The HCI core FIFO decouples the request path from the response path, easing // timing closure when integrating the accelerator in a cluster. hci_core_fifo #( .FIFO_DEPTH ( TCDM_FIFO_DEPTH ) @@ -194,7 +206,7 @@ module datamover_streamer endgenerate // The HCI core filter is meant to filter out r_valid strobes that the - // cluster may generate even when the TCDM access is a write. These + // cluster may generate even when the TCDM access is a write. These // pollute HCI TCDM FIFOs and mixers, and it is better to remove them // altogether. hci_core_r_valid_filter #( diff --git a/rtl/datamover_top.sv b/rtl/datamover_top.sv index 55dfea2..3def209 100644 --- a/rtl/datamover_top.sv +++ b/rtl/datamover_top.sv @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020 ETH Zurich and University of Bologna + * Copyright (C) 2020-2026 ETH Zurich and University of Bologna * * Copyright and related rights are licensed under the Solderpad Hardware * License, Version 0.51 (the "License"); you may not use this file except in @@ -14,6 +14,7 @@ /* * Authors: Francesco Conti * Sergio Mazzola + * Cyrill Durrer */ `include "hci_helpers.svh" @@ -23,17 +24,18 @@ module datamover_top import hci_package::*; import datamover_package::*; #( - parameter int unsigned ID = 10, // control slave peripheral ID width - parameter int unsigned BANDWIDTH = 288, // total bandwidth of HWPE to TCDM (in bits) - parameter int unsigned NUM_ELEM_WORD = 4, // number of elements in a memory bank word + parameter int unsigned ID = 4, // control slave peripheral ID width + parameter int unsigned BANDWIDTH = 512, // total bandwidth of HWPE to TCDM (in bits) + parameter int unsigned NUM_ELEM_WORD = 8, // number of elements in a memory bank word parameter int unsigned ELEM_WIDTH = 8, // element width (in bits) - parameter int unsigned N_CORES = 8, // number of cores for event inputs - parameter int unsigned N_CONTEXT = 4, // number of context for control slave regfile + parameter int unsigned N_CORES = 2, // number of cores for event inputs + parameter int unsigned N_CONTEXT = 2, // number of context for control slave regfile parameter int unsigned MISALIGNED_ACCESSES = 0, // enable misaligned accesses on TCDM interface parameter hci_size_parameter_t `HCI_SIZE_PARAM(tcdm) = '0, // Dependent parameters: do not modify! localparam int unsigned WORD_WIDTH = NUM_ELEM_WORD * ELEM_WIDTH, // should correspond to bank width - localparam int unsigned NUM_WORDS = BANDWIDTH / WORD_WIDTH // TCDM interface width in number of words + localparam int unsigned NUM_WORDS = BANDWIDTH / WORD_WIDTH, // TCDM interface width in number of words + localparam int unsigned N_IO_REGS = 15 // number of configuration registers exposed by the control slave, adapt here if number of configuration registers changes ) ( // global signals input logic clk_i, @@ -49,7 +51,7 @@ module datamover_top // We "sacrifice" 1 word of memory interface bandwidth in order to support // realignment at a word boundary if the access are misaligned. - localparam BANDWIDTH_ALIGNED = MISALIGNED_ACCESSES === 0 ? BANDWIDTH : BANDWIDTH-WORD_WIDTH; + localparam BANDWIDTH_ALIGNED = MISALIGNED_ACCESSES == 0 ? BANDWIDTH : BANDWIDTH-WORD_WIDTH; // State for the FSM declared directly in datamover_top. typedef enum { DM_IDLE, DM_STARTING, DM_WORKING, DM_FINISHED } dm_state; @@ -71,11 +73,6 @@ module datamover_top flags_slave_t slave_flags; ctrl_regfile_t reg_file; - // Data in and data out internal HWPE-Streams. Notice that the data width - // is set to 256 bits by default, 32 bits less than the default external - // bandwidth. The additional 32 bits of memory bandwidth are used to - // support access to non-word-aligned data packets. - // number of elements (in the full bandwidth, not a single bank word) localparam NB_ELEMENTS = BANDWIDTH_ALIGNED / ELEM_WIDTH; @@ -95,10 +92,10 @@ module datamover_top .clk(clk_i) ); - // The streamer exposes on the memory side a single TCDM 288-bit interface + // The streamer exposes on the memory side a single TCDM 512-bit interface // meant to be directly plugged into an Heterogeneous Cluster Interconnect. // On the accelerator side, it exposes an outgoing data in stream and - // an incoming data out HWPE-Streams, each 256-bit wide. + // an incoming data out HWPE-Streams, each 512-bit wide. datamover_streamer #( .BANDWIDTH ( BANDWIDTH ), .NUM_ELEM_WORD ( NUM_ELEM_WORD ), @@ -119,8 +116,9 @@ module datamover_top .flags_o ( streamer_flags ) ); - // The "engine", i.e., the datapath of the HWPE, is as simple as it gets: - // a FIFO copying the data in stream into the data out one! + // The engine transforms the data_in stream into data_out. Supported modes: + // copy, transpose, CIM layout conversion, unfold, and fold. + // An internal buffer (elem_matrix) of size BWxBW is used to reshuffle the data. datamover_engine #( .FIFO_DEPTH ( 4 ), .BANDWIDTH_ALIGNED ( BANDWIDTH_ALIGNED ), @@ -136,19 +134,13 @@ module datamover_top .data_in ( data_in ), .data_out ( data_out ) ); - - // The slave module exposes a peripheral interconnect HWPE-Periph plug; - // in the default configuration, it provides 4 contexts with 11 registers - // each, which are exposed into `reg_file.hwpe_params` - // Previously it was 2 contexts with 13 registers: since the datamover is used for relatively - // fine-grained jobs, the new config makes offloading faster by compacting the LEN registers - // and allows for more contexts + // The slave module exposes a peripheral interconnect HWPE-Periph plug hwpe_ctrl_slave #( .REGFILE_SCM ( 0 ), .N_CORES ( N_CORES ), .N_CONTEXT ( N_CONTEXT ), - .N_IO_REGS ( 11 ), + .N_IO_REGS ( N_IO_REGS ), .N_GENERIC_REGS ( 8 ), .ID_WIDTH ( ID ) ) i_slave ( @@ -213,47 +205,54 @@ module datamover_top always_comb begin streamer_ctrl_cfg = '0; - streamer_ctrl_cfg.data_in_source_ctrl.addressgen_ctrl.dim_enable_1h = '1; - streamer_ctrl_cfg.data_out_sink_ctrl.addressgen_ctrl.dim_enable_1h = '1; + streamer_ctrl_cfg.data_in_source_ctrl.addressgen_ctrl.dim_enable_1h = reg_file.hwpe_params[DATAMOVER_REG_CTRL_ENGINE >> 2][11:8]; // Enabled dimensions (d0 is always enabled) + streamer_ctrl_cfg.data_out_sink_ctrl.addressgen_ctrl.dim_enable_1h = reg_file.hwpe_params[DATAMOVER_REG_CTRL_ENGINE >> 2][15:12]; // Enabled dimensions (d0 is always enabled) streamer_ctrl_cfg.data_in_source_ctrl.addressgen_ctrl.base_addr = reg_file.hwpe_params[DATAMOVER_REG_IN_PTR >> 2]; streamer_ctrl_cfg.data_out_sink_ctrl.addressgen_ctrl.base_addr = reg_file.hwpe_params[DATAMOVER_REG_OUT_PTR >> 2]; - streamer_ctrl_cfg.data_in_source_ctrl.addressgen_ctrl.tot_len = reg_file.hwpe_params[DATAMOVER_REG_LEN0 >> 2][11:0]; - streamer_ctrl_cfg.data_out_sink_ctrl.addressgen_ctrl.tot_len = reg_file.hwpe_params[DATAMOVER_REG_LEN0 >> 2][11:0]; - streamer_ctrl_cfg.data_in_source_ctrl.addressgen_ctrl.d0_len = reg_file.hwpe_params[DATAMOVER_REG_LEN0 >> 2][23:12]; - streamer_ctrl_cfg.data_in_source_ctrl.addressgen_ctrl.d0_stride = reg_file.hwpe_params[DATAMOVER_REG_IN_D0_STRIDE >> 2]; - streamer_ctrl_cfg.data_in_source_ctrl.addressgen_ctrl.d1_len = { reg_file.hwpe_params[DATAMOVER_REG_LEN1 >> 2][27:24], reg_file.hwpe_params[DATAMOVER_REG_LEN0 >> 2][31:24] }; - streamer_ctrl_cfg.data_in_source_ctrl.addressgen_ctrl.d1_stride = reg_file.hwpe_params[DATAMOVER_REG_IN_D1_STRIDE >> 2]; - streamer_ctrl_cfg.data_in_source_ctrl.addressgen_ctrl.d2_stride = reg_file.hwpe_params[DATAMOVER_REG_IN_D2_STRIDE >> 2]; - streamer_ctrl_cfg.data_out_sink_ctrl.addressgen_ctrl.d0_len = reg_file.hwpe_params[DATAMOVER_REG_LEN1 >> 2][11:0]; - streamer_ctrl_cfg.data_out_sink_ctrl.addressgen_ctrl.d0_stride = reg_file.hwpe_params[DATAMOVER_REG_OUT_D0_STRIDE >> 2]; - streamer_ctrl_cfg.data_out_sink_ctrl.addressgen_ctrl.d1_len = reg_file.hwpe_params[DATAMOVER_REG_LEN1 >> 2][23:12]; - streamer_ctrl_cfg.data_out_sink_ctrl.addressgen_ctrl.d1_stride = reg_file.hwpe_params[DATAMOVER_REG_OUT_D1_STRIDE >> 2]; - streamer_ctrl_cfg.data_out_sink_ctrl.addressgen_ctrl.d2_stride = reg_file.hwpe_params[DATAMOVER_REG_OUT_D2_STRIDE >> 2]; + streamer_ctrl_cfg.data_in_source_ctrl.addressgen_ctrl.tot_len = reg_file.hwpe_params[DATAMOVER_REG_TOT_LEN >> 2][31:0]; + streamer_ctrl_cfg.data_out_sink_ctrl.addressgen_ctrl.tot_len = reg_file.hwpe_params[DATAMOVER_REG_TOT_LEN >> 2][31:0]; + streamer_ctrl_cfg.data_in_source_ctrl.addressgen_ctrl.d0_len = reg_file.hwpe_params[DATAMOVER_REG_IN_D0 >> 2][15:0]; + streamer_ctrl_cfg.data_in_source_ctrl.addressgen_ctrl.d0_stride = reg_file.hwpe_params[DATAMOVER_REG_IN_D0 >> 2][31:16]; + streamer_ctrl_cfg.data_in_source_ctrl.addressgen_ctrl.d1_len = reg_file.hwpe_params[DATAMOVER_REG_IN_D1 >> 2][15:0]; + streamer_ctrl_cfg.data_in_source_ctrl.addressgen_ctrl.d1_stride = reg_file.hwpe_params[DATAMOVER_REG_IN_D1 >> 2][31:16]; + streamer_ctrl_cfg.data_in_source_ctrl.addressgen_ctrl.d2_len = reg_file.hwpe_params[DATAMOVER_REG_IN_D2 >> 2][15:0]; + streamer_ctrl_cfg.data_in_source_ctrl.addressgen_ctrl.d2_stride = reg_file.hwpe_params[DATAMOVER_REG_IN_D2 >> 2][31:16]; + streamer_ctrl_cfg.data_in_source_ctrl.addressgen_ctrl.d3_len = reg_file.hwpe_params[DATAMOVER_REG_IN_D3 >> 2][15:0]; + streamer_ctrl_cfg.data_in_source_ctrl.addressgen_ctrl.d3_stride = reg_file.hwpe_params[DATAMOVER_REG_IN_D3 >> 2][31:16]; + streamer_ctrl_cfg.data_in_source_ctrl.addressgen_ctrl.d4_stride = reg_file.hwpe_params[DATAMOVER_REG_IN_OUT_D4_STRIDE >> 2][15:0]; + streamer_ctrl_cfg.data_out_sink_ctrl.addressgen_ctrl.d0_len = reg_file.hwpe_params[DATAMOVER_REG_OUT_D0 >> 2][15:0]; + streamer_ctrl_cfg.data_out_sink_ctrl.addressgen_ctrl.d0_stride = reg_file.hwpe_params[DATAMOVER_REG_OUT_D0 >> 2][31:16]; + streamer_ctrl_cfg.data_out_sink_ctrl.addressgen_ctrl.d1_len = reg_file.hwpe_params[DATAMOVER_REG_OUT_D1 >> 2][15:0]; + streamer_ctrl_cfg.data_out_sink_ctrl.addressgen_ctrl.d1_stride = reg_file.hwpe_params[DATAMOVER_REG_OUT_D1 >> 2][31:16]; + streamer_ctrl_cfg.data_out_sink_ctrl.addressgen_ctrl.d2_len = reg_file.hwpe_params[DATAMOVER_REG_OUT_D2 >> 2][15:0]; + streamer_ctrl_cfg.data_out_sink_ctrl.addressgen_ctrl.d2_stride = reg_file.hwpe_params[DATAMOVER_REG_OUT_D2 >> 2][31:16]; + streamer_ctrl_cfg.data_out_sink_ctrl.addressgen_ctrl.d3_len = reg_file.hwpe_params[DATAMOVER_REG_OUT_D3 >> 2][15:0]; + streamer_ctrl_cfg.data_out_sink_ctrl.addressgen_ctrl.d3_stride = reg_file.hwpe_params[DATAMOVER_REG_OUT_D3 >> 2][31:16]; + streamer_ctrl_cfg.data_out_sink_ctrl.addressgen_ctrl.d4_stride = reg_file.hwpe_params[DATAMOVER_REG_IN_OUT_D4_STRIDE >> 2][31:16]; end // Binding of engine configuration always_comb begin engine_ctrl = '0; - engine_ctrl.transp_mode = reg_file.hwpe_params[DATAMOVER_REG_TRANSP_MODE >> 2][2:0] == 3'b000 ? TRANSP_NONE : - reg_file.hwpe_params[DATAMOVER_REG_TRANSP_MODE >> 2][2:0] == 3'b001 ? TRANSP_1ELEM : - reg_file.hwpe_params[DATAMOVER_REG_TRANSP_MODE >> 2][2:0] == 3'b010 ? TRANSP_2ELEM : TRANSP_4ELEM; - engine_ctrl.transp_stride = reg_file.hwpe_params[DATAMOVER_REG_TRANSP_MODE >> 2][2:0] == 3'b000 ? 1 : - reg_file.hwpe_params[DATAMOVER_REG_TRANSP_MODE >> 2][2:0] == 3'b001 ? 1 : - reg_file.hwpe_params[DATAMOVER_REG_TRANSP_MODE >> 2][2:0] == 3'b010 ? 2 : 4; - if(reg_file.hwpe_params[DATAMOVER_REG_TRANSP_MODE >> 2][31:16] == '0) begin // no leftover - engine_ctrl.transp_len = BANDWIDTH_ALIGNED/ELEM_WIDTH; - end - else begin // in case of leftover, use the reg content as length - engine_ctrl.transp_len = reg_file.hwpe_params[DATAMOVER_REG_TRANSP_MODE >> 2][31:16]; - end + engine_ctrl.transp_mode = reg_file.hwpe_params[DATAMOVER_REG_CTRL_ENGINE >> 2][2:0] == 3'b000 ? TRANSP_NONE : + reg_file.hwpe_params[DATAMOVER_REG_CTRL_ENGINE >> 2][2:0] == 3'b001 ? TRANSP_1ELEM : + reg_file.hwpe_params[DATAMOVER_REG_CTRL_ENGINE >> 2][2:0] == 3'b010 ? TRANSP_2ELEM : TRANSP_4ELEM; + engine_ctrl.transp_stride = reg_file.hwpe_params[DATAMOVER_REG_CTRL_ENGINE >> 2][2:0] == 3'b000 ? 1 : + reg_file.hwpe_params[DATAMOVER_REG_CTRL_ENGINE >> 2][2:0] == 3'b001 ? 1 : + reg_file.hwpe_params[DATAMOVER_REG_CTRL_ENGINE >> 2][2:0] == 3'b010 ? 2 : 4; + engine_ctrl.datamover_mode = datamover_mode_e'(reg_file.hwpe_params[DATAMOVER_REG_CTRL_ENGINE >> 2][7:3]); + engine_ctrl.tensor_size_m = reg_file.hwpe_params[DATAMOVER_REG_MATRIX_DIM >> 2][15:0]; + engine_ctrl.tensor_size_n = reg_file.hwpe_params[DATAMOVER_REG_MATRIX_DIM >> 2][31:16]; + engine_ctrl.num_channels = reg_file.hwpe_params[DATAMOVER_REG_CHANNELS >> 2][10:0]; + engine_ctrl.total_elements = reg_file.hwpe_params[DATAMOVER_REG_CHANNELS >> 2][31:11]; + engine_ctrl.transp_len = BANDWIDTH_ALIGNED/ELEM_WIDTH; end // Bind the output event, which is propagated to the event unit and used // to implement HWPE datamover barriers. assign evt_o = slave_flags.evt[N_CORES-1:0]; - localparam int unsigned DEBUG_DW = `HCI_SIZE_GET_DW(tcdm); localparam int unsigned DEBUG_BW = `HCI_SIZE_GET_BW(tcdm); localparam int unsigned DEBUG_AW = `HCI_SIZE_GET_AW(tcdm); diff --git a/rtl/datamover_top_wrap.sv b/rtl/datamover_top_wrap.sv index 0024ff3..eec1a6f 100644 --- a/rtl/datamover_top_wrap.sv +++ b/rtl/datamover_top_wrap.sv @@ -1,5 +1,5 @@ /* - * Copyright (C) 2025 ETH Zurich and University of Bologna + * Copyright (C) 2025-2026 ETH Zurich and University of Bologna * * Copyright and related rights are licensed under the Solderpad Hardware * License, Version 0.51 (the "License"); you may not use this file except in @@ -13,7 +13,7 @@ /* * Authors: Sergio Mazzola - * Arpan Suravi Prasad s + * Arpan Suravi Prasad */ // A wrapper for datamover_top that unrolls the interfaces to HCI and @@ -33,7 +33,7 @@ module datamover_top_wrap parameter bit WAIVE_RSP3_ASSERT = 1'b0, parameter bit WAIVE_RSP5_ASSERT = 1'b0, `endif - parameter int unsigned ADDR_WIDTH = 32, // width of addres bus + parameter int unsigned ADDR_WIDTH = 32, // width of address bus parameter int unsigned ID = 10, // control slave peripheral ID width parameter int unsigned BANDWIDTH = 288, // total bandwidth of HWPE to TCDM (in bits) parameter int unsigned NUM_ELEM_WORD = 4, // number of elements in a memory bank word @@ -112,13 +112,13 @@ module datamover_top_wrap // All banks are accessed at the same time, so `req` and `wen` are the same for all banks assign tcdm_req[i] = tcdm.req; assign tcdm_wen[i] = tcdm.wen; - // The datamover accessess a number of NUM_WORDS adjacent words. + // The datamover accesses a number of NUM_WORDS adjacent words. // Assuming the memory is element-indexed, i.e., every word-element is individually addressable // on the address bus: to go from word `n` to word `n+1` we have to skip NUM_ELEM_WORD addresses. assign tcdm_add[i] = tcdm.add + i * NUM_ELEM_WORD; assign tcdm_be[i] = tcdm.be[(i+1)*NUM_ELEM_WORD - 1 : i*NUM_ELEM_WORD]; assign tcdm_data[i] = tcdm.data[(i+1)*WORD_WIDTH - 1 : i*WORD_WIDTH]; - end + end assign tcdm.gnt = &(tcdm_gnt); // only when all words are granted assign tcdm.r_data = { >> {tcdm_r_data}}; assign tcdm.r_valid = &(tcdm_r_valid); // only when all words are valid @@ -190,4 +190,4 @@ module datamover_top_wrap `endif `endif -endmodule // datamover_top_wrap \ No newline at end of file +endmodule // datamover_top_wrap diff --git a/test/datamover_test.c b/test/datamover_test.c new file mode 100644 index 0000000..958412c --- /dev/null +++ b/test/datamover_test.c @@ -0,0 +1,193 @@ +// Copyright 2023 ETH Zurich and University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 +// +// Authors: Cyrill Durrer +// Daniel Keller +// Sergio Mazzola + +#include +#include +#include + +#include "snrt.h" +#include "konark_cluster_raw_addrmap.h" + +#include "konark/hal_datamover.h" +#include "konark/hal_konark.h" + +#include "data.h" + +#define VERBOSE 1 +#if VERBOSE +#include "printf.h" +#endif + +#define P 4 // Unfold patch size (number of elements in a patch) for datamover_unfold + +static inline int datamover_compare_int (uint64_t *actual, uint64_t *golden, int total_bytes) { + int errors = 0; + int len = (int)(total_bytes / 8); // Number of complete uint64_t words + int remaining_elements = total_bytes % 8; + + printf("[DM-INFO] Comparing %d 64b-words + %d 8b-elements:\n", len, remaining_elements); + for (int i=0; i 0) { + for (int i=0; i 0) return 0; + + const uint32_t TOT_SIZE = SIZE_C * SIZE_M * SIZE_N; + datamover_status_t datamover_status; + int errors = 0; + datamover_transp_mode_t transp_mode = DATAMOVER_TRANSP_NONE; + + switch (TRANSP_MODE) { + case 1: + transp_mode = DATAMOVER_TRANSP_1ELEM; + break; + case 2: + transp_mode = DATAMOVER_TRANSP_2ELEM; + break; + case 4: + transp_mode = DATAMOVER_TRANSP_4ELEM; + break; + default: + if (DATAMOVER_MODE == 1 || DATAMOVER_MODE == 3) { + printf("[DM-ERR] Unknown DATAMOVER_TRANSPOSE_MODE=%d: Must be 1, 2, or 4\n", TRANSP_MODE); + return -1; + } + } + + #if VERBOSE + printf("[INFO] cluster = %u, core(cluster) = %u, core(global) = %u\n", snrt_cluster_idx(), snrt_cluster_core_idx(), snrt_global_core_idx()); + #endif + + // Allocate and load buffers on DMA core + static uint8_t *local_in; + static uint8_t *local_out; + static uint8_t *local_gold; + if (snrt_is_dm_core()) { + local_in = (uint8_t *) snrt_l1_alloc_cluster_local(TOT_SIZE, 8); // Alignment paramter: 64: bank-aligned, 8: word-aligned + local_out = (uint8_t *) snrt_l1_alloc_cluster_local(TOT_SIZE, 8); + local_gold = (uint8_t *) snrt_l1_alloc_cluster_local(TOT_SIZE, 8); + + printf("[DM-INFO] Allocated L1 buffers: local_in=%p, local_out=%p, local_gold=%p\n", local_in, local_out, local_gold); + + // Move DMA input and expected goldens into TCDM + snrt_dma_start_1d(local_in, golden_in, TOT_SIZE); + snrt_dma_start_1d(local_gold, golden_out, TOT_SIZE); + snrt_dma_wait_all(); + + // Initialize output buffer with dummy value + for(int i=0; iCIM-layout) or 1 (CIM-layout->CHW)\n", CIM_MODE); + } + break; + case 3: + printf("[DM-INFO] Starting Datamover CIM LAYOUT + TRANSPOSE (%u elements) operation of %ux%u (CONVERTED DIMENSIONS) tensor (CIM inner dimension = %u)\n", transp_mode, SIZE_M, SIZE_N, ROW_TILE_SIZE); + datamover_status = datamover_cim_layout_transpose_blocking(local_in, local_out, SIZE_M, SIZE_N, ROW_TILE_SIZE, transp_mode, timeout); + break; + case 4: + printf("[DM-INFO] Starting Datamover UNFOLD operation of %ux%ux%u tensor (CHW) (unfolded dimension = %ux%ux%u)\n", SIZE_C, SIZE_M, SIZE_N, P, SIZE_M*SIZE_N / P, SIZE_C); + datamover_status = datamover_unfold_blocking(local_in, local_out, SIZE_C, SIZE_M, SIZE_N, timeout); + break; + case 5: + printf("[DM-INFO] Starting Datamover FOLD operation of %ux%ux%u tensor (unfolded PNC) (folded dimension = %ux%ux%u)\n", P, SIZE_M*SIZE_N / P, SIZE_C, SIZE_C, SIZE_M, SIZE_N); + datamover_status = datamover_fold_blocking(local_in, local_out, SIZE_C, SIZE_M, SIZE_N, timeout); + break; + default: + printf("[DM-ERR] Unknown DATAMOVER_MODE=%d: Must be 0 (COPY), 1 (TRANSPOSE), 2 (CIM LAYOUT), 3 (CIM LAYOUT TRANSPOSE), 4 (UNFOLD), or 5 (FOLD)\n", DATAMOVER_MODE); + return -1; + } + + // Disable all HWPE clocks + konark_hwpe_disable_all_clk(); + + // Verification + #if VERBOSE + printf("[INFO] Verifying result...\n"); + #endif + errors = datamover_compare_int((uint64_t*)local_out, (uint64_t*)local_gold, TOT_SIZE); + #if VERBOSE + if (errors == 0) { + printf("[DM-OK] ======= DATAMOVER TEST PASSED =======\n"); + printf(" *** *** \n"); + printf(" ** ** ** ** \n"); + printf(" *** *** \n"); + printf(" /_ \n"); + printf(" , , \n"); + printf(" '-......-' \n\n"); + } else { + printf("[DM-ERR] !!!!!!! DATAMOVER TEST FAILED !!!!!!!\n"); + printf("[DM-ERR] mismatches: %d (%ux%ux%u tensor)\n", errors, SIZE_C, SIZE_M, SIZE_N); + } + #endif + } + + return errors; +} diff --git a/test/hal_datamover.c b/test/hal_datamover.c new file mode 100644 index 0000000..c650663 --- /dev/null +++ b/test/hal_datamover.c @@ -0,0 +1,551 @@ +// Copyright 2025 ETH Zurich and University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 +// +// Authors: Sergio Mazzola +// Cyrill Durrer +// Daniel Keller +// Francesco Conti + +#include +#include + +#include "konark/hal_datamover.h" +#include "konark/hal_hwpe.h" + +///////////// +// Drivers // +///////////// + +/* HWPE mandatory registers */ + +int datamover_acquire_task(void) { + return hwpe_task_queue_acquire_task(DATAMOVER_BASE_ADDR); +} + +void datamover_trigger_task(void) { + hwpe_task_queue_release_and_run(DATAMOVER_BASE_ADDR); +} + +uint32_t datamover_finished(void) { + return hwpe_finished(DATAMOVER_BASE_ADDR); +} + +uint32_t datamover_get_status(void) { + return hwpe_task_queue_status(DATAMOVER_BASE_ADDR); +} + +void datamover_soft_clear(void) { + hwpe_soft_clear(DATAMOVER_BASE_ADDR); +} + +/* Datamover-specific registers */ +//TODO(smazzola): For now only usage of context 0 is supported + +void datamover_in_set(uint32_t value) { + DATAMOVER_HWPE_REG_WRITE(DATAMOVER_REGISTER_OFFSET + DATAMOVER_REG_IN_PTR_OFFSET, value); +} + +void datamover_out_set(uint32_t value) { + DATAMOVER_HWPE_REG_WRITE(DATAMOVER_REGISTER_OFFSET + DATAMOVER_REG_OUT_PTR_OFFSET, value); +} + +void datamover_tot_len_set(uint32_t value) { + DATAMOVER_HWPE_REG_WRITE(DATAMOVER_REGISTER_OFFSET + DATAMOVER_REG_TOT_LEN_OFFSET, value); +} + +void datamover_in_d0_set(uint32_t stride, uint32_t len) { + DATAMOVER_HWPE_REG_WRITE(DATAMOVER_REGISTER_OFFSET + DATAMOVER_REG_IN_D0_OFFSET, ((stride & 0xFFFF) << 16) | (len & 0xFFFF)); +} + +void datamover_in_d1_set(uint32_t stride, uint32_t len) { + DATAMOVER_HWPE_REG_WRITE(DATAMOVER_REGISTER_OFFSET + DATAMOVER_REG_IN_D1_OFFSET, ((stride & 0xFFFF) << 16) | (len & 0xFFFF)); +} + +void datamover_in_d2_set(uint32_t stride, uint32_t len) { + DATAMOVER_HWPE_REG_WRITE(DATAMOVER_REGISTER_OFFSET + DATAMOVER_REG_IN_D2_OFFSET, ((stride & 0xFFFF) << 16) | (len & 0xFFFF)); +} + +void datamover_in_d3_set(uint32_t stride, uint32_t len) { + DATAMOVER_HWPE_REG_WRITE(DATAMOVER_REGISTER_OFFSET + DATAMOVER_REG_IN_D3_OFFSET, ((stride & 0xFFFF) << 16) | (len & 0xFFFF)); +} + +void datamover_out_d0_set(uint32_t stride, uint32_t len) { + DATAMOVER_HWPE_REG_WRITE(DATAMOVER_REGISTER_OFFSET + DATAMOVER_REG_OUT_D0_OFFSET, ((stride & 0xFFFF) << 16) | (len & 0xFFFF)); +} + +void datamover_out_d1_set(uint32_t stride, uint32_t len) { + DATAMOVER_HWPE_REG_WRITE(DATAMOVER_REGISTER_OFFSET + DATAMOVER_REG_OUT_D1_OFFSET, ((stride & 0xFFFF) << 16) | (len & 0xFFFF)); +} + +void datamover_out_d2_set(uint32_t stride, uint32_t len) { + DATAMOVER_HWPE_REG_WRITE(DATAMOVER_REGISTER_OFFSET + DATAMOVER_REG_OUT_D2_OFFSET, ((stride & 0xFFFF) << 16) | (len & 0xFFFF)); +} + +void datamover_out_d3_set(uint32_t stride, uint32_t len) { + DATAMOVER_HWPE_REG_WRITE(DATAMOVER_REGISTER_OFFSET + DATAMOVER_REG_OUT_D3_OFFSET, ((stride & 0xFFFF) << 16) | (len & 0xFFFF)); +} + +void datamover_in_out_d4_stride_set(uint32_t out_stride, uint32_t in_stride) { + DATAMOVER_HWPE_REG_WRITE(DATAMOVER_REGISTER_OFFSET + DATAMOVER_REG_IN_OUT_D4_STRIDE_OFFSET, ((out_stride & 0xFFFF) << 16) | (in_stride & 0xFFFF)); +} + +void datamover_matrix_dim_set(uint32_t tensor_size_n, uint32_t tensor_size_m) { + DATAMOVER_HWPE_REG_WRITE(DATAMOVER_REGISTER_OFFSET + DATAMOVER_REG_MATRIX_DIM_OFFSET, ((tensor_size_n & 0xFFFF) << 16) | (tensor_size_m & 0xFFFF)); +} + +void datamover_channels_set(uint32_t total_elements, uint32_t num_channels) { + DATAMOVER_HWPE_REG_WRITE(DATAMOVER_REGISTER_OFFSET + DATAMOVER_REG_CHANNELS_OFFSET, ((total_elements & 0x1FFFFF) << 11) | (num_channels & 0x7FF)); +} + +void datamover_ctrl_engine_set(datamover_mode_t datamover_mode, uint32_t write_dim_en, uint32_t read_dim_en, datamover_transp_mode_t transp_mode) { // dim_en: bit-mask (bit0=d1, bit1=d2, bit2=d3, bit3=d4) + DATAMOVER_HWPE_REG_WRITE(DATAMOVER_REGISTER_OFFSET + DATAMOVER_REG_CTRL_ENGINE_OFFSET, ((write_dim_en & 0xF) << 12) | ((read_dim_en & 0xF) << 8) | ((datamover_mode & 0x1F) << 3) | (transp_mode & 0x7)); +} + +///////// +// HAL // +///////// + +datamover_status_t datamover_wait_done(uint64_t timeout) { + int status = 0; + int finished = 0; + + // Wait for all jobs or timeout expiration (NOTE: FINISHED register can behave unexpectedly) + // do { + // finished = datamover_finished(); + // } while (finished == 0 && --timeout); + + // Wait for all jobs or timeout expiration + do { + status = datamover_get_status(); // ToDo: Why STATUS register and not FINISHED register? + } while (status != 0 && --timeout); + + if (timeout == 0) { + #if VERBOSE + printf("[ERROR] datamover_wait_done(): Timeout expired, jobs are stuck.\n"); + #endif + return DATAMOVER_TO; + } + #if VERBOSE + printf("[DM-HAL] datamover_wait_done: Job(s) finished successfully.\n"); + #endif + return DATAMOVER_OK; +} + +datamover_status_t datamover_copy(uint8_t *src, uint8_t *dst, uint32_t size_m, uint32_t size_n) { + int acq_to = 1000000; + int job_id = -1; + uint32_t total_accesses = (size_m * size_n) / DATAMOVER_BANDWIDTH_ELEMS; // floored division + + if ((size_m * size_n) % DATAMOVER_BANDWIDTH_ELEMS != 0) { + total_accesses += 1; // additional access for remaining elements + } + #if VERBOSE + printf("[DM-HAL] datamover_copy: size_m=%u, size_n=%u, total_elements=%u, BANDWIDTH_ELEMS=%u, total_accesses=%u\n", size_m, size_n, size_m * size_n, DATAMOVER_BANDWIDTH_ELEMS, total_accesses); + #endif + + while ((job_id = datamover_acquire_task()) < 0 && --acq_to) {} + if (acq_to == 0) { + return DATAMOVER_TO; + } + + datamover_in_set((uint32_t)src); + datamover_out_set((uint32_t)dst); + datamover_tot_len_set(total_accesses); + datamover_in_d0_set(DATAMOVER_BANDWIDTH_ELEMS, total_accesses); + datamover_in_d1_set(0, 0); + datamover_in_d2_set(0, 0); + datamover_in_d3_set(0, 0); + datamover_out_d0_set(DATAMOVER_BANDWIDTH_ELEMS, total_accesses); + datamover_out_d1_set(0, 0); + datamover_out_d2_set(0, 0); + datamover_out_d3_set(0, 0); + datamover_in_out_d4_stride_set(0, 0); + datamover_matrix_dim_set(size_n, size_m); + datamover_channels_set(size_m * size_n, 1); + datamover_ctrl_engine_set(DATAMOVER_COPY, 0, 0, DATAMOVER_TRANSP_NONE); + + datamover_trigger_task(); + + return DATAMOVER_OK; +} + +datamover_status_t datamover_copy_blocking(uint8_t *src, uint8_t *dst, uint32_t size_m, uint32_t size_n, uint64_t timeout) { + datamover_status_t status; + status = datamover_copy(src, dst, size_m, size_n); + if (status != DATAMOVER_OK) { + return status; + } + status = datamover_wait_done(timeout); + return status; +} + +datamover_status_t datamover_transpose(uint8_t *matrix_in, uint8_t *matrix_out, uint32_t size_m, uint32_t size_n, datamover_transp_mode_t transp_mode) { + int acq_to = 1000000; + int job_id = -1; + + int m_tiles = size_m % DATAMOVER_BANDWIDTH_ELEMS ? (size_m / DATAMOVER_BANDWIDTH_ELEMS) + 1 : (size_m / DATAMOVER_BANDWIDTH_ELEMS); // number of tiles in M dimension (rounded up) + int n_tiles = size_n % DATAMOVER_BANDWIDTH_ELEMS ? (size_n / DATAMOVER_BANDWIDTH_ELEMS) + 1 : (size_n / DATAMOVER_BANDWIDTH_ELEMS); // number of tiles in N dimension (rounded up) + + while ((job_id = datamover_acquire_task()) < 0 && --acq_to) {} + if (acq_to == 0) { + return DATAMOVER_TO; + } + + datamover_in_set((uint32_t)matrix_in); + datamover_out_set((uint32_t)matrix_out); + datamover_tot_len_set(m_tiles * n_tiles * DATAMOVER_BANDWIDTH_ELEMS); + datamover_in_d0_set(size_n, m_tiles * DATAMOVER_BANDWIDTH_ELEMS); + datamover_in_d1_set(DATAMOVER_BANDWIDTH_ELEMS, n_tiles); + datamover_in_d2_set(0, 0); + datamover_in_d3_set(0, 0); + datamover_out_d0_set(size_m * transp_mode, DATAMOVER_BANDWIDTH_ELEMS / transp_mode); + datamover_out_d1_set(DATAMOVER_BANDWIDTH_ELEMS, m_tiles * transp_mode); + datamover_out_d2_set(size_m * DATAMOVER_BANDWIDTH_ELEMS, 0); + datamover_out_d3_set(0, 0); + datamover_in_out_d4_stride_set(0, 0); + datamover_matrix_dim_set(size_n, size_m); + datamover_channels_set(size_m * size_n, 1); + datamover_ctrl_engine_set(DATAMOVER_TRANSP, 0x3, 0x1, transp_mode); + + datamover_trigger_task(); + + return DATAMOVER_OK; +} + +datamover_status_t datamover_transpose_blocking(uint8_t *matrix_in, uint8_t *matrix_out, uint32_t size_m, uint32_t size_n, datamover_transp_mode_t transp_mode, uint64_t timeout) { + datamover_status_t status; + + status = datamover_transpose(matrix_in, matrix_out, size_m, size_n, transp_mode); + if (status != DATAMOVER_OK) { + return status; + } + + status = datamover_wait_done(timeout); + return status; +} + +void datamover_cim_layout_config_complete_tiles(uint8_t *matrix_in, uint8_t *matrix_out, uint32_t size_m, uint32_t size_n, uint32_t row_tile_size) { // Configure datamover for complete tiles in N dimension + uint32_t m_tiles = size_m % DATAMOVER_BANDWIDTH_ELEMS ? (size_m / DATAMOVER_BANDWIDTH_ELEMS) + 1 : (size_m / DATAMOVER_BANDWIDTH_ELEMS); // number of tiles in M dimension (rounded up) + uint32_t complete_n_tiles = size_n / row_tile_size; // number of complete tiles in N dimension + + datamover_in_set((uint32_t)matrix_in); + datamover_out_set((uint32_t)matrix_out); + datamover_tot_len_set(m_tiles * complete_n_tiles * DATAMOVER_BANDWIDTH_ELEMS); + datamover_in_d0_set(DATAMOVER_BANDWIDTH_ELEMS, row_tile_size / DATAMOVER_BANDWIDTH_ELEMS); + datamover_in_d1_set(size_n, m_tiles * DATAMOVER_BANDWIDTH_ELEMS); + datamover_in_d2_set(row_tile_size, 0); + datamover_in_d3_set(0, 0); + datamover_out_d0_set(DATAMOVER_BANDWIDTH_ELEMS, m_tiles * DATAMOVER_BANDWIDTH_ELEMS); + datamover_out_d1_set(row_tile_size * size_m, complete_n_tiles); + datamover_out_d2_set(0, 0); + datamover_out_d3_set(0, 0); + datamover_in_out_d4_stride_set(0, 0); + datamover_matrix_dim_set(complete_n_tiles * DATAMOVER_BANDWIDTH_ELEMS, size_m); + datamover_channels_set(complete_n_tiles * DATAMOVER_BANDWIDTH_ELEMS * size_m, 1); + datamover_ctrl_engine_set(DATAMOVER_CIM_LAYOUT, 0x1, 0x3, DATAMOVER_TRANSP_NONE); +} + +void datamover_cim_layout_config_leftover_tiles(uint8_t *matrix_in, uint8_t *matrix_out, uint32_t size_m, uint32_t size_n, uint32_t row_tile_size) { // Configure datamover for leftover tiles in N dimension + uint32_t m_tiles = size_m % DATAMOVER_BANDWIDTH_ELEMS ? (size_m / DATAMOVER_BANDWIDTH_ELEMS) + 1 : (size_m / DATAMOVER_BANDWIDTH_ELEMS); // number of tiles in M dimension (rounded up) + uint32_t complete_n_tiles = size_n / row_tile_size; // number of complete tiles in N dimension + uint32_t leftover_columns = size_n % DATAMOVER_BANDWIDTH_ELEMS; + uint8_t *matrix_in_shifted = matrix_in + complete_n_tiles * DATAMOVER_BANDWIDTH_ELEMS; + uint8_t *matrix_out_shifted = matrix_out + complete_n_tiles * size_m * DATAMOVER_BANDWIDTH_ELEMS; + + datamover_in_set((uint32_t)matrix_in_shifted); + datamover_out_set((uint32_t)matrix_out_shifted); + datamover_tot_len_set(m_tiles * DATAMOVER_BANDWIDTH_ELEMS); + datamover_in_d0_set(DATAMOVER_BANDWIDTH_ELEMS, row_tile_size / DATAMOVER_BANDWIDTH_ELEMS); // Unused if row_tile_size == BANDWIDTH_ELEMS + datamover_in_d1_set(size_n, m_tiles * DATAMOVER_BANDWIDTH_ELEMS); + datamover_in_d2_set(0, 0); + datamover_in_d3_set(0, 0); + datamover_out_d0_set(leftover_columns, m_tiles * DATAMOVER_BANDWIDTH_ELEMS); // different stride for leftover columns + datamover_out_d1_set(0, 0); + datamover_out_d2_set(0, 0); + datamover_out_d3_set(0, 0); + datamover_in_out_d4_stride_set(0, 0); + datamover_matrix_dim_set(leftover_columns, size_m); + datamover_channels_set(leftover_columns * size_m, 1); + datamover_ctrl_engine_set(DATAMOVER_CIM_LAYOUT, 0x0, 0x1, DATAMOVER_TRANSP_NONE); +} + +datamover_status_t datamover_cim_layout(uint8_t *matrix_in, uint8_t *matrix_out, uint32_t size_m, uint32_t size_n, uint32_t row_tile_size) { + // For row-major to A-layout: row_tile_size = CIM inner dimension [elements] (64) + // For row-major to B-layout: row_tile_size = CIM outer dimension [elements] (8x #CIM) + // NOTE: Only supports row_tile_size = BANDWIDTH_ELEMS + // ToDo: Currently BLOCKING for misaligned matrices (waiting for completion of complete tiles before handling leftovers) + int acq_to = 1000000; + int job_id = -1; + + uint32_t complete_n_tiles = size_n / row_tile_size; // number of complete tiles in N dimension + + uint32_t leftover_columns = size_n % DATAMOVER_BANDWIDTH_ELEMS; + if(leftover_columns != 0) { // Misaligned matrix: Handle complete tiles first, then leftover columns + if(size_n > DATAMOVER_BANDWIDTH_ELEMS) { // Handle complete tiles + while ((job_id = datamover_acquire_task()) < 0 && --acq_to) {} + if (acq_to == 0) { + return DATAMOVER_TO; + } + datamover_cim_layout_config_complete_tiles(matrix_in, matrix_out, size_m, size_n, row_tile_size); + datamover_trigger_task(); + } // Handle leftover columns + while ((job_id = datamover_acquire_task()) < 0 && --acq_to) {} + if (acq_to == 0) { + return DATAMOVER_TO; + } + datamover_status_t wait_status = datamover_wait_done(5000000); // ToDo: use second context instead of waiting for completion and reconfiguring datamover for leftover columns + if (wait_status != DATAMOVER_OK) { + return wait_status; + } + + datamover_cim_layout_config_leftover_tiles(matrix_in, matrix_out, size_m, size_n, row_tile_size); + datamover_trigger_task(); + } + else { // Aligned matrix: Handle complete tiles only + while ((job_id = datamover_acquire_task()) < 0 && --acq_to) {} + if (acq_to == 0) { + return DATAMOVER_TO; + } + datamover_cim_layout_config_complete_tiles(matrix_in, matrix_out, size_m, size_n, row_tile_size); + datamover_trigger_task(); + } + return DATAMOVER_OK; +} + +datamover_status_t datamover_cim_layout_blocking(uint8_t *matrix_in, uint8_t *matrix_out, uint32_t size_m, uint32_t size_n, uint32_t row_tile_size, uint64_t timeout) { + datamover_status_t status; + + status = datamover_cim_layout(matrix_in, matrix_out, size_m, size_n, row_tile_size); + if (status != DATAMOVER_OK) { + return status; + } + + status = datamover_wait_done(timeout); + return status; +} + +void datamover_cim_layout_reverse_config_complete_tiles(uint8_t *matrix_in, uint8_t *matrix_out, uint32_t size_m, uint32_t size_n, uint32_t row_tile_size) { // Configure datamover for complete tiles in N dimension + uint32_t complete_n_tiles = size_n / row_tile_size; // number of complete tiles in N dimension + uint32_t cim_layout_m_tiles = (size_m * complete_n_tiles) % DATAMOVER_BANDWIDTH_ELEMS ? ((size_m * complete_n_tiles) / DATAMOVER_BANDWIDTH_ELEMS) + 1 : (size_m * complete_n_tiles) / DATAMOVER_BANDWIDTH_ELEMS; // number of tiles in M dimension (rounded up) + + datamover_in_set((uint32_t)matrix_in); + datamover_out_set((uint32_t)matrix_out); + datamover_tot_len_set(cim_layout_m_tiles * DATAMOVER_BANDWIDTH_ELEMS); + datamover_in_d0_set(DATAMOVER_BANDWIDTH_ELEMS, size_m * complete_n_tiles); + datamover_in_d1_set(0, 0); + datamover_in_d2_set(0, 0); + datamover_in_d3_set(0, 0); + datamover_out_d0_set(size_n, size_m); + datamover_out_d1_set(DATAMOVER_BANDWIDTH_ELEMS, complete_n_tiles); + datamover_out_d2_set(0, 0); + datamover_out_d3_set(0, 0); + datamover_in_out_d4_stride_set(0, 0); + datamover_matrix_dim_set(DATAMOVER_BANDWIDTH_ELEMS, size_m * complete_n_tiles); // Representing CIM layout instead of original dimensions + datamover_channels_set(DATAMOVER_BANDWIDTH_ELEMS * size_m * complete_n_tiles, 1); + datamover_ctrl_engine_set(DATAMOVER_CIM_LAYOUT, 0x1, 0x0, DATAMOVER_TRANSP_NONE); +} + +void datamover_cim_layout_reverse_config_leftover_tiles(uint8_t *matrix_in, uint8_t *matrix_out, uint32_t size_m, uint32_t size_n, uint32_t row_tile_size) { // Configure datamover for leftover tiles in N dimension + uint32_t m_tiles = size_m % DATAMOVER_BANDWIDTH_ELEMS ? (size_m / DATAMOVER_BANDWIDTH_ELEMS) + 1 : (size_m / DATAMOVER_BANDWIDTH_ELEMS); // number of tiles in M dimension (rounded up) + uint32_t complete_n_tiles = size_n / row_tile_size; // number of complete tiles in N dimension + uint32_t leftover_columns = size_n % DATAMOVER_BANDWIDTH_ELEMS; + uint8_t *matrix_in_shifted = matrix_in + complete_n_tiles * size_m * DATAMOVER_BANDWIDTH_ELEMS; + uint8_t *matrix_out_shifted = matrix_out + complete_n_tiles * DATAMOVER_BANDWIDTH_ELEMS; + + datamover_in_set((uint32_t)matrix_in_shifted); + datamover_out_set((uint32_t)matrix_out_shifted); + datamover_tot_len_set(m_tiles * DATAMOVER_BANDWIDTH_ELEMS); + datamover_in_d0_set(leftover_columns, m_tiles * DATAMOVER_BANDWIDTH_ELEMS); + datamover_in_d1_set(0, 0); + datamover_in_d2_set(0, 0); + datamover_in_d3_set(0, 0); + datamover_out_d0_set(size_n, m_tiles * DATAMOVER_BANDWIDTH_ELEMS); // different stride for leftover columns + datamover_out_d1_set(0, 0); + datamover_out_d2_set(0, 0); + datamover_out_d3_set(0, 0); + datamover_in_out_d4_stride_set(0, 0); + datamover_matrix_dim_set(leftover_columns, size_m); + datamover_channels_set(leftover_columns * size_m, 1); + datamover_ctrl_engine_set(DATAMOVER_CIM_LAYOUT, 0x0, 0x0, DATAMOVER_TRANSP_NONE); +} + +datamover_status_t datamover_cim_layout_reverse(uint8_t *matrix_in, uint8_t *matrix_out, uint32_t size_m, uint32_t size_n, uint32_t row_tile_size) { + // For A-layout to row-major: row_tile_size = CIM inner dimension [elements] (64) + // For B-layout to row-major: row_tile_size = CIM outer dimension [elements] (8x #CIM) + // NOTE: Only supports row_tile_size = BANDWIDTH_ELEMS for now + int acq_to = 1000000; + int job_id = -1; + + uint32_t complete_n_tiles = size_n / row_tile_size; // number of complete tiles in N dimension + + // Handle leftover columns (if any) + uint32_t leftover_columns = size_n % DATAMOVER_BANDWIDTH_ELEMS; + if(leftover_columns != 0) { + if(size_n > DATAMOVER_BANDWIDTH_ELEMS) { // Handle complete tiles + while ((job_id = datamover_acquire_task()) < 0 && --acq_to) {} + if (acq_to == 0) { + return DATAMOVER_TO; + } + datamover_cim_layout_reverse_config_complete_tiles(matrix_in, matrix_out, size_m, size_n, row_tile_size); + datamover_trigger_task(); + } + // Handle leftover columns + while ((job_id = datamover_acquire_task()) < 0 && --acq_to) {} + if (acq_to == 0) { + return DATAMOVER_TO; + } + datamover_status_t wait_status = datamover_wait_done(5000000); // ToDo: use second context instead of waiting for completion and reconfiguring datamover for leftover columns + if (wait_status != DATAMOVER_OK) { + return wait_status; + } + + datamover_cim_layout_reverse_config_leftover_tiles(matrix_in, matrix_out, size_m, size_n, row_tile_size); + datamover_trigger_task(); + } + else { // Aligned matrix: Handle complete tiles only + while ((job_id = datamover_acquire_task()) < 0 && --acq_to) {} + if (acq_to == 0) { + return DATAMOVER_TO; + } + datamover_cim_layout_reverse_config_complete_tiles(matrix_in, matrix_out, size_m, size_n, row_tile_size); + datamover_trigger_task(); + } + return DATAMOVER_OK; +} + +datamover_status_t datamover_cim_layout_reverse_blocking(uint8_t *matrix_in, uint8_t *matrix_out, uint32_t size_m, uint32_t size_n, uint32_t row_tile_size, uint64_t timeout) { + datamover_status_t status; + status = datamover_cim_layout_reverse(matrix_in, matrix_out, size_m, size_n, row_tile_size); + if (status != DATAMOVER_OK) { + return status; + } + + status = datamover_wait_done(timeout); + return status; +} + +datamover_status_t datamover_cim_layout_transpose_blocking(uint8_t *matrix_in, uint8_t *matrix_out, uint32_t size_m, uint32_t size_n, uint32_t row_tile_size, datamover_transp_mode_t transp_mode, uint64_t timeout) { + // Performs transposition of a matrix in CIM layout, with input and output in CIM layout, by internally performing the necessary layout conversions and transposition in row-major layout. + // DATAMOVER_MODE = 3 is not passed to the HW! It is currently a placeholder for an optimized implementation. + // IMPORTANT: This function uses the input buffer as temporary storage for the transposed matrix in row-major layout. THE ORIGINAL CONTENT OF THE INPUT BUFFER WILL BE OVERWRITTEN! + // ToDo: Implement non-blocking version + datamover_status_t status; + if (size_n <= row_tile_size && size_m <= row_tile_size) { + // If the matrix has only one tile in N and M dimensions, no need to perform 3 separate operations, because CIM layout is the same as row-major layout + #if VERBOSE + printf("[DM-INFO] Single tile in N and M dimensions, performing direct transpose from CIM-layout to row-major and vice versa, %ux%u matrix\n", size_n, size_m); + #endif + status = datamover_transpose_blocking(matrix_in, matrix_out, size_m, size_n, transp_mode, timeout); + } // ToDo(optional): Add special handling for matrices with only one tile in N dimension (size_n <= row_tile_size) but multiple tiles in M dimension, and the opposite case, omitting the unnecessary layout conversion. Requires either an additional memory space or a copy operation. + else { + // Transposition with input and output in CIM layout: 3-phase execution: 1) CIM-layout to row-major, 2) Transpose in row-major, 3) Row-major to CIM-layout + #if VERBOSE + printf("[DM-INFO] OPERATION 1: CIM-layout to row-major, %ux%u matrix\n", (size_n/row_tile_size), (size_m*row_tile_size)); + #endif + status = datamover_cim_layout_reverse_blocking(matrix_in, matrix_out, size_m, size_n, row_tile_size, timeout); + if (status != DATAMOVER_OK) return status; + #if VERBOSE + printf("[DM-INFO] OPERATION 2: Transpose of %ux%u matrix\n", size_m, size_n); + #endif + status = datamover_transpose_blocking(matrix_out, matrix_in, size_m, size_n, transp_mode, timeout); + if (status != DATAMOVER_OK) return status; + #if VERBOSE + printf("[DM-INFO] OPERATION 3: Row-major to CIM-layout, %ux%u matrix\n", size_n, size_m); + #endif + status = datamover_cim_layout_blocking(matrix_in, matrix_out, size_n, size_m, row_tile_size, timeout); + } + return status; +} + +datamover_status_t datamover_unfold(uint8_t *matrix_in, uint8_t *matrix_out, uint32_t size_c, uint32_t size_h, uint32_t size_w) { + // Converts tensor (C,H,W) to unfolded tensor (P,N,C), patch size P = 2x2 = 4 (other patch dimensions not supported yet) + // matrix_in: input tensor in (C,H,W) layout + // matrix_out: output tensor in unfolded (P,N=(H*W)/P,C) layout + // size_c, size_h, size_w: dimensions of the input tensor + const int P = 4; // Patch size (number of elements in a patch), only tested for P=4 (2x2) + const int side_P = 2; // Patch sidelength + int acq_to = 1000000; + int job_id = -1; + uint32_t c_tiles = (size_c + DATAMOVER_BANDWIDTH_ELEMS - 1) / DATAMOVER_BANDWIDTH_ELEMS; // number of tiles in C dimension (rounded up) + uint32_t w_tiles = (size_w + DATAMOVER_BANDWIDTH_ELEMS - 1) / DATAMOVER_BANDWIDTH_ELEMS; // number of tiles in W dimension (rounded up) + + while ((job_id = datamover_acquire_task()) < 0 && --acq_to) {} + if (acq_to == 0) { + return DATAMOVER_TO; + } + datamover_in_set((uint32_t)matrix_in); + datamover_out_set((uint32_t)matrix_out); + datamover_tot_len_set(c_tiles * w_tiles * DATAMOVER_BANDWIDTH_ELEMS * size_h); + datamover_in_d0_set(size_h * size_w, c_tiles * DATAMOVER_BANDWIDTH_ELEMS); + datamover_in_d1_set(DATAMOVER_BANDWIDTH_ELEMS, w_tiles); + datamover_in_d2_set(size_w, size_h); + datamover_in_d3_set(0, 0); + datamover_out_d0_set(size_c * size_h * size_w / P, side_P); + datamover_out_d1_set(size_c, (w_tiles * DATAMOVER_BANDWIDTH_ELEMS) / side_P); + datamover_out_d2_set(DATAMOVER_BANDWIDTH_ELEMS, c_tiles); + datamover_out_d3_set(size_c * size_h * size_w / side_P, side_P); + datamover_in_out_d4_stride_set(size_c * size_w / side_P, 0); + datamover_matrix_dim_set(size_w, size_h); + datamover_channels_set(size_c * size_h * size_w, size_c); + datamover_ctrl_engine_set(DATAMOVER_UNFOLD, 0xF, 0x3, DATAMOVER_TRANSP_1ELEM); + + datamover_trigger_task(); + return DATAMOVER_OK; +} + +datamover_status_t datamover_unfold_blocking(uint8_t *matrix_in, uint8_t *matrix_out, uint32_t size_c, uint32_t size_h, uint32_t size_w, uint64_t timeout) { + datamover_status_t status; + status = datamover_unfold(matrix_in, matrix_out, size_c, size_h, size_w); + if (status != DATAMOVER_OK) { + return status; + } + status = datamover_wait_done(timeout); + return status; +} + +datamover_status_t datamover_fold(uint8_t *matrix_in, uint8_t *matrix_out, uint32_t size_c, uint32_t size_h, uint32_t size_w) { + // Converts unfolded tensor (P,N,C) to folded ("normal") tensor (C,H,W), patch size P = 2x2 = 4 + // matrix_in: input tensor in unfolded (P,N=(H*W)/P,C) layout + // matrix_out: output tensor in folded (C,H,W) layout + // size_c, size_h, size_w: dimensions of the folded (OUTPUT) tensor + const int P = 4; // Patch size (number of elements in a patch), only tested for P=4 (2x2) + const int side_P = 2; // Patch sidelength + int acq_to = 1000000; + int job_id = -1; + uint32_t c_tiles = (size_c + DATAMOVER_BANDWIDTH_ELEMS - 1) / DATAMOVER_BANDWIDTH_ELEMS; // number of tiles in C dimension (rounded up) + uint32_t w_tiles = (size_w + DATAMOVER_BANDWIDTH_ELEMS - 1) / DATAMOVER_BANDWIDTH_ELEMS; // number of tiles in W dimension (rounded up) + + while ((job_id = datamover_acquire_task()) < 0 && --acq_to) {} + if (acq_to == 0) { + return DATAMOVER_TO; + } + datamover_in_set((uint32_t)matrix_in); + datamover_out_set((uint32_t)matrix_out); + datamover_tot_len_set(c_tiles * w_tiles * DATAMOVER_BANDWIDTH_ELEMS * size_h); + datamover_in_d0_set(size_c * size_h * size_w / P, side_P); + datamover_in_d1_set(size_c, (w_tiles * DATAMOVER_BANDWIDTH_ELEMS) / side_P); + datamover_in_d2_set(DATAMOVER_BANDWIDTH_ELEMS, c_tiles); + datamover_in_d3_set(size_c * size_h * size_w / side_P, side_P); + datamover_out_d0_set(size_h * size_w, c_tiles * DATAMOVER_BANDWIDTH_ELEMS); + datamover_out_d1_set(DATAMOVER_BANDWIDTH_ELEMS, w_tiles); + datamover_out_d2_set(size_w, size_h); + datamover_out_d3_set(0, 0); + datamover_in_out_d4_stride_set(0, size_c * size_w / side_P); + datamover_matrix_dim_set(size_w, size_h); + datamover_channels_set(size_c * size_h * size_w, size_c); + datamover_ctrl_engine_set(DATAMOVER_FOLD, 0x3, 0xF, DATAMOVER_TRANSP_1ELEM); + + datamover_trigger_task(); + return DATAMOVER_OK; +} + +datamover_status_t datamover_fold_blocking(uint8_t *matrix_in, uint8_t *matrix_out, uint32_t size_c, uint32_t size_h, uint32_t size_w, uint64_t timeout) { + datamover_status_t status; + status = datamover_fold(matrix_in, matrix_out, size_c, size_h, size_w); + if (status != DATAMOVER_OK) { + return status; + } + status = datamover_wait_done(timeout); + return status; +} diff --git a/test/hal_datamover.h b/test/hal_datamover.h index 40cfb05..08df498 100644 --- a/test/hal_datamover.h +++ b/test/hal_datamover.h @@ -1,155 +1,154 @@ -/* - * Copyright (C) 2020 ETH Zurich and University of Bologna - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Authors: Francesco Conti - */ - -#include +// Copyright 2025 ETH Zurich and University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 +// +// Authors: Sergio Mazzola +// Cyrill Durrer +// Daniel Keller +// Francesco Conti #ifndef __HAL_DATAMOVER_H__ #define __HAL_DATAMOVER_H__ -/* REGISTER MAP */ - -// global address map + event IDs -#define DATAMOVER_ADDR_BASE 0x00201000 -#define CLUS_CTRL_ADDR_BASE 0x00200000 -#define DATAMOVER_EVT0 12 -#define DATAMOVER_EVT1 13 - -// commands -#define DATAMOVER_COMMIT_AND_TRIGGER 0x00 -#define DATAMOVER_ACQUIRE 0x04 -#define DATAMOVER_FINISHED 0x08 -#define DATAMOVER_STATUS 0x0c -#define DATAMOVER_RUNNING_JOB 0x10 -#define DATAMOVER_SOFT_CLEAR 0x14 -#define DATAMOVER_SWSYNC 0x18 -#define DATAMOVER_URISCY_IMEM 0x1c - -// job configuration -#define DATAMOVER_REGISTER_OFFS 0x40 -#define DATAMOVER_REGISTER_CXT0_OFFS 0x80 -#define DATAMOVER_REGISTER_CXT1_OFFS 0x120 -#define DATAMOVER_REG_IN_PTR 0x00 -#define DATAMOVER_REG_OUT_PTR 0x04 -#define DATAMOVER_REG_TOT_LEN 0x08 -#define DATAMOVER_REG_IN_D0_LEN 0x0c -#define DATAMOVER_REG_IN_D0_STRIDE 0x10 -#define DATAMOVER_REG_IN_D1_LEN 0x14 -#define DATAMOVER_REG_IN_D1_STRIDE 0x18 -#define DATAMOVER_REG_IN_D2_STRIDE 0x1c -#define DATAMOVER_REG_OUT_D0_LEN 0x20 -#define DATAMOVER_REG_OUT_D0_STRIDE 0x24 -#define DATAMOVER_REG_OUT_D1_LEN 0x28 -#define DATAMOVER_REG_OUT_D1_STRIDE 0x2c -#define DATAMOVER_REG_OUT_D2_STRIDE 0x30 - -// cluster controller register offset and bits -#define CLUS_CTRL_DATAMOVER_OFFS 0x18 -#define CLUS_CTRL_DATAMOVER_CG_EN_MASK 0x800 -#define CLUS_CTRL_DATAMOVER_HCI_PRIO_MASK 0x100 -#define CLUS_CTRL_DATAMOVER_HCI_MAXSTALL_MASK 0xff - -// others -#define DATAMOVER_COMMIT_CMD 1 -#define DATAMOVER_TRIGGER_CMD 0 -#define DATAMOVER_SOFT_CLEAR_ALL 0 -#define DATAMOVER_SOFT_CLEAR_STATE 1 - -/* LOW-LEVEL HAL */ -// For all the following functions we use __builtin_pulp_OffsetedWrite and __builtin_pulp_OffsetedRead -// instead of classic load/store because otherwise the compiler is not able to correctly factorize -// the DATAMOVER base in case several accesses are done, ending up with twice more code -#if defined(__riscv__) && !defined(RV_ISA_RV32) - #define DATAMOVER_WRITE_CMD(offset, value) __builtin_pulp_OffsetedWrite(value, (int volatile *)(DATAMOVER_ADDR_BASE), offset) - #define DATAMOVER_WRITE_CMD_BE(offset, value, be) *(char volatile *)(DATAMOVER_ADDR_BASE + offset + be) = value - // #define DATAMOVER_READ_CMD(offset) (__builtin_pulp_OffsetedRead(*(int volatile *)(DATAMOVER_ADDR_BASE), offset)) - #define DATAMOVER_READ_CMD(ret, offset) ret = (*(int volatile *)(DATAMOVER_ADDR_BASE + offset)) - - #define DATAMOVER_WRITE_REG(offset, value) __builtin_pulp_OffsetedWrite(value, (int volatile *)(DATAMOVER_ADDR_BASE + DATAMOVER_REGISTER_OFFS), offset) - #define DATAMOVER_WRITE_REG_BE(offset, value, be) *(char volatile *)(DATAMOVER_ADDR_BASE + DATAMOVER_REGISTER_OFFS + offset + be) = value - // #define DATAMOVER_READ_REG(offset) (__builtin_pulp_OffsetedRead(*(int volatile *)(DATAMOVER_ADDR_BASE + DATAMOVER_REGISTER_OFFS), offset)) - #define DATAMOVER_READ_REG(ret, offset) ret = (*(int volatile *)(DATAMOVER_ADDR_BASE + DATAMOVER_REGISTER_OFFS + offset)) - - #define DATAMOVER_WRITE_REG_CXT0(offset, value) __builtin_pulp_OffsetedWrite(value, (int volatile *)(DATAMOVER_ADDR_BASE + DATAMOVER_REGISTER_CXT0_OFFS), offset) - #define DATAMOVER_WRITE_REG_CXT0_BE(offset, value, be) *(char volatile *)(DATAMOVER_ADDR_BASE + DATAMOVER_REGISTER_CXT0_OFFS + offset + be) = value - #define DATAMOVER_READ_REG_CXT0(offset) (__builtin_pulp_OffsetedRead(*(int volatile *)(DATAMOVER_ADDR_BASE + DATAMOVER_REGISTER_CXT0_OFFS), offset)) - - #define DATAMOVER_WRITE_REG_CXT1(offset, value) __builtin_pulp_OffsetedWrite(value, (int volatile *)(DATAMOVER_ADDR_BASE + DATAMOVER_REGISTER_CXT1_OFFS), offset) - #define DATAMOVER_WRITE_REG_CXT1_BE(offset, value, be) *(char volatile *)(DATAMOVER_ADDR_BASE + DATAMOVER_REGISTER_CXT1_OFFS + offset + be) = value - #define DATAMOVER_READ_REG_CXT1(offset) (__builtin_pulp_OffsetedRead(*(int volatile *)(DATAMOVER_ADDR_BASE + DATAMOVER_REGISTER_CXT1_OFFS), offset)) -#else - #define DATAMOVER_WRITE_CMD(offset, value) *(int volatile *)(DATAMOVER_ADDR_BASE + offset) = value - #define DATAMOVER_WRITE_CMD_BE(offset, value, be) *(char volatile *)(DATAMOVER_ADDR_BASE + offset + be) = value - #define DATAMOVER_READ_CMD(ret, offset) ret = (*(int volatile *)(DATAMOVER_ADDR_BASE + offset)) +#include + +#include "konark_cluster_raw_addrmap.h" + +/////////// +// Archi // +/////////// + +#define DATAMOVER_BASE_ADDR KONARK_CLUSTER_ADDRMAP_CLUSTER_HWPE_SUBSYSTEM_HWPE_DATAMOVER_BASE_ADDR - #define DATAMOVER_WRITE_REG(offset, value) *(int volatile *)(DATAMOVER_ADDR_BASE + DATAMOVER_REGISTER_OFFS + offset) = value - #define DATAMOVER_WRITE_REG_BE(offset, value, be) *(char volatile *)(DATAMOVER_ADDR_BASE + DATAMOVER_REGISTER_OFFS + offset + be) = value - #define DATAMOVER_READ_REG(ret, offset) ret = (*(int volatile *)(DATAMOVER_ADDR_BASE + DATAMOVER_REGISTER_OFFS + offset)) +/* Architecture */ - #define DATAMOVER_WRITE_REG_CXT0(offset, value) __builtin_pulp_OffsetedWrite(value, (int volatile *)(DATAMOVER_ADDR_BASE + DATAMOVER_REGISTER_CXT0_OFFS), offset) - #define DATAMOVER_WRITE_REG_CXT0_BE(offset, value, be) *(char volatile *)(DATAMOVER_ADDR_BASE + DATAMOVER_REGISTER_CXT0_OFFS + offset + be) = value - #define DATAMOVER_READ_REG_CXT0(offset) (__builtin_pulp_OffsetedRead(*(int volatile *)(DATAMOVER_ADDR_BASE + DATAMOVER_REGISTER_CXT0_OFFS), offset)) +#define DATAMOVER_BANDWIDTH 512 +#define DATAMOVER_WORD_WIDTH 64 +#define DATAMOVER_ELEM_WIDTH 8 +#define DATAMOVER_MISALIGNED_ACCESSES 0 - #define DATAMOVER_WRITE_REG_CXT1(offset, value) __builtin_pulp_OffsetedWrite(value, (int volatile *)(DATAMOVER_ADDR_BASE + DATAMOVER_REGISTER_CXT1_OFFS), offset) - #define DATAMOVER_WRITE_REG_CXT1_BE(offset, value, be) *(char volatile *)(DATAMOVER_ADDR_BASE + DATAMOVER_REGISTER_CXT1_OFFS + offset + be) = value - #define DATAMOVER_READ_REG_CXT1(offset) (__builtin_pulp_OffsetedRead(*(int volatile *)(DATAMOVER_ADDR_BASE + DATAMOVER_REGISTER_CXT1_OFFS), offset)) +#if DATAMOVER_MISALIGNED_ACCESSES + #define DATAMOVER_BANDWIDTH_ALIGNED (DATAMOVER_BANDWIDTH - DATAMOVER_WORD_WIDTH) +#else + #define DATAMOVER_BANDWIDTH_ALIGNED (DATAMOVER_BANDWIDTH) +#endif +#define DATAMOVER_BANDWIDTH_ELEMS (DATAMOVER_BANDWIDTH_ALIGNED / DATAMOVER_ELEM_WIDTH) +#define DATAMOVER_WORD_ELEMS (DATAMOVER_WORD_WIDTH / DATAMOVER_ELEM_WIDTH) + +/* Registers */ +// To access a register add: DATAMOVER_BASE_ADDR + context offset + register offset + +#define DATAMOVER_REGISTER_OFFSET 0x40 // Alias of DATAMOVER_REGISTER_CXT0_OFFS +#define DATAMOVER_REGISTER_CXT0_OFFS 0x80 // Regfile context 0 +#define DATAMOVER_REGISTER_CXT1_OFFS 0x120 // Regfile context 1 + +#define DATAMOVER_REG_IN_PTR_OFFSET 0x00 // Input pointer +#define DATAMOVER_REG_OUT_PTR_OFFSET 0x04 // Output pointer +#define DATAMOVER_REG_TOT_LEN_OFFSET 0x08 // Total length in number of accesses (BW) +#define DATAMOVER_REG_IN_D0_OFFSET 0x0C // [31:16] in_d0_stride; [15:0] in_d0_len +#define DATAMOVER_REG_IN_D1_OFFSET 0x10 // [31:16] in_d1_stride; [15:0] in_d1_len +#define DATAMOVER_REG_IN_D2_OFFSET 0x14 // [31:16] in_d2_stride; [15:0] in_d2_len +#define DATAMOVER_REG_IN_D3_OFFSET 0x18 // [31:16] in_d3_stride; [15:0] in_d3_len +#define DATAMOVER_REG_OUT_D0_OFFSET 0x1C // [31:16] out_d0_stride; [15:0] out_d0_len +#define DATAMOVER_REG_OUT_D1_OFFSET 0x20 // [31:16] out_d1_stride; [15:0] out_d1_len +#define DATAMOVER_REG_OUT_D2_OFFSET 0x24 // [31:16] out_d2_stride; [15:0] out_d2_len +#define DATAMOVER_REG_OUT_D3_OFFSET 0x28 // [31:16] out_d3_stride; [15:0] out_d3_len +#define DATAMOVER_REG_IN_OUT_D4_STRIDE_OFFSET 0x2C // [31:16] out_d4_stride; [15:0] in_d4_stride (d4_len unnecessary due to tot_len) +#define DATAMOVER_REG_MATRIX_DIM_OFFSET 0x30 // [31:16] tensor_size_n; [15:0] tensor_size_m +#define DATAMOVER_REG_CHANNELS_OFFSET 0x34 // [31:11] total_elements = num_channels * size_m * size_n (pre-compute to save HW resources); [10:0] num_channels (for unfolding/folding) +#define DATAMOVER_REG_CTRL_ENGINE_OFFSET 0x38 // [15:12] write_dim_en; [11:8] read_dim_en; [7:3] datamover_mode; [2:0] transp_mode (LSB: 000=none, 001=1 elem, 010=2 elem, 100=4 elem) + + +/////////// +// Types // +/////////// +typedef enum { + DATAMOVER_COPY = 0x0, + DATAMOVER_TRANSP = 0x1, + DATAMOVER_CIM_LAYOUT = 0x2, + DATAMOVER_CIM_LAYOUT_TRANSPOSE = 0x3, + DATAMOVER_UNFOLD = 0x4, + DATAMOVER_FOLD = 0x5 +} datamover_mode_t; // Must match datamover_mode_e in datamover_package.sv +typedef enum { + DATAMOVER_TRANSP_NONE = 0x0, + DATAMOVER_TRANSP_1ELEM = 0x1, + DATAMOVER_TRANSP_2ELEM = 0x2, + DATAMOVER_TRANSP_4ELEM = 0x4 +} datamover_transp_mode_t; // Must match transp_mode_e in datamover_package.sv + +typedef enum { + DATAMOVER_OK = 0, // Success + DATAMOVER_TO, // Timeout + DATAMOVER_ERR // Generic error +} datamover_status_t; + +///////////// +// Defines // +///////////// + +#if VERBOSE +/* Verbose read/write register */ +#define __HAL_DATAMOVER_REG_WRITE(base, offset, value) do { \ + *(volatile uint32_t *)(base + offset) = value; \ + printf("__HAL_DATAMOVER_REG_WRITE: Addr 0x%08x <= 0x%08x\n", (uint32_t)(base + offset), (uint32_t)(value)); \ + } while(0) +#define __HAL_DATAMOVER_REG_READ(base, offset) ({ \ + uint32_t read_value = *(volatile uint32_t *)(base + offset); \ + printf("__HAL_DATAMOVER_REG_READ: Addr 0x%08x => 0x%08x\n", (uint32_t)(base + offset), (uint32_t)(read_value)); \ + read_value; \ + }) +#else +/* Normal read/write register */ +#define __HAL_DATAMOVER_REG_WRITE(base, offset, value) *(volatile uint32_t *)(base + offset) = value +#define __HAL_DATAMOVER_REG_READ(base, offset) *(volatile uint32_t *)(base + offset) #endif -#define DATAMOVER_CG_ENABLE() *(volatile int*) (CLUS_CTRL_ADDR_BASE + CLUS_CTRL_DATAMOVER_OFFS) |= CLUS_CTRL_DATAMOVER_CG_EN_MASK -#define DATAMOVER_CG_DISABLE() *(volatile int*) (CLUS_CTRL_ADDR_BASE + CLUS_CTRL_DATAMOVER_OFFS) &= ~CLUS_CTRL_DATAMOVER_CG_EN_MASK - -#define DATAMOVER_SETPRIORITY_CORE() *(volatile int*) (CLUS_CTRL_ADDR_BASE + CLUS_CTRL_DATAMOVER_OFFS) &= ~CLUS_CTRL_DATAMOVER_HCI_PRIO_MASK -#define DATAMOVER_SETPRIORITY_DATAMOVER() *(volatile int*) (CLUS_CTRL_ADDR_BASE + CLUS_CTRL_DATAMOVER_OFFS) |= CLUS_CTRL_DATAMOVER_HCI_PRIO_MASK - -#define DATAMOVER_RESET_MAXSTALL() *(volatile int*) (CLUS_CTRL_ADDR_BASE + CLUS_CTRL_DATAMOVER_OFFS) &= ~CLUS_CTRL_DATAMOVER_HCI_MAXSTALL_MASK -#define DATAMOVER_SET_MAXSTALL(val) *(volatile int*) (CLUS_CTRL_ADDR_BASE + CLUS_CTRL_DATAMOVER_OFFS) |= (val & CLUS_CTRL_DATAMOVER_HCI_MAXSTALL_MASK) - -#define DATAMOVER_BARRIER_NOSTATUS() eu_evt_maskWaitAndClr (1 << DATAMOVER_EVT0) -#define DATAMOVER_BARRIER() do { eu_evt_maskWaitAndClr (1 << DATAMOVER_EVT0); } while((*(int volatile *)(DATAMOVER_ADDR_BASE + DATAMOVER_STATUS)) != 0) -#define DATAMOVER_BUSYWAIT() do { } while((*(int volatile *)(DATAMOVER_ADDR_BASE + DATAMOVER_STATUS)) != 0) -#define DATAMOVER_BARRIER_ACQUIRE(job_id) job_id = DATAMOVER_READ_CMD(job_id, DATAMOVER_ACQUIRE); \ - while(job_id < 0) { eu_evt_maskWaitAndClr (1 << DATAMOVER_EVT0); DATAMOVER_READ_CMD(job_id, DATAMOVER_ACQUIRE); }; - -/* UTILITY FUNCTIONS */ -int DATAMOVER_compare_int(uint32_t *actual_y, uint32_t *golden_y, int len) { - uint32_t actual_word = 0; - uint32_t golden_word = 0; - uint32_t actual = 0; - uint32_t golden = 0; - - int errors = 0; - int non_zero_values = 0; - - for (int i=0; i +// Cyrill Durrer +// Sergio Mazzola + +#include + +#include "konark/hal_hwpe.h" + +///////////// +// Drivers // +///////////// + +// Access to HWPE mandatory registers + +void hwpe_task_queue_release_and_run(uint32_t hwpe_base_addr) { + __HAL_HWPE_REG_WRITE(hwpe_base_addr, HWPE_TRIGGER_OFFSET, 0); +} + +void hwpe_task_queue_release(uint32_t hwpe_base_addr) { + __HAL_HWPE_REG_WRITE(hwpe_base_addr, HWPE_TRIGGER_OFFSET, 1); +} + +int hwpe_task_queue_acquire_task(uint32_t hwpe_base_addr) { + int read_value = (int)__HAL_HWPE_REG_READ(hwpe_base_addr, HWPE_ACQUIRE_OFFSET); + return (int)read_value; +} + +uint32_t hwpe_finished(uint32_t hwpe_base_addr) { + return (uint32_t)__HAL_HWPE_REG_READ(hwpe_base_addr, HWPE_FINISHED_OFFSET); +} + +uint32_t hwpe_task_queue_status(uint32_t hwpe_base_addr) { + return (uint32_t)__HAL_HWPE_REG_READ(hwpe_base_addr, HWPE_STATUS_OFFSET); +} + +uint8_t hwpe_last_task_id(uint32_t hwpe_base_addr) { + return (uint8_t)__HAL_HWPE_REG_READ(hwpe_base_addr, HWPE_RUNNING_JOB_OFFSET); +} + +void hwpe_soft_clear(uint32_t hwpe_base_addr) { + __HAL_HWPE_REG_WRITE(hwpe_base_addr, HWPE_SOFT_CLEAR_OFFSET, 0); + // for (volatile int i = 0; i < 10; i++); // ToDo: Is this necessary to ensure the soft clear takes effect before next commands? +} diff --git a/test/hal_hwpe.h b/test/hal_hwpe.h new file mode 100644 index 0000000..b835608 --- /dev/null +++ b/test/hal_hwpe.h @@ -0,0 +1,65 @@ +// Copyright 2025 ETH Zurich and University of Bologna. +// Licensed under the Apache License, Version 2.0, see LICENSE for details. +// SPDX-License-Identifier: Apache-2.0 +// +// Authors: Daniel Keller +// Cyrill Durrer +// Sergio Mazzola + +#ifndef __HAL_HWPE_H__ +#define __HAL_HWPE_H__ + +#include + +/////////// +// Archi // +/////////// + +// Register offsets of HWPE Ctrl (32-bit registers) +#define HWPE_TRIGGER_OFFSET 0*4 +#define HWPE_ACQUIRE_OFFSET 1*4 +#define HWPE_FINISHED_OFFSET 2*4 +#define HWPE_STATUS_OFFSET 3*4 +#define HWPE_RUNNING_JOB_OFFSET 4*4 +#define HWPE_SOFT_CLEAR_OFFSET 5*4 +// #define HWPE_RESERVED_OFFSET 6*4 +#define HWPE_SWSYNC_OFFSET 7*4 + +///////////// +// Defines // +///////////// + +#define __HAL_HWPE_VERBOSE 0 + +#if __HAL_HWPE_VERBOSE +/* Verbose read/write register */ +#include "printf.h" +#define __HAL_HWPE_REG_WRITE(base, offset, value) do { \ + *(volatile uint32_t *)(base + offset) = value; \ + printf("__HAL_HWPE_REG_WRITE: Addr 0x%08x <= 0x%08x\n", (uint32_t)(base + offset), (uint32_t)(value)); \ + } while(0) +#define __HAL_HWPE_REG_READ(base, offset) ({ \ + uint32_t read_value = *(volatile uint32_t *)(base + offset); \ + printf("__HAL_HWPE_REG_READ: Addr 0x%08x => 0x%08x\n", (uint32_t)(base + offset), (uint32_t)(read_value)); \ + read_value; \ + }) +#else +/* Normal read/write register */ +#define __HAL_HWPE_REG_WRITE(base, offset, value) *(volatile uint32_t *)(base + offset) = value +#define __HAL_HWPE_REG_READ(base, offset) *(volatile uint32_t *)(base + offset) +#endif + +//////////////// +// Prototypes // +//////////////// + +// Drivers +void hwpe_task_queue_release_and_run(uint32_t hwpe_base_addr); +void hwpe_task_queue_release(uint32_t hwpe_base_addr); +int hwpe_task_queue_acquire_task(uint32_t hwpe_base_addr); +uint32_t hwpe_finished(uint32_t hwpe_base_addr); +uint32_t hwpe_task_queue_status(uint32_t hwpe_base_addr); +uint8_t hwpe_last_task_id(uint32_t hwpe_base_addr); +void hwpe_soft_clear(uint32_t hwpe_base_addr); + +#endif // __HAL_HWPE_H__ diff --git a/test/lfsr32.c b/test/lfsr32.c deleted file mode 100644 index 6d0635b..0000000 --- a/test/lfsr32.c +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright 2019-2020 Francesco Conti - * - * Adapted from https://github.com/russm/lfsr64 - * This is a simple 32-bit linear feedback shift register, printing - * pseudo-random bytes to stdout. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "pmsis.h" -#include -#include -#include "lfsr32.h" - -#define DEFAULT_SEED 0xdeadbeef -#define USE_BYTE_FEEDBACK - -int glob_errors; - -#ifdef USE_BYTE_FEEDBACK -// not actually extern, just down the bottom -extern uint32_t __lfsr_byte_feedback[]; -#else -uint32_t *__lfsr_byte_feedback; -#endif - -#ifndef USE_BYTE_FEEDBACK -uint32_t __lfsr_iter_bit(uint32_t lfsr) { - return (lfsr & 1) ? ((lfsr >> 1) ^ FEEDBACK) : (lfsr >> 1); -} -#endif - -uint32_t __lfsr_iter_byte(uint32_t lfsr, uint32_t *lfsr_byte_feedback) { -#ifdef USE_BYTE_FEEDBACK - // this shift/lookup/xor is equivalent to 8 iterations of - // lfsr = (lfsr & 1) ? ((lfsr >> 1) ^ 0x800000000000000D) : (lfsr >> 1); - return (lfsr >> 8) ^ lfsr_byte_feedback[lfsr & 0xff]; -#else - uint32_t l = lfsr; - for(int i=0; i<8; i++) - l = __lfsr_iter_bit(l); - return l; -#endif -} - -uint32_t __lfsr_iter_word(uint32_t lfsr, uint32_t *lfsr_byte_feedback) { - uint32_t l = __lfsr_iter_byte(lfsr, lfsr_byte_feedback); - l = __lfsr_iter_byte(l, lfsr_byte_feedback); - l = __lfsr_iter_byte(l, lfsr_byte_feedback); - return __lfsr_iter_byte(l, lfsr_byte_feedback); -} - -int generate_random_buffer(int addr_first, int addr_last, uint32_t seed) { - uint32_t lfsr = seed; - for(uint32_t addr=addr_first; addr - * - * Adapted from https://github.com/russm/lfsr64 - * This is a simple 32-bit linear feedback shift register, printing - * pseudo-random bytes to stdout. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define DEFAULT_SEED 0xdeadbeef -#define USE_BYTE_FEEDBACK - -int generate_random_buffer(int addr_first, int addr_last, uint32_t seed); -int check_random_buffer(int addr_first, int addr_last, uint32_t seed); diff --git a/test/test_datamover.c b/test/test_datamover.c deleted file mode 100644 index dfe0f7d..0000000 --- a/test/test_datamover.c +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright (C) 2019-2020 ETH Zurich and University of Bologna - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Authors: Francesco Conti - */ - -#include "pmsis.h" -#include "stdio.h" -#include -#include "hal_datamover.h" -#include "lfsr32.h" - -#define DATA_SIZE (16*1024) -#define DATAMOVER_BW (256 / 8) - -static int ret_value; - -static void pe_entry(void *arg) { - - printf("Entered cluster on cluster %d core %d\n", pi_cluster_id(), pi_core_id()); - - pi_cl_team_barrier(); - - int errors = 0; - - if (pi_core_id() == 0) { - - uint8_t volatile *x = (uint8_t volatile *) pi_cl_l1_malloc(NULL, DATA_SIZE); - uint8_t volatile *y = (uint8_t volatile *) pi_cl_l1_malloc(NULL, DATA_SIZE); - generate_random_buffer((int) x, (int) x + DATA_SIZE, DEFAULT_SEED); - - // enable clock - DATAMOVER_CG_ENABLE(); - - // setup HCI - DATAMOVER_SETPRIORITY_DATAMOVER(); // priority to DATAMOVER w.r.t. cores, DMA - DATAMOVER_RESET_MAXSTALL(); // reset maximum stall - DATAMOVER_SET_MAXSTALL(8); // set maximum consecutive stall to 8 cycles for cores, DMA side - - // soft-clear DATAMOVER - DATAMOVER_WRITE_CMD(DATAMOVER_SOFT_CLEAR, DATAMOVER_SOFT_CLEAR_ALL); - for(volatile int kk=0; kk<10; kk++); - - // acquire job - int job_id = -1; - DATAMOVER_BARRIER_ACQUIRE(job_id); - - // set up datamover - DATAMOVER_WRITE_REG(DATAMOVER_REG_IN_PTR, x); - DATAMOVER_WRITE_REG(DATAMOVER_REG_OUT_PTR, y); - DATAMOVER_WRITE_REG(DATAMOVER_REG_TOT_LEN, DATA_SIZE / DATAMOVER_BW); - DATAMOVER_WRITE_REG(DATAMOVER_REG_IN_D0_LEN, DATA_SIZE / DATAMOVER_BW); - DATAMOVER_WRITE_REG(DATAMOVER_REG_IN_D0_STRIDE, DATAMOVER_BW); - DATAMOVER_WRITE_REG(DATAMOVER_REG_IN_D1_LEN, DATA_SIZE / DATAMOVER_BW); - DATAMOVER_WRITE_REG(DATAMOVER_REG_IN_D1_STRIDE, 0); - DATAMOVER_WRITE_REG(DATAMOVER_REG_IN_D2_STRIDE, 0); - DATAMOVER_WRITE_REG(DATAMOVER_REG_OUT_D0_LEN, DATA_SIZE / DATAMOVER_BW); - DATAMOVER_WRITE_REG(DATAMOVER_REG_OUT_D0_STRIDE, DATAMOVER_BW); - DATAMOVER_WRITE_REG(DATAMOVER_REG_OUT_D1_LEN, DATA_SIZE / DATAMOVER_BW); - DATAMOVER_WRITE_REG(DATAMOVER_REG_OUT_D1_STRIDE, 0); - DATAMOVER_WRITE_REG(DATAMOVER_REG_OUT_D2_STRIDE, 0); - - // commit and trigger datamover operation - DATAMOVER_WRITE_CMD(DATAMOVER_COMMIT_AND_TRIGGER, DATAMOVER_TRIGGER_CMD); - - // wait for end of computation - DATAMOVER_BARRIER(); - - // disable clock - DATAMOVER_CG_DISABLE(); - - // set priority to core side - DATAMOVER_SETPRIORITY_CORE(); - - ret_value = check_random_buffer((int) y, (int) y + DATA_SIZE, DEFAULT_SEED); - - } - pi_cl_team_barrier(); -} - -static void cluster_entry(void *arg) { - pi_cl_team_fork(0, pe_entry, 0); -} - -void test_kickoff(void *arg) -{ - struct pi_device cluster_dev; - struct pi_cluster_conf conf; - struct pi_cluster_task task; - ret_value = 0; - - pi_cluster_conf_init(&conf); - conf.id = 0; - - pi_open_from_conf(&cluster_dev, &conf); - - pi_cluster_open(&cluster_dev); - - pi_cluster_task(&task, cluster_entry, NULL); - - pi_cluster_send_task_to_cl(&cluster_dev, &task); - - pi_cluster_close(&cluster_dev); - - pmsis_exit(ret_value); -} - -int main() -{ - return pmsis_kickoff((void *)test_kickoff); -} diff --git a/verif/python/datamover_golden_model.py b/verif/python/datamover_golden_model.py new file mode 100644 index 0000000..1595783 --- /dev/null +++ b/verif/python/datamover_golden_model.py @@ -0,0 +1,224 @@ +# Copyright 2026 ETH Zurich and University of Bologna. +# Licensed under the Apache License, Version 2.0, see LICENSE for details. +# SPDX-License-Identifier: Apache-2.0 +# +# Authors: Cyrill Durrer + +import os +import math +import argparse +import numpy as np + +def data_header_format(data, elements_per_line=16): + lines = [] + for i in range(0, len(data), elements_per_line): + line_elements = data[i:i + elements_per_line] + formatted_elements = [f"0x{elem:02x}" for elem in line_elements] + if i + elements_per_line < len(data): + line = " " + ", ".join(formatted_elements) + "," + else: + line = " " + ", ".join(formatted_elements) + lines.append(line) + return lines + +def write_data_header_file(output_dir, input_tensor, output_tensor, config_params, filename="data.h"): + os.makedirs(output_dir, exist_ok=True) + filepath = os.path.join(output_dir, filename) + + input_flat = np.asarray(input_tensor, dtype=np.uint8).reshape(-1).tolist() + output_flat = np.asarray(output_tensor, dtype=np.uint8).reshape(-1).tolist() + + data_h_string = [ + "#pragma once", + "", + "#include ", + "", + "// Configuration Parameters", + f"#define DATAMOVER_MODE {config_params['datamover_mode']}", + f"#define TRANSP_MODE {config_params['transp_mode']}", + f"#define CIM_MODE {config_params['cim_mode']}", + f"#define ROW_TILE_SIZE {config_params['row_tile_size']}", + f"#define SIZE_C {config_params['size_c']}", + f"#define SIZE_M {config_params['size_m']}", + f"#define SIZE_N {config_params['size_n']}", + "", + "uint8_t golden_in [SIZE_C*SIZE_M*SIZE_N] = {", + # "PI_L1 uint8_t golden_in [SIZE_C*SIZE_M*SIZE_N] = {", # PI_L1 only for GVSoC (siracusa) + ] + data_h_string.extend(data_header_format(input_flat)) + data_h_string.extend([ + "};", + "", + "uint8_t golden_out [SIZE_C*SIZE_M*SIZE_N] = {", + # "PI_L1 uint8_t golden_out [SIZE_C*SIZE_M*SIZE_N] = {", # PI_L1 only for GVSoC (siracusa) + ]) + data_h_string.extend(data_header_format(output_flat)) + data_h_string.extend([ + "};", + "", + ]) + + with open(filepath, "w", encoding="utf-8") as file: + file.write("\n".join(data_h_string)) + + return filepath + +def cim_layout(tensor, row_tile_size, size_m, size_n): + # Input tensor shape: (size_m, size_n) + # Output shape: (1, size_m * size_n) + # Transformation: columns are grouped into tiles of row_tile_size; + # for each tile, all rows' slices are stored consecutively in the output. + + # size_m, size_n = tensor.shape + n_tiles = size_n // row_tile_size + leftover = size_n % row_tile_size + + # Reshape to (size_m, n_tiles, row_tile_size), transpose to (n_tiles, size_m, row_tile_size), flatten + parts = [tensor[:, :n_tiles * row_tile_size].reshape(size_m, n_tiles, row_tile_size).transpose(1, 0, 2).reshape(-1)] + if leftover > 0: + parts.append(tensor[:, n_tiles * row_tile_size:].reshape(-1)) + return np.concatenate(parts).reshape(1, -1) + +def cim_layout_reverse(tensor, row_tile_size, size_m, size_n): + # Input tensor shape: (1, size_m * size_n) in CIM layout + # Output shape: (size_m, size_n) in row-major layout + # size_m, size_n are the dimensions of the original row-major tensor + + n_tiles = size_n // row_tile_size + leftover = size_n % row_tile_size + flat = tensor.reshape(-1) + + # Reshape to (n_tiles, size_m, row_tile_size), transpose to (size_m, n_tiles, row_tile_size) + complete = flat[:n_tiles * size_m * row_tile_size].reshape(n_tiles, size_m, row_tile_size).transpose(1, 0, 2) + if leftover > 0: + leftover_part = flat[n_tiles * size_m * row_tile_size:].reshape(size_m, leftover) + return np.concatenate([complete.reshape(size_m, n_tiles * row_tile_size), leftover_part], axis=1) + return complete.reshape(size_m, size_n) + +def cim_layout_transpose(tensor, row_tile_size, size_m, size_n): + # Input tensor in CIM-layout (size_m and size_n are the dimensions of the original row-major tensor) + # Output tensor: transposed input tensor in CIM-layout + row_major_tensor = cim_layout_reverse(tensor, row_tile_size, size_m, size_n) + row_major_transposed = np.transpose(row_major_tensor) + output_tensor = cim_layout(row_major_transposed, row_tile_size, size_n, size_m) + return output_tensor + + +def unfold(tensor, patch_size): + # Input tensor shape: (CHANNELS, HEIGHT, WIDTH) + # Output tensor shape: (PATCH_SIZE, NUM_PATCHES, CHANNELS) + channels, height, width = tensor.shape + patch_sidelength = int(math.sqrt(patch_size)) + assert (height % patch_sidelength == 0) and (width % patch_sidelength == 0), "Height and Width must be divisible by patch sidelength" + num_patches_h = height // patch_sidelength + num_patches_w = width // patch_sidelength + num_patches = num_patches_h * num_patches_w + tensor_unfolded = np.zeros((patch_size, num_patches, channels), dtype=tensor.dtype) + for p in range(patch_size): + for h in range(num_patches_h): + for w in range(num_patches_w): + n = h * num_patches_w + w + h_idx = h * patch_sidelength + (p // patch_sidelength) + w_idx = w * patch_sidelength + (p % patch_sidelength) + tensor_unfolded[p, n, :] = tensor[:, h_idx, w_idx] + return tensor_unfolded + +def fold(tensor, patch_size, num_channels, height, width): # Parameters: output tensor dimensions (CHW) + # Input tensor shape: (PATCH_SIZE, NUM_PATCHES, CHANNELS) -- calculated from output dimensions + # Output tensor shape: (CHANNELS, HEIGHT, WIDTH) -- num_channels, height, width are the folded dimensions + patch_sidelength = int(math.sqrt(patch_size)) + # size_n = (height * width) // patch_size + assert (height % patch_sidelength == 0) and (width % patch_sidelength == 0), "Height and Width must be divisible by patch sidelength" + num_patches_h = height // patch_sidelength + num_patches_w = width // patch_sidelength + tensor_folded = np.zeros((num_channels, height, width), dtype=tensor.dtype) + for p in range(patch_size): + for h in range(num_patches_h): + for w in range(num_patches_w): + n = h * num_patches_w + w + h_idx = h * patch_sidelength + (p // patch_sidelength) + w_idx = w * patch_sidelength + (p % patch_sidelength) + tensor_folded[:, h_idx, w_idx] = tensor[p, n, :] + return tensor_folded + +def main(): + parser = argparse.ArgumentParser(description="Datamover golden model generator") + parser.add_argument("--datamover-mode", type=int, default=0, dest="datamover_mode", help="0: copy, 1: transpose, 2: CIM layout, 3: CIM layout transpose, 4: unfold, 5: fold") + parser.add_argument("--transp-mode", type=int, default=1, dest="transp_mode", help="Transpose element width: 1, 2, or 4") + parser.add_argument("--cim-mode", type=int, default=0, dest="cim_mode", help="0: row-major -> CIM-layout, 1: CIM-layout -> row-major") + parser.add_argument("--row-tile-size", type=int, default=64, dest="row_tile_size", help="CIM row tile size (inner dimension)") + parser.add_argument("--size-c", type=int, default=1, dest="size_c", help="Tensor dimension C") + parser.add_argument("--size-m", type=int, default=1, dest="size_m", help="Tensor dimension M (rows)") + parser.add_argument("--size-n", type=int, default=1, dest="size_n", help="Tensor dimension N (columns)") + parser.add_argument("--patch-size", type=int, default=4, dest="patch_size", help="MobileViT patch size (unfold/fold)") + parser.add_argument("--memory-size", type=int, default=1048576, dest="memory_size", help="TCDM size in Bytes") + parser.add_argument("--output-filename",type=str, default="data.h", dest="output", help="Output header filename") + parser.add_argument("--count", action="store_true", dest="count", help="Use counting stimuli instead of random (for debugging)") + args = parser.parse_args() + + # Check configuration: maximum tensor size limited by memory divided by three (input, output, golden) + # required_elements = 3 * args.size_c * args.size_m * args.size_n + # assert required_elements <= args.memory_size, ( + # f"MEMORY_SIZE ({args.memory_size}) is too small: requires at least " + # f"3*SIZE_C*SIZE_M*SIZE_N = {required_elements} elements" + # ) + + # Create a tensor of size (SIZE_C, SIZE_M, SIZE_N) with random or counting values + if not args.count: + input_tensor = np.random.randint(0, 256, (args.size_c, args.size_m, args.size_n), dtype=np.uint8) + else: + input_tensor = np.arange(args.size_c * args.size_m * args.size_n, dtype=np.uint8).reshape((args.size_c, args.size_m, args.size_n)) + + # Generate golden output tensor based on the selected datamover mode + if args.datamover_mode==0: + output_tensor = input_tensor.copy() + elif args.datamover_mode==1: + t = args.transp_mode + if t not in [1, 2, 4]: + raise ValueError(f"Unsupported TRANSP_MODE: {t}. Supported modes: 1, 2, 4.") + C, M, N = input_tensor.shape + assert N % t == 0, f"SIZE_N ({N}) must be divisible by transp_mode ({t})" + output_tensor = input_tensor.reshape(C, M, N // t, t).transpose(0, 2, 1, 3).reshape(C, N // t, M * t) + elif args.datamover_mode==2: + if args.cim_mode == 0: # row-major -> CIM-layout + output_tensor = cim_layout(input_tensor.reshape(args.size_m, args.size_n), args.row_tile_size, args.size_m, args.size_n) + elif args.cim_mode == 1: # CIM-layout -> row-major (reverse of mode 0) + output_tensor = cim_layout_reverse(input_tensor.reshape(args.size_m, args.size_n), args.row_tile_size, args.size_m, args.size_n) + else: + raise ValueError(f"Unsupported CIM_MODE: {args.cim_mode}") + elif args.datamover_mode==3: + input_tensor = input_tensor.reshape(args.size_m, args.size_n) + output_tensor = cim_layout_transpose(input_tensor, args.row_tile_size, args.size_m, args.size_n) + elif args.datamover_mode==4: + output_tensor = unfold(input_tensor, args.patch_size) + elif args.datamover_mode==5: + # For fold mode, generate the input as an unfolded tensor by first unfolding a counting tensor + # base_tensor = np.arange(SIZE_C * SIZE_M * SIZE_N, dtype=np.uint8).reshape((SIZE_C, SIZE_M, SIZE_N)) + unfolded_tensor = unfold(input_tensor, args.patch_size) + input_tensor = unfolded_tensor.copy() # Use the unfolded tensor as input for fold mode + output_tensor = fold(unfolded_tensor, args.patch_size, args.size_c, args.size_m, args.size_n) + else: + raise ValueError(f"Unsupported DATAMOVER_MODE: {args.datamover_mode}") + + # print("Input Tensor:") + # print(input_tensor) + # print("\nGolden Output Tensor:") + # print(output_tensor) + + print("\nSuccessfully generated golden model output tensor.") + + output_dir = os.path.join(os.path.dirname(__file__), "data") + config_params = { + "datamover_mode": args.datamover_mode, + "transp_mode": args.transp_mode, + "cim_mode": args.cim_mode, + "row_tile_size": args.row_tile_size, + "size_c": args.size_c, + "size_m": args.size_m, + "size_n": args.size_n, + } + header_file = write_data_header_file(output_dir, input_tensor, output_tensor, config_params, args.output) + print(f"\nWrote golden header to: {header_file}") + +if __name__ == "__main__": + main() diff --git a/verif/python/datamover_golden_model_numpy.py b/verif/python/datamover_golden_model_numpy.py new file mode 100644 index 0000000..e465a07 --- /dev/null +++ b/verif/python/datamover_golden_model_numpy.py @@ -0,0 +1,177 @@ +import os +import math +import numpy as np + +RANDOM_STIMULI = True # If False, stimuli are generated in a counting fashion for easier debugging + +CHANNELS = 32 +HEIGHT = 32 +WIDTH = 32 +PATCH_SIZE = 4 # MobileViT: 2x2 = 4 (square) (not tested for other patch sizes) + +# data.h generation parameters +BANDWIDTH = 512 +WORD_WIDTH = 64 +ELEM_WIDTH = 8 +MEMORY_SIZE = 131072 +DATAMOVER_MODE = 5 # 0: copy, 1: transpose, 2: CIM data layout conversion, 3: CIM layout transpose, 4: unfold (MobileViT), 5: fold (MobileViT), other values: not accepted +TRANSP_MODE = 1 +CIM_MODE = 0 +ROW_TILE_SIZE = 64 +SIZE_C = CHANNELS +SIZE_M = HEIGHT +SIZE_N = WIDTH + +def data_header_format(data, elements_per_line=16): + lines = [] + for i in range(0, len(data), elements_per_line): + line_elements = data[i:i + elements_per_line] + formatted_elements = [f"0x{elem:02x}" for elem in line_elements] + if i + elements_per_line < len(data): + line = " " + ", ".join(formatted_elements) + "," + else: + line = " " + ", ".join(formatted_elements) + lines.append(line) + return lines + + +def write_data_header_file(output_dir, input_matrix, output_matrix, config_params): + os.makedirs(output_dir, exist_ok=True) + filepath = os.path.join(output_dir, "data.h") + + input_flat = np.asarray(input_matrix, dtype=np.uint8).reshape(-1).tolist() + output_flat = np.asarray(output_matrix, dtype=np.uint8).reshape(-1).tolist() + + data_h_string = [ + "#pragma once", + "", + "#include ", + "", + "// Configuration Parameters", + f"#define BANDWIDTH {config_params['bandwidth']}", + f"#define WORD_WIDTH {config_params['word_width']}", + f"#define ELEM_WIDTH {config_params['elem_width']}", + f"#define MEMORY_SIZE {config_params['memory_size']}", + f"#define DATAMOVER_MODE {config_params['datamover_mode']}", + f"#define TRANSP_MODE {config_params['transp_mode']}", + f"#define CIM_MODE {config_params['cim_mode']}", + f"#define ROW_TILE_SIZE {config_params['row_tile_size']}", + f"#define SIZE_C {config_params['size_c']}", + f"#define SIZE_M {config_params['size_m']}", + f"#define SIZE_N {config_params['size_n']}", + "", + "PI_L1 uint8_t golden_in [SIZE_C*SIZE_M*SIZE_N] = {", # PI_L1 only for GVSoC (siracusa) + ] + data_h_string.extend(data_header_format(input_flat)) + data_h_string.extend([ + "};", + "", + "PI_L1 uint8_t golden_out [SIZE_C*SIZE_M*SIZE_N] = {", # PI_L1 only for GVSoC (siracusa) + ]) + data_h_string.extend(data_header_format(output_flat)) + data_h_string.extend([ + "};", + "", + ]) + + with open(filepath, "w", encoding="utf-8") as file: + file.write("\n".join(data_h_string)) + + return filepath + + +def unfold(tensor, patch_size): + # Input tensor shape: (CHANNELS, HEIGHT, WIDTH) + # Output tensor shape: (PATCH_SIZE, NUM_PATCHES, CHANNELS) + channels, height, width = tensor.shape + patch_sidelength = int(math.sqrt(patch_size)) + assert (height % patch_sidelength == 0) and (width % patch_sidelength == 0), "Height and Width must be divisible by patch sidelength" + num_patches_h = height // patch_sidelength + num_patches_w = width // patch_sidelength + num_patches = num_patches_h * num_patches_w + tensor_unfolded = np.zeros((patch_size, num_patches, channels), dtype=tensor.dtype) + for p in range(patch_size): + for h in range(num_patches_h): + for w in range(num_patches_w): + n = h * num_patches_w + w + h_idx = h * patch_sidelength + (p // patch_sidelength) + w_idx = w * patch_sidelength + (p % patch_sidelength) + tensor_unfolded[p, n, :] = tensor[:, h_idx, w_idx] + return tensor_unfolded + +def fold(tensor, patch_size, num_channels, height, width): # Parameters: output tensor dimensions (CHW) + # Input tensor shape: (PATCH_SIZE, NUM_PATCHES, CHANNELS) -- calculated from output dimensions + # Output tensor shape: (CHANNELS, HEIGHT, WIDTH) -- num_channels, height, width are the folded dimensions + patch_sidelength = int(math.sqrt(patch_size)) + # size_n = (height * width) // patch_size + assert (height % patch_sidelength == 0) and (width % patch_sidelength == 0), "Height and Width must be divisible by patch sidelength" + num_patches_h = height // patch_sidelength + num_patches_w = width // patch_sidelength + tensor_folded = np.zeros((num_channels, height, width), dtype=tensor.dtype) + for p in range(patch_size): + for h in range(num_patches_h): + for w in range(num_patches_w): + n = h * num_patches_w + w + h_idx = h * patch_sidelength + (p // patch_sidelength) + w_idx = w * patch_sidelength + (p % patch_sidelength) + tensor_folded[:, h_idx, w_idx] = tensor[p, n, :] + return tensor_folded + +def main(): + # Check configuration + required_elements = 3 * CHANNELS * HEIGHT * WIDTH + assert required_elements <= MEMORY_SIZE, ( + f"MEMORY_SIZE ({MEMORY_SIZE}) is too small: requires at least " + f"3*CHANNELS*HEIGHT*WIDTH = {required_elements} elements" + ) + + # Create a tensor of size (CHANNELS, HEIGHT, WIDTH) with random or counting values + if RANDOM_STIMULI: + input_tensor = np.random.randint(0, 256, (CHANNELS, HEIGHT, WIDTH), dtype=np.uint8) + else: + input_tensor = np.arange(CHANNELS * HEIGHT * WIDTH, dtype=np.uint8).reshape((CHANNELS, HEIGHT, WIDTH)) + print("Input Tensor:") + print(input_tensor) + if DATAMOVER_MODE==0: + output_tensor = input_tensor.copy() + elif DATAMOVER_MODE==1: + output_tensor = np.transpose(input_tensor, (0, 2, 1)) + # elif DATAMOVER_MODE==2: + # output_tensor = cim_layout(input_tensor, ) + # elif DATAMOVER_MODE==3: + # output_tensor = cim_transpose(input_tensor, ) + elif DATAMOVER_MODE==4: + output_tensor = unfold(input_tensor, PATCH_SIZE) + elif DATAMOVER_MODE==5: + # For fold mode, generate the input as an unfolded tensor by first unfolding a counting tensor + # base_tensor = np.arange(CHANNELS * HEIGHT * WIDTH, dtype=np.uint8).reshape((CHANNELS, HEIGHT, WIDTH)) + unfolded_tensor = unfold(input_tensor, PATCH_SIZE) + input_tensor = unfolded_tensor.copy() # Use the unfolded tensor as input for fold mode + print("\nUnfolded Tensor (input for fold mode):") + print(unfolded_tensor) + output_tensor = fold(unfolded_tensor, PATCH_SIZE, CHANNELS, HEIGHT, WIDTH) + else: + raise ValueError(f"Unsupported DATAMOVER_MODE: {DATAMOVER_MODE}") + + print("\nOutput Tensor:") + print(output_tensor) + + output_dir = os.path.join(os.path.dirname(__file__), "generated") + config_params = { + "bandwidth": BANDWIDTH, + "word_width": WORD_WIDTH, + "elem_width": ELEM_WIDTH, + "memory_size": MEMORY_SIZE, + "datamover_mode": DATAMOVER_MODE, + "transp_mode": TRANSP_MODE, + "cim_mode": CIM_MODE, + "row_tile_size": ROW_TILE_SIZE, + "size_c": SIZE_C, + "size_m": SIZE_M, + "size_n": SIZE_N, + } + header_file = write_data_header_file(output_dir, input_tensor, output_tensor, config_params) + print(f"\nWrote golden header to: {header_file}") + +if __name__ == "__main__": + main() diff --git a/verif/python/datamover_microarchitectural_model.py b/verif/python/datamover_microarchitectural_model.py new file mode 100644 index 0000000..f014461 --- /dev/null +++ b/verif/python/datamover_microarchitectural_model.py @@ -0,0 +1,301 @@ +import random +import argparse +import os +import math +from unittest import case +import numpy as np +from dataclasses import dataclass + +# OUTDATED: replaced by datamover_golden_model.py + +RANDOM_STIMULI = False # If False, stimuli are generated in a counting fashion + +# HW Parameters +BANDWIDTH = 32 # in bits +WORD_WIDTH = 32 # in bits +ELEM_WIDTH = 8 # in bits +BANDWIDTH_ELEMS = BANDWIDTH // ELEM_WIDTH +WORDWIDTH_ELEMS = WORD_WIDTH // ELEM_WIDTH + +# Operation Modes +DATAMOVER_MODE = 4 # 0: copy, 1: transpose, 2: CIM data layout conversion, 3: CIM layout transpose, 4: unfold +TRANSP_MODE = 1 # 1 = 1 elem, 2 = 2 elem, 4 = 4 elem, other values: not accepted +CIM_ROWTILE_SIZE = 2 # in elements (64 in Konark) +PATCH_SIZE = 4 # in elements (2x2 = 4 in MobileViT) + +CHANNELS = 2 +HEIGHT = 8 +WIDTH = 8 + +@dataclass +class addrgen_ctrl_c: + D0_LENGTH: int = HEIGHT * WIDTH + D0_STRIDE: int = BANDWIDTH_ELEMS + D1_LENGTH: int = 0 + D1_STRIDE: int = 0 + D2_LENGTH: int = 0 + D2_STRIDE: int = 0 + D3_LENGTH: int = 0 + D3_STRIDE: int = 0 + D4_STRIDE: int = 0 + TOT_LENGTH: int = math.ceil(HEIGHT * WIDTH / BANDWIDTH_ELEMS) + # DIM_ENABLE: str = "0000" # !!! REVERSE ORDER than in SystemVerilog testbench / config.mk + +class addrgen_c: + def __init__(self, start, addrgen_ctrl): + self.addr = start + self.addrgen_ctrl = addrgen_ctrl + + # loop indices (counters) + self.d0 = 0 + self.d1 = 0 + self.d2 = 0 + self.d3 = 0 + self.d4 = 0 + + def tick(self): + out = int(self.addr) + # ---- D0 ---- + self.d0 += 1 + self.addr += self.addrgen_ctrl.D0_STRIDE + if self.d0 < self.addrgen_ctrl.D0_LENGTH: + return out + # wrap D0 + self.d0 = 0 + self.addr -= self.addrgen_ctrl.D0_LENGTH * self.addrgen_ctrl.D0_STRIDE + # ---- D1 ---- + self.d1 += 1 + self.addr += self.addrgen_ctrl.D1_STRIDE + if self.d1 < self.addrgen_ctrl.D1_LENGTH: + return out + self.d1 = 0 + self.addr -= self.addrgen_ctrl.D1_LENGTH * self.addrgen_ctrl.D1_STRIDE + # ---- D2 ---- + self.d2 += 1 + self.addr += self.addrgen_ctrl.D2_STRIDE + if self.d2 < self.addrgen_ctrl.D2_LENGTH: + return out + self.d2 = 0 + self.addr -= self.addrgen_ctrl.D2_LENGTH * self.addrgen_ctrl.D2_STRIDE + # ---- D3 ---- + self.d3 += 1 + self.addr += self.addrgen_ctrl.D3_STRIDE + if self.d3 < self.addrgen_ctrl.D3_LENGTH: + return out + self.d3 = 0 + self.addr -= self.addrgen_ctrl.D3_LENGTH * self.addrgen_ctrl.D3_STRIDE + # ---- D4 ---- + self.d4 += 1 + self.addr += self.addrgen_ctrl.D4_STRIDE + # if self.d4 < self.addrgen_ctrl.D4_LENGTH: + # return self.addr + # final wrap (optional) + # self.d4 = 0 + # self.addr = self.start + return out + +def datamover_execute(input_memory, input_channels, input_height, input_width, addrgen_in_ctrl, addrgen_out_ctrl): + nof_input_tiles = math.ceil(addrgen_in_ctrl.TOT_LENGTH / BANDWIDTH_ELEMS) + elem_matrix = np.zeros((BANDWIDTH_ELEMS, BANDWIDTH_ELEMS), dtype=input_memory.dtype) + output_memory = np.zeros_like(input_memory) + addrgen_in = addrgen_c(0, addrgen_in_ctrl) + addrgen_out = addrgen_c(0, addrgen_out_ctrl) + + for input_tile_idx in range(nof_input_tiles): + # Fill the element matrix (internal buffer) + for i in range(BANDWIDTH_ELEMS): + in_addr = addrgen_in.tick() + # print(f"Reading input address: {in_addr}") + elem_matrix[i, :] = input_memory[in_addr:in_addr + BANDWIDTH_ELEMS] + print(f"elem_matrix (tile {input_tile_idx}):\n{elem_matrix}\n") + # Write out the element matrix to output memory (rearranged as needed) + for i in range(BANDWIDTH_ELEMS): + out_addr = addrgen_out.tick() + if DATAMOVER_MODE==0 or DATAMOVER_MODE==2: + output_memory[out_addr:out_addr + BANDWIDTH_ELEMS] = elem_matrix[i, :] + elif DATAMOVER_MODE==1: + if TRANSP_MODE==1: + output_memory[out_addr:out_addr + BANDWIDTH_ELEMS] = elem_matrix[:, i] + else: + print(f"Unsupported TRANSP_MODE {TRANSP_MODE} in datamover_execute") + elif DATAMOVER_MODE==4: # ToDo(cdurrer) + for x in range(BANDWIDTH_ELEMS): + for y in range(BANDWIDTH_ELEMS): + output_memory[out_addr + x * BANDWIDTH_ELEMS + y] = elem_matrix[x, y] + else: + print(f"Unsupported DATAMOVER_MODE {DATAMOVER_MODE} in datamover_execute") + return output_memory + +def cim_copy_config(input_channels, input_height, input_width): + total_elems = input_channels * input_height * input_width + total_accesses = math.ceil(total_elems / BANDWIDTH_ELEMS) + addrgen_in_ctrl = addrgen_ctrl_c( + D0_LENGTH=total_accesses, + D0_STRIDE=BANDWIDTH_ELEMS, + D1_LENGTH=0, + D1_STRIDE=0, + D2_LENGTH=0, + D2_STRIDE=0, + D3_LENGTH=0, + D3_STRIDE=0, + D4_STRIDE=0, + TOT_LENGTH=total_accesses, + # DIM_ENABLE="0000", + ) + addrgen_out_ctrl = addrgen_ctrl_c( + D0_LENGTH=total_accesses, + D0_STRIDE=BANDWIDTH_ELEMS, + D1_LENGTH=0, + D1_STRIDE=0, + D2_LENGTH=0, + D2_STRIDE=0, + D3_LENGTH=0, + D3_STRIDE=0, + D4_STRIDE=0, + TOT_LENGTH=total_accesses, + # DIM_ENABLE="0000", + ) + return addrgen_in_ctrl, addrgen_out_ctrl + +def transpose_config(input_height, input_width, transp_mode): + m_aligned = ((input_height + BANDWIDTH_ELEMS - 1) // BANDWIDTH_ELEMS) * BANDWIDTH_ELEMS + n_aligned = ((input_width + BANDWIDTH_ELEMS - 1) // BANDWIDTH_ELEMS) * BANDWIDTH_ELEMS + addrgen_in_ctrl = addrgen_ctrl_c( + D0_LENGTH=m_aligned, + D0_STRIDE=input_width, + D1_LENGTH=n_aligned // BANDWIDTH_ELEMS, + D1_STRIDE=BANDWIDTH_ELEMS, + D2_LENGTH=0, + D2_STRIDE=0, + D3_LENGTH=0, + D3_STRIDE=0, + D4_STRIDE=0, + TOT_LENGTH=m_aligned * (n_aligned // BANDWIDTH_ELEMS), + # DIM_ENABLE="0001", + ) + write_d0_stride = input_height * transp_mode + addrgen_out_ctrl = addrgen_ctrl_c( + D0_LENGTH=BANDWIDTH_ELEMS // transp_mode, + D0_STRIDE=write_d0_stride, + D1_LENGTH=write_d0_stride // BANDWIDTH_ELEMS, + D1_STRIDE=BANDWIDTH_ELEMS, + D2_LENGTH=0, + D2_STRIDE=write_d0_stride * (BANDWIDTH_ELEMS // transp_mode), + D3_LENGTH=0, + D3_STRIDE=0, + D4_STRIDE=0, + TOT_LENGTH=m_aligned * (n_aligned // BANDWIDTH_ELEMS), + # DIM_ENABLE="0011", + ) + return addrgen_in_ctrl, addrgen_out_ctrl + +def cim_layout_config(input_height, input_width, rowtile_size): + addrgen_in_ctrl = addrgen_ctrl_c( + D0_LENGTH=rowtile_size // BANDWIDTH_ELEMS, + D0_STRIDE=BANDWIDTH_ELEMS, + D1_LENGTH=input_height, + D1_STRIDE=input_width, + D2_LENGTH=input_width // rowtile_size, + D2_STRIDE=rowtile_size, + D3_LENGTH=0, + D3_STRIDE=0, + D4_STRIDE=0, + TOT_LENGTH=(rowtile_size // BANDWIDTH_ELEMS) * input_height * (input_width // rowtile_size), + # DIM_ENABLE="0011", + ) + addrgen_out_ctrl = addrgen_ctrl_c( + D0_LENGTH=(rowtile_size // BANDWIDTH_ELEMS) * input_height, + D0_STRIDE=BANDWIDTH_ELEMS, + D1_LENGTH=input_width // rowtile_size, + D1_STRIDE=((rowtile_size // BANDWIDTH_ELEMS) * input_height) * BANDWIDTH_ELEMS, + D2_LENGTH=0, + D2_STRIDE=0, + D3_LENGTH=0, + D3_STRIDE=0, + D4_STRIDE=0, + TOT_LENGTH=(rowtile_size // BANDWIDTH_ELEMS) * input_height * (input_width // rowtile_size), + # DIM_ENABLE="0001", + ) + return addrgen_in_ctrl, addrgen_out_ctrl + +def unfold_config(input_channels, input_height, input_width, patch_size): # TODO(cdurrer): verify correctness + patch_sidelength = int(math.sqrt(patch_size)) + num_patches_h = input_height // patch_sidelength + num_patches_w = input_width // patch_sidelength + num_patches = num_patches_h * num_patches_w + total_elems = input_channels * input_height * input_width + total_accesses = math.ceil(total_elems / BANDWIDTH_ELEMS) + addrgen_in_ctrl = addrgen_ctrl_c( + D0_LENGTH=CHANNELS, + D0_STRIDE=HEIGHT * WIDTH, + D1_LENGTH=HEIGHT, + D1_STRIDE=WIDTH, + D2_LENGTH=math.ceil(WIDTH / BANDWIDTH_ELEMS), + D2_STRIDE=BANDWIDTH_ELEMS, + D3_LENGTH=0, + D3_STRIDE=0, + D4_STRIDE=0, + TOT_LENGTH=total_accesses, + # DIM_ENABLE="0000", + ) + addrgen_out_ctrl = addrgen_ctrl_c( + D0_LENGTH=patch_size, + D0_STRIDE=input_channels, + D1_LENGTH=num_patches, + D1_STRIDE=patch_size * input_channels, + D2_LENGTH=0, + D2_STRIDE=0, + D3_LENGTH=0, + D3_STRIDE=0, + D4_STRIDE=0, + TOT_LENGTH=total_accesses, + # DIM_ENABLE="0011", + ) + return addrgen_in_ctrl, addrgen_out_ctrl + + +def main(): + # Create a tensor of size (CHANNELS, HEIGHT, WIDTH) with random or counting values + if RANDOM_STIMULI: + input_tensor = np.random.randint(0, 256, (CHANNELS, HEIGHT, WIDTH), dtype=np.uint8) + else: + input_tensor = np.arange(CHANNELS * HEIGHT * WIDTH, dtype=np.uint8).reshape(CHANNELS, HEIGHT, WIDTH) + print(f"Input Tensor:\n{input_tensor}\n") + input_memory = input_tensor.flatten() + output_memory = np.zeros_like(input_memory) + print(f"Input Memory (flat):\n{input_memory}\n") + if DATAMOVER_MODE==0: + print("\nDATAMOVER_MODE 0: Copy\n") + (addrgen_in_ctrl, addrgen_out_ctrl) = cim_copy_config(CHANNELS, HEIGHT, WIDTH) + output_memory = datamover_execute(input_memory, CHANNELS, HEIGHT, WIDTH, addrgen_in_ctrl, addrgen_out_ctrl) + output_matrix = output_memory.reshape(CHANNELS, HEIGHT, WIDTH) + elif DATAMOVER_MODE==1: + print(f"\nDATAMOVER_MODE 1: Transpose\n") + (addrgen_in_ctrl, addrgen_out_ctrl) = transpose_config(HEIGHT, WIDTH, TRANSP_MODE) + for channel in range(CHANNELS): + channel_transposed_flat = datamover_execute(input_memory[channel*HEIGHT*WIDTH:(channel+1)*HEIGHT*WIDTH], 1, HEIGHT, WIDTH, addrgen_in_ctrl, addrgen_out_ctrl) + output_memory[channel*HEIGHT*WIDTH:(channel+1)*HEIGHT*WIDTH] = channel_transposed_flat + # output_memory = datamover_execute(input_memory, HEIGHT, WIDTH, addrgen_in_ctrl, addrgen_out_ctrl) + output_matrix = output_memory.reshape(CHANNELS, WIDTH * TRANSP_MODE, HEIGHT // TRANSP_MODE) + elif DATAMOVER_MODE==2: + print(f"\nDATAMOVER_MODE 2: CIM Data Layout Conversion\n") + (addrgen_in_ctrl, addrgen_out_ctrl) = cim_layout_config(HEIGHT, WIDTH, CIM_ROWTILE_SIZE) + for channel in range(CHANNELS): + channel_transposed_flat = datamover_execute(input_memory[channel*HEIGHT*WIDTH:(channel+1)*HEIGHT*WIDTH], 1, HEIGHT, WIDTH, addrgen_in_ctrl, addrgen_out_ctrl) + output_memory[channel*HEIGHT*WIDTH:(channel+1)*HEIGHT*WIDTH] = channel_transposed_flat + output_matrix = output_memory.reshape(CHANNELS, WIDTH // CIM_ROWTILE_SIZE, HEIGHT * CIM_ROWTILE_SIZE) + # elif DATAMOVER_MODE==3: + # output_tensor = cim_transpose(input_memory, ) + elif DATAMOVER_MODE==4: + print(f"\nDATAMOVER_MODE 4: Unfold\n") + (addrgen_in_ctrl, addrgen_out_ctrl) = unfold_config(CHANNELS, HEIGHT, WIDTH, PATCH_SIZE) + output_memory = datamover_execute(input_memory, CHANNELS, HEIGHT, WIDTH, addrgen_in_ctrl, addrgen_out_ctrl) + output_matrix = output_memory.reshape(PATCH_SIZE, int((HEIGHT*WIDTH) / PATCH_SIZE), CHANNELS) # PNC format + else: + raise ValueError(f"Unsupported DATAMOVER_MODE: {DATAMOVER_MODE}") + + print(f"\nOutput Memory (flat):\n{output_memory}\n") + print(f"\nOutput Matrix:\n{output_matrix}\n") + +if __name__ == "__main__": + main() diff --git a/verif/python/generate_stimuli.py b/verif/python/generate_stimuli.py index 1253c4a..4d6b724 100644 --- a/verif/python/generate_stimuli.py +++ b/verif/python/generate_stimuli.py @@ -1,41 +1,122 @@ import random import argparse import os +import math -def generate_random_hex_32bit(size): - """Generate random WORD_SIZE_BITS hex values.""" - def ceildiv(a, b): - return -(a // -b) - hex_length = ceildiv(WORD_SIZE_BITS, 4) # Each hex digit represents 4 bits - return [f"{random.randint(0, 2**WORD_SIZE_BITS - 1):0{hex_length}X}" for _ in range(size)] +# OUTDATED -def generate_addresses(start, d0_stride, d0_length, d1_stride, d1_length, transactions, N): +RANDOM_STIMULI = False # If False, stimuli are generated in a counting fashion + +def extract_elements_from_word(word, word_width, elem_width): + """Extract elements from a word based on the specified widths.""" + word_int = int(word, 16) # Convert hex string to integer + elements = [] + for i in range(word_width // elem_width): + # Extract each element using shift and mask + elem_val = (word_int >> (i * elem_width)) & ((1 << elem_width) - 1) + elements.append(elem_val) + return elements + +def convert_memory_to_vector(memory, elem_width, word_width): + """Convert memory (list of hex words) to a flat vector of elements.""" + vector = [] + for word in memory: + elements = extract_elements_from_word(word, word_width, elem_width) + vector.extend(elements) + return vector + +def data_header_format(data, elements_per_line=16): + lines = [] + for i in range(0, len(data), elements_per_line): + line_elements = data[i:i + elements_per_line] + formatted_elements = [f"0x{elem:02x}" for elem in line_elements] + # Add comma except for the last element + if i + elements_per_line < len(data): + line = " " + ", ".join(formatted_elements) + "," + else: + # Last line - no trailing comma + line = " " + ", ".join(formatted_elements) + lines.append(line) + return lines + +def generate_random_hex(size, word_width): + """Generate random word_width hex values.""" + hex_length = math.ceil(word_width / 4) # Each hex digit represents 4 bits + return [f"{random.randint(0, 2**word_width - 1):0{hex_length}X}" for _ in range(size)] + +def generate_counting_hex(size, elem_width, word_width): """ - Generate addresses ensuring total transactions match, processing N words per transaction. - Strides are already adjusted with `>> elem_offset_bit` to align with word-based addressing. + Generate counting series of hex values with specified element and word widths. + Each word contains multiple elements in little-endian format. """ - addresses = [] - addr = start - count = 0 - - for d1 in range(d1_length): - addr_d1 = addr + d1 * d1_stride - for d0 in range(d0_length): - addr_d0 = addr_d1 + d0 * d0_stride - if addr_d0 + N <= MEMORY_SIZE: # Ensure full block fits - addresses.append([addr_d0 + i for i in range(N)]) # Read/Write N words - count += 1 - if count == transactions: # Stop when enough transactions are generated - return addresses - - return addresses - -def update_memory(memory, write_addresses, extracted_data): - """Update memory with extracted data using generated write addresses, processing N words at a time.""" - for addr_block, data_block in zip(write_addresses, extracted_data): - for addr, value in zip(addr_block, data_block): - if addr < len(memory): - memory[addr] = value # Write N words at a time + elems_per_word = word_width // elem_width + result = [] + for i in range(size): + word_val = 0 + for j in range(elems_per_word): + elem_val = (i * elems_per_word + j) & ((1 << elem_width) - 1) + word_val |= (elem_val << (j * elem_width)) + result.append(f"{word_val:0{word_width // 4}X}") # Format as hex string + return result + +def pack_elements_to_word(elements, elem_width, word_width): + """Pack multiple elements into a single word.""" + elems_per_word = word_width // elem_width + word_val = 0 + for i, elem in enumerate(elements[:elems_per_word]): # Take only what fits in a word + word_val |= (elem << (i * elem_width)) + return f"{word_val:0{word_width // 4}X}" + +def matrix_to_hex_words(matrix, elem_width, word_width): + """Convert a matrix of elements to a list of hex words.""" + elems_per_word = word_width // elem_width + hex_words = [] + matrix_flat = sum(matrix, []) + for i in range(math.ceil(len(matrix_flat) / elems_per_word)): + hex_word = pack_elements_to_word(matrix_flat[i*elems_per_word:i*elems_per_word+elems_per_word], elem_width, word_width) + hex_words.append(hex_word) + return hex_words + +def write_element_to_memory(element, memory, elem_width, word_width, address): + """Write a single element back to memory at specified address.""" + elems_per_word = word_width // elem_width + word_index = address // elems_per_word + elem_index = address % elems_per_word + existing_word = memory[word_index] + existing_elements = extract_elements_from_word(existing_word, word_width, elem_width) + existing_elements[elem_index] = element + hex_word = pack_elements_to_word(existing_elements, elem_width, word_width) + memory[word_index] = hex_word + return memory + +def write_matrix_to_memory(matrix, memory, elem_width, word_width, write_base_addr): + """Write output matrix back to memory at specified base address.""" + matrix_flat = sum(matrix, []) + matrix_elems = len(matrix_flat) + for i in range(matrix_elems): + address = write_base_addr + i + element = matrix_flat[i] + memory = write_element_to_memory(element, memory, elem_width, word_width, address) + return memory + +# ToDo(cdurrer): obsolete, delete? +def matrix_to_hex_words_word_aligned(matrix, elem_width, word_width): + """Convert a matrix of elements to a list of hex words - aligned to word boundaries by zero-padding.""" + elems_per_word = word_width // elem_width + hex_words = [] + for row in matrix: + # Process each row, grouping elements into words + for i in range(0, len(row), elems_per_word): + elements_for_word = row[i:i + elems_per_word] + + # Pad with zeros if the row doesn't fill a complete word + while len(elements_for_word) < elems_per_word: + elements_for_word.append(0) + + # Pack elements into a word + hex_word = pack_elements_to_word(elements_for_word, elem_width, word_width) + hex_words.append(hex_word) + return hex_words def write_file(output_dir, filename, content): """Write list content to a file.""" @@ -44,92 +125,258 @@ def write_file(output_dir, filename, content): with open(filepath, "w") as file: file.write("\n".join(content) + "\n") +def write_data_header_file(output_dir, input_matrix, output_matrix, config_params): + """Write input and output matrices to a C header file with configuration parameters.""" + os.makedirs(output_dir, exist_ok=True) # Ensure directory exists + filepath = os.path.join(output_dir, "data.h") + + input_flat = [elem for row in input_matrix for elem in row] + output_flat = [elem for row in output_matrix for elem in row] + + data_h_string = [ + "#pragma once", + "", + "#include ", + "", + "// Configuration Parameters", + # f"#define READ_BASE_ADDR {config_params['read_base_addr']}", + # f"#define WRITE_BASE_ADDR {config_params['write_base_addr']}", + f"#define BANDWIDTH {config_params['bandwidth']}", + f"#define WORD_WIDTH {config_params['word_width']}", + f"#define ELEM_WIDTH {config_params['elem_width']}", + f"#define MEMORY_SIZE {config_params['memory_size']}", + # f"#define MISALIGNED_ACCESSES {config_params['misaligned_accesses']}", + f"#define DATAMOVER_MODE {config_params['datamover_mode']}", + f"#define TRANSP_MODE {config_params['transp_mode']}", + f"#define CIM_MODE {config_params['cim_mode']}", + f"#define ROW_TILE_SIZE {config_params['row_tile_size']}", + f"#define SIZE_C {config_params['num_channels']}", + f"#define SIZE_M {config_params['size_m']}", + f"#define SIZE_N {config_params['size_n']}", + "", + "uint8_t golden_in [SIZE_C*SIZE_M*SIZE_N] = {", + ] + data_h_string.extend(data_header_format(input_flat)) + data_h_string.extend([ + "};", + "", + "uint8_t golden_out [SIZE_C*SIZE_M*SIZE_N] = {", + ]) + data_h_string.extend(data_header_format(output_flat)) + data_h_string.extend([ + "};", + "" + ]) + with open(filepath, "w") as file: + file.write("\n".join(data_h_string)) + return + +def transpose(matrix, size_m, size_n, transp_mode): + transposed = [[0 for _ in range(size_m * transp_mode)] for _ in range(size_n // transp_mode)] + for d1 in range(size_m): + for d0 in range(size_n // transp_mode): + for i in range(transp_mode): + # print(f"Transposing element [{d1}][{(d0*transp_mode)+i}] to [{d0}][{(d1*transp_mode)+i}]") + transposed[d0][(d1*transp_mode)+i] = matrix[d1][(d0*transp_mode)+i] + return transposed + + +def cim_layout(matrix, size_m, size_n, row_tile_size, word_width_elems): + complete_n_tiles = size_n // row_tile_size + leftover_columns = size_n % row_tile_size + leftover_words = math.ceil(leftover_columns / word_width_elems) + out_words = (size_m * size_n) // word_width_elems + words_per_tile = row_tile_size // word_width_elems + # cim_matrix is a 2D list with a single row to represent the flattened layout + cim_matrix = [[0] * (size_m * size_n)] + + for d2 in range(complete_n_tiles): + for d1 in range(size_m): + chunk = matrix[d1][d2*(row_tile_size):(d2*row_tile_size+row_tile_size)] + for i in range(words_per_tile): + index = d2*size_m*row_tile_size + d1*row_tile_size + i*word_width_elems + cim_matrix[0][index : index + word_width_elems] = chunk[i*word_width_elems:(i*word_width_elems)+word_width_elems] + + if (leftover_columns > 0): + d2 = complete_n_tiles + for d1 in range(size_m): + chunk = matrix[d1][d2*(row_tile_size):(d2*row_tile_size+leftover_columns)] + for i in range(leftover_words): + index = d2*size_m*row_tile_size + d1*leftover_columns + i*leftover_columns + cim_matrix[0][index : index + leftover_columns] = chunk[i*leftover_columns:(i*leftover_columns)+leftover_columns] + return cim_matrix + +def cim_layout_reverse(matrix, size_m, size_n, row_tile_size, word_width_elems): # dimensions of original row-major layout are used + complete_n_tiles = size_n // row_tile_size + leftover_columns = size_n % row_tile_size + cim_matrix = [[0] * (size_m * size_n)] + flattened_input = [elem for row in matrix for elem in row] + for d1 in range(complete_n_tiles): + for d0 in range(size_m): + input_index = d1*size_m*row_tile_size + d0*row_tile_size + output_index = d0*size_n + d1*row_tile_size + cim_matrix[0][output_index : output_index + row_tile_size] = flattened_input[input_index : input_index + row_tile_size] + print(f"Processing tile {d1}, row {d0}: input index {input_index} to output index {output_index}") + + if (leftover_columns > 0): + for d0 in range (size_m): + input_index = complete_n_tiles*size_m*row_tile_size + d0*leftover_columns + output_index = complete_n_tiles*row_tile_size +d0*size_n + cim_matrix[0][output_index : output_index + leftover_columns] = flattened_input[input_index : input_index + leftover_columns] + print(f"Processing leftover columns for row {d0}: input index {input_index} to output index {output_index}") + return cim_matrix + def main(): - # Parse command-line arguments + # Parse command-line arguments parser = argparse.ArgumentParser(description="Memory Read/Write Simulation with Word-Aligned Strides") - parser.add_argument("--mem_size", type=int, default=0x30, help="Memory size in entries") + parser.add_argument("--mem_size", type=int, default=0x30, help="Memory size in words") parser.add_argument("--read_base_addr", type=int, default=0x00, help="Base address for read operations") parser.add_argument("--write_base_addr", type=int, default=0x20, help="Base address for write operations") - parser.add_argument("--read_d0_stride", type=int, default=4, help="Stride for d0 read (in bytes)") - parser.add_argument("--read_d1_stride", type=int, default=16, help="Stride for d1 read (in bytes)") - parser.add_argument("--read_d0_length", type=int, default=4, help="Length for d0 read") - parser.add_argument("--read_d1_length", type=int, default=4, help="Length for d1 read") - parser.add_argument("--write_d0_stride", type=int, default=4, help="Stride for d0 write (in bytes)") - parser.add_argument("--write_d1_stride", type=int, default=16, help="Stride for d1 write (in bytes)") - parser.add_argument("--write_d0_length", type=int, default=4, help="Length for d0 write") - parser.add_argument("--write_d1_length", type=int, default=4, help="Length for d1 write") parser.add_argument("--bandwidth_bits", type=int, default=4, help="Number of bits per transaction") parser.add_argument("--num_elem_word", type=int, default=4, help="Number of elements in a memory bank word") parser.add_argument("--elem_width", type=int, default=8, help="Width of each element (in bits)") + # parser.add_argument("--misaligned_accesses", type=int, default=0, help="Enable misaligned accesses (0=disabled, 1=enabled)") + parser.add_argument("--datamover_mode", type=int, default=0, help="Datamover mode (0=normal, 1=CIM)") parser.add_argument("--transp_mode", type=int, default=0, help="Transposition mode (3'b000 = none, 3'b001 = 1 elem, 3'b010 = 2 elem, 3'b100 = 4 elem)") + parser.add_argument("--cim_mode", type=int, default=0, help="CIM mode (0=normal, 1=CIM)") + parser.add_argument("--row_tile_size", type=int, default=4, help="Row tile size") + parser.add_argument("--num_channels", type=int, default=1, help="Number of channels") + parser.add_argument("--size_m", type=int, default=64, help="Matrix height in elements") + parser.add_argument("--size_n", type=int, default=64, help="Matrix width in elements") parser.add_argument("--output_dir", type=str, default="output", help="Directory for storing output files") args = parser.parse_args() - global MEMORY_SIZE - global WORD_SIZE_BITS + BANDWIDTH_ALIGNED = args.bandwidth_bits #- (args.misaligned_accesses * (args.elem_width * args.num_elem_word)) MEMORY_SIZE = args.mem_size # Set global memory size - WORD_SIZE_BITS = args.num_elem_word * args.elem_width # Set global word size in bits + BANDWIDTH_ELEMS = BANDWIDTH_ALIGNED // args.elem_width + + ELEM_WIDTH = args.elem_width + WORD_WIDTH = args.num_elem_word * args.elem_width + READ_BASE_ADDR = args.read_base_addr + WRITE_BASE_ADDR = args.write_base_addr + TRANSP_MODE = args.transp_mode + TENSOR_SIZE_C = args.num_channels + TENSOR_SIZE_N = args.size_n + TENSOR_SIZE_M = args.size_m + TOT_LENGTH = (args.size_m * args.size_n) // BANDWIDTH_ELEMS + OUTPUT_DIR = args.output_dir + + if MEMORY_SIZE < ((TENSOR_SIZE_C * TENSOR_SIZE_N * TENSOR_SIZE_M * ELEM_WIDTH // WORD_WIDTH) * 2): + raise ValueError(f"MEMORY_SIZE ({MEMORY_SIZE}) is too small for the given matrix size " + f"({TENSOR_SIZE_C}x{TENSOR_SIZE_M}x{TENSOR_SIZE_N}) and element width ({ELEM_WIDTH})") # num_elem_word must be power of two and greater than zero if args.num_elem_word & (args.num_elem_word - 1) != 0 or args.num_elem_word <= 0: - raise ValueError("num_elem_word must be a power of two and greater than zero.") + raise ValueError("[GM] num_elem_word must be a power of two and greater than zero.") # bandwidth width must be a multiple of word size - if args.bandwidth_bits % WORD_SIZE_BITS != 0: - raise ValueError("bandwidth_bits must be a multiple of the word size (num_elem_word * elem_width).") - # no transposition currently supported - if args.transp_mode != 0: - raise NotImplementedError("Transposition modes other than 'none' are not currently supported.") - - bandwidth_N = args.bandwidth_bits // WORD_SIZE_BITS - - # Step 1: Generate initial memory - memory = generate_random_hex_32bit(MEMORY_SIZE) - - # Convert addresses from element-addressing (e.g., byte-addressing) to word-addressing - elem_offset_bit = (args.num_elem_word).bit_length() - 1 - - args.write_base_addr = args.write_base_addr >> elem_offset_bit - args.read_base_addr = args.read_base_addr >> elem_offset_bit - args.write_d0_stride = args.write_d0_stride >> elem_offset_bit - args.write_d1_stride = args.write_d1_stride >> elem_offset_bit - args.read_d0_stride = args.read_d0_stride >> elem_offset_bit - args.read_d1_stride = args.read_d1_stride >> elem_offset_bit - - # Step 2: Generate read addresses (Word-aligned) - read_transactions = args.read_d0_length * args.read_d1_length - read_addresses = generate_addresses( - args.read_base_addr, args.read_d0_stride, args.read_d0_length, - args.read_d1_stride, args.read_d1_length, read_transactions, bandwidth_N - ) - - # Step 3: Extract memory values based on read addresses - extracted_data = [[memory[addr] for addr in block] for block in read_addresses if all(addr < MEMORY_SIZE for addr in block)] - - # Step 4: Save initial memory - write_file(args.output_dir, "initial_memory.txt", memory) - - # Step 5: Save debug info (addresses read and values extracted) - debug_info = [ - f"Read Block: {', '.join(hex(addr) for addr in block)} -> Data: {', '.join(memory[addr] for addr in block)}" - for block in read_addresses if all(addr < MEMORY_SIZE for addr in block) - ] - write_file(args.output_dir, "debug_values.txt", debug_info) + if BANDWIDTH_ALIGNED % WORD_WIDTH != 0: + raise ValueError("[GM] BANDWIDTH_ALIGNED must be a multiple of the word size (num_elem_word * elem_width).") + # # bandwidth width must be a multiple of word size + # if ((TENSOR_SIZE_N * ELEM_WIDTH) < BANDWIDTH_ALIGNED): + # raise ValueError("[GM] Matrix width (N) in bits must be at least as large as BANDWIDTH_ALIGNED.") + # read_tot_length must not exceed 12-bit register capacity (4096) + + # # BANDWIDTH_ALIGNED must be a power of 2 + # if ((BANDWIDTH_ALIGNED & (BANDWIDTH_ALIGNED - 1)) != 0) or (BANDWIDTH_ALIGNED < WORD_WIDTH): + # raise ValueError(f"[GM] BANDWIDTH_ALIGNED ({BANDWIDTH_ALIGNED}) must be a power of 2 and greater than the WORD_SIZE ({WORD_WIDTH}).") + + # if ((TOT_LENGTH >= 4096) & (args.datamover_mode != 0)): + # raise ValueError("[GM] TOT_LENGTH (MxN / BW_ELEM) must be less than 4096 in transpose and CIM modes (12-bit register limit).") + + + if(args.datamover_mode == 1): # transpose mode + # transp_mode must be valid (1=1elem, 2=2elem, 4=4elem) + if args.transp_mode not in [1, 2, 4]: + raise ValueError("[GM] transp_mode must be 1 (1 elem), 2 (2 elem), or 4 (4 elem).") + if (TENSOR_SIZE_N % args.transp_mode) != 0: + raise ValueError(f"[GM] Matrix width N ({TENSOR_SIZE_N}) must be a multiple of transp_mode ({args.transp_mode}).") + + print(f"Memory Size: {MEMORY_SIZE} entries") + print(f"Word Size: {WORD_WIDTH} bits") + + # memory = generate_counting_hex(MEMORY_SIZE, ELEM_WIDTH, WORD_WIDTH) + if RANDOM_STIMULI: + memory = generate_random_hex(MEMORY_SIZE, WORD_WIDTH) # for testing + else: + memory = generate_counting_hex(MEMORY_SIZE, ELEM_WIDTH, WORD_WIDTH) # for debugging + + write_file(OUTPUT_DIR, "initial_memory.txt", memory) + + # Convert memory to flat vector + memory_flat = convert_memory_to_vector(memory, ELEM_WIDTH, WORD_WIDTH) + + # Extract matrix (read dimensions) from memory + input_matrix = [[0 for _ in range(TENSOR_SIZE_N)] for _ in range(TENSOR_SIZE_M)] + for d1 in range(TENSOR_SIZE_M): + row = [] + for d0 in range(TENSOR_SIZE_N): + input_matrix[d1][d0] = memory_flat[(READ_BASE_ADDR + d1 * TENSOR_SIZE_N + d0)] + + # Print input matrix + print("Input Matrix:") + for i, row in enumerate(input_matrix): + print(f"Row {i}: {[format(elem, 'X') for elem in row]}") + + if args.datamover_mode == 0: # Copy mode + output_matrix = input_matrix + elif args.datamover_mode == 1: # Transpose mode + output_matrix = transpose(input_matrix, TENSOR_SIZE_M, TENSOR_SIZE_N, TRANSP_MODE) + elif args.datamover_mode == 2: # CIM mode + if args.cim_mode == 0: # row-major -> CIM-layout + output_matrix = cim_layout(input_matrix, TENSOR_SIZE_M, TENSOR_SIZE_N, args.row_tile_size, args.num_elem_word) + elif args.cim_mode == 1: # CIM-layout -> row-major (use matrix dimenstions of original row-major layout) + output_matrix = cim_layout_reverse(input_matrix, TENSOR_SIZE_M, TENSOR_SIZE_N, args.row_tile_size, args.num_elem_word) + elif args.datamover_mode == 3: # CIM-layout transpose mode (INPUT SIZES EXPECTED IN ORIGINAL (ROW-MAJOR) LAYOUT FORM!) + input_matrix_chw = cim_layout_reverse(input_matrix, TENSOR_SIZE_M, TENSOR_SIZE_N, args.row_tile_size, args.num_elem_word) + # Reshape input_matrix_chw to TENSOR_SIZE_M x TENSOR_SIZE_N + input_matrix_chw_flat = [elem for row in input_matrix_chw for elem in row] + input_matrix_chw = [input_matrix_chw_flat[i * TENSOR_SIZE_N:(i + 1) * TENSOR_SIZE_N] for i in range(TENSOR_SIZE_M)] + print("\nInput Matrix reshaped to CHW layout:") + print(input_matrix_chw) + transposed_chw = transpose(input_matrix_chw, TENSOR_SIZE_M, TENSOR_SIZE_N, TRANSP_MODE) + print("\nTransposed CHW Matrix:") + print(transposed_chw) + output_matrix = cim_layout(transposed_chw, TENSOR_SIZE_N, TENSOR_SIZE_M, args.row_tile_size, args.num_elem_word) + else: + raise ValueError("[GM] datamover_mode must be 0 (copy), 1 (transpose), or 2 (CIM).") + + # Print output matrix + print("\nOutput Matrix:") + for i, row in enumerate(output_matrix): + print(f"Row {i}: {[format(elem, 'X') for elem in row]}") + print("\n") + + # Compare input and output matrix: equality check + if ([val for row in output_matrix for val in row] == [val for row in input_matrix for val in row]): + print("Output matrix equals input matrix in memory (flattened).") - # Step 6: Generate write addresses (Word-aligned) - write_transactions = read_transactions - write_addresses = generate_addresses( - args.write_base_addr, args.write_d0_stride, args.write_d0_length, - args.write_d1_stride, args.write_d1_length, write_transactions, bandwidth_N - ) + memory = write_matrix_to_memory(output_matrix, memory, ELEM_WIDTH, WORD_WIDTH, WRITE_BASE_ADDR) - # Step 7: Update memory with extracted data at write addresses - update_memory(memory, write_addresses, extracted_data) + # print("\nFinal Memory Content:") + # for i, word in enumerate(memory): + # print(f"Memory[{i}]: {word}") - # Step 8: Save updated memory - write_file(args.output_dir, "updated_memory.txt", memory) + write_file(OUTPUT_DIR, "updated_memory.txt", memory) - print(f"Files generated in '{args.output_dir}': initial_memory.txt, debug_values.txt, updated_memory.txt") + # Write data header file for C testing + config_params = { + # 'read_base_addr': args.read_base_addr, + # 'write_base_addr': args.write_base_addr, + 'bandwidth': args.bandwidth_bits, + 'word_width': WORD_WIDTH, + 'elem_width': args.elem_width, + 'memory_size': args.mem_size, + # 'misaligned_accesses': args.misaligned_accesses, + 'datamover_mode': args.datamover_mode, + 'transp_mode': args.transp_mode, + 'cim_mode': args.cim_mode, + 'row_tile_size': args.row_tile_size, + 'size_m': args.size_m, + 'num_channels': args.num_channels, + 'size_n': args.size_n + } + write_data_header_file(OUTPUT_DIR, input_matrix, output_matrix, config_params) if __name__ == "__main__": main() diff --git a/verif/python/validate_config.py b/verif/python/validate_config.py new file mode 100755 index 0000000..d32b5d4 --- /dev/null +++ b/verif/python/validate_config.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +""" +Configuration validation script for datamover HWPE +Validates that configuration parameters are compatible and reasonable +""" + +import sys +import argparse + +def validate_config(bandwidth, word_width, elem_width, memory_size, num_channels, + datamover_mode, transp_mode, cim_mode, row_tile_size, + size_m, size_n): + """Validate configuration parameters""" + errors = [] + warnings = [] + + bandwidth_aligned = bandwidth + + # Computed values + bandwidth_elems = bandwidth_aligned // elem_width + num_elem_word = word_width // elem_width + + # Basic parameter validation + if bandwidth_aligned % word_width != 0: + errors.append(f"BANDWIDTH_ALIGNED ({bandwidth_aligned}) must be divisible by WORD_WIDTH ({word_width})") + + if word_width % elem_width != 0: + errors.append(f"WORD_WIDTH ({word_width}) must be divisible by ELEM_WIDTH ({elem_width})") + + if memory_size < (num_channels * size_n * size_m * elem_width // word_width) * 2: + errors.append(f"MEMORY_SIZE ({memory_size}) is too small for the given matrix size " + f"({num_channels}x{size_m}x{size_n}) and element width ({elem_width})") + + # Mode validation (based on config.mk) + if datamover_mode not in [0, 1, 2, 3, 4, 5]: + errors.append(f"DATAMOVER_MODE ({datamover_mode}) must be 0 (copy), 1 (transpose), 2 (CIM data layout conversion), 3 (CIM layout transpose), 4 (unfold), or 5 (fold)") + + # if transp_mode not in [0, 1, 2, 4]: + # errors.append(f"TRANSP_MODE ({transp_mode}) must be 0, 1, 2, or 4") + + # # Mode consistency validation + # if datamover_mode == 0 and transp_mode != 0: + # warnings.append(f"Copy mode (DATAMOVER_MODE=0) typically uses TRANSP_MODE=0, but got {transp_mode}") + + if datamover_mode == 1 and transp_mode not in [1, 2, 4]: + errors.append(f"Transpose mode (DATAMOVER_MODE=1) requires TRANSP_MODE = [1,2,4] but got {transp_mode}") + + # CIM-specific validation + if datamover_mode in [2, 3]: + if cim_mode not in [0, 1]: + errors.append(f"CIM_MODE ({cim_mode}) must be 0 (row-major -> CIM-layout) or 1 (CIM-layout -> row-major) for CIM modes") + if row_tile_size % bandwidth_elems != 0: + errors.append(f"ROW_TILE_SIZE ({row_tile_size}) must be a multiple of bandwidth ({bandwidth_elems})") + # if row_tile_size > size_n: + # errors.append(f"ROW_TILE_SIZE ({row_tile_size}) cannot be greater than matrix width ({size_n})") + + # Memory requirements + matrix_elements = num_channels * size_m * size_n + matrix_words = (matrix_elements * elem_width + word_width - 1) // word_width + total_memory_needed = matrix_words * 2 # Input + output matrices + + if total_memory_needed > memory_size: + errors.append(f"Memory size ({memory_size} words) insufficient for matrices " + f"({total_memory_needed} words needed for {num_channels}x{size_m}x{size_n} input+output)") + + # Matrix dimension alignment errors + # if size_n % bandwidth_elems != 0: + # errors.append(f"Matrix width ({size_n}) not aligned to bandwidth " + # f"({bandwidth_elems} elements)") + + # if size_m % bandwidth_elems != 0: + # errors.append(f"Matrix height ({size_m}) not aligned to bandwidth " + # f"({bandwidth_elems} elements)") + + # Transpose-specific validation + if datamover_mode == 1 and transp_mode > 0: + if bandwidth_elems % transp_mode != 0: + errors.append(f"Bandwidth elements ({bandwidth_elems}) must be divisible " + f"by TRANSP_MODE ({transp_mode})") + + return errors, warnings + +def main(): + parser = argparse.ArgumentParser(description="Validate datamover configuration") + parser.add_argument("--bandwidth", type=int, required=True) + parser.add_argument("--word_width", type=int, required=True) + parser.add_argument("--elem_width", type=int, required=True) + parser.add_argument("--memory_size", type=int, required=True) + parser.add_argument("--num_channels", type=int, required=True) + parser.add_argument("--datamover_mode", type=int, required=True) + parser.add_argument("--transp_mode", type=int, required=True) + parser.add_argument("--cim_mode", type=int, required=True) + parser.add_argument("--row_tile_size", type=int, required=True) + parser.add_argument("--size_m", type=int, required=True) + parser.add_argument("--size_n", type=int, required=True) + + args = parser.parse_args() + + errors, warnings = validate_config( + args.bandwidth, args.word_width, args.elem_width, args.memory_size, + args.num_channels, args.datamover_mode, args.transp_mode, args.cim_mode, + args.row_tile_size, + args.size_m, args.size_n + ) + + # Print results + if warnings: + print("WARNINGS:") + for warning in warnings: + print(f" - {warning}") + print() + + if errors: + print("ERRORS:") + for error in errors: + print(f" - {error}") + print() + print("Configuration validation FAILED!") + return 1 + else: + print("Configuration validation PASSED!") + + # Print mode information + mode_names = {0: "Copy", 1: "Transpose", 2: "CIM Data Layout Conversion", 3: "CIM Layout Transpose", 4: "Unfold", 5: "Fold"} + cim_mode_names = {0: "row-major -> CIM-layout", 1: "CIM-layout -> row-major"} + + print(f"\nMode Configuration:") + print(f" DATAMOVER_MODE: {args.datamover_mode} ({mode_names.get(args.datamover_mode, 'Unknown')})") + print(f" TRANSP_MODE: {args.transp_mode}") + print(f" CIM_MODE: {args.cim_mode} ({cim_mode_names.get(args.cim_mode, 'Unknown')})") + if args.datamover_mode == 2: # CIM mode + print(f" ROW_TILE_SIZE: {args.row_tile_size}") + + # Print computed values + bandwidth_elems = args.bandwidth // args.elem_width + num_elem_word = args.word_width // args.elem_width + matrix_words = (args.num_channels * args.size_m * args.size_n) // num_elem_word + + print(f"\nComputed values:") + print(f" Channels: {args.num_channels}") + print(f" Elements per bandwidth: {bandwidth_elems}") + print(f" Elements per word: {num_elem_word}") + print(f" Matrix memory usage: {matrix_words} words ({matrix_words * 2} total)") + print(f" Memory utilization: {(matrix_words * 2 * 100) // args.memory_size}% ({matrix_words*2} / {args.memory_size})") + + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/verif/tb/tb_datamover_top_wrap.sv b/verif/tb/tb_datamover_top_wrap.sv index 5f102a2..b1753bd 100644 --- a/verif/tb/tb_datamover_top_wrap.sv +++ b/verif/tb/tb_datamover_top_wrap.sv @@ -1,5 +1,5 @@ /* - * Copyright (C) 2025 ETH Zurich and University of Bologna + * Copyright (C) 2025-2026 ETH Zurich and University of Bologna * * Copyright and related rights are licensed under the Solderpad Hardware * License, Version 0.51 (the "License"); you may not use this file except in @@ -13,9 +13,12 @@ /* * Authors: Sergio Mazzola - * Arpan Suravi Prasad s + * Arpan Suravi Prasad + * Cyrill Durrer */ +// OUTDATED! +// ToDo: Implement SW-based testing with a CPU core module tb_datamover_top_wrap; import datamover_package::*; @@ -29,6 +32,7 @@ import tb_package::*; logic clear_i = 1'b0; logic randomize_mem = 1'b0; + logic enable_mem = 1'b1; logic stallable_mem = 1'b1; hwpe_stream_intf_tcdm #( @@ -60,7 +64,14 @@ import tb_package::*; logic periph_r_valid; logic [PERIPH_ID-1:0] periph_r_id; - + logic [2:0] transp_mode; + // logic [15:0] transp_len; + logic [11:0] tensor_size_m; + logic [11:0] tensor_size_n; + logic [3:0] read_dim_enable; + logic [3:0] write_dim_enable; + logic [10:0] num_channels; + logic [20:0] total_elements; // Performs one entire clock cycle. task cycle; @@ -101,19 +112,31 @@ import tb_package::*; typedef struct { logic [31:0] base_addr; - logic [31:0] d0_stride; - logic [31:0] d1_stride; - logic [11:0] d0_length; - logic [11:0] d1_length; - logic [11:0] tot_length; + logic [15:0] d0_stride; + logic [15:0] d1_stride; + logic [15:0] d2_stride; + logic [15:0] d3_stride; + logic [15:0] d4_stride; + logic [15:0] d0_length; + logic [15:0] d1_length; + logic [15:0] d2_length; + logic [15:0] d3_length; + logic [31:0] tot_length; } addressgen_t; addressgen_t read_addr, write_addr; - assign read_addr = '{`STIM_READ_BASE_ADDR, `STIM_READ_D0_STRIDE, `STIM_READ_D1_STRIDE, `STIM_READ_D0_LENGTH, `STIM_READ_D1_LENGTH, `STIM_READ_TOT_LENGTH}; - assign write_addr = '{`STIM_WRITE_BASE_ADDR, `STIM_WRITE_D0_STRIDE, `STIM_WRITE_D1_STRIDE, `STIM_WRITE_D0_LENGTH, `STIM_WRITE_D1_LENGTH, `STIM_WRITE_TOT_LENGTH}; - // assign read_addr = '{`STIM_READ_BASE_ADDR, 32'h4, 32'h10, 32'h4, 32'h4, 32'h10}; - // assign write_addr = '{32'h40, 32'h4, 32'h10, 32'h4, 32'h4, 32'h10}; + assign read_addr = '{`STIM_READ_BASE_ADDR, `STIM_READ_D0_STRIDE, `STIM_READ_D1_STRIDE, `STIM_READ_D2_STRIDE, `STIM_READ_D3_STRIDE, `STIM_READ_D4_STRIDE, `STIM_READ_D0_LENGTH, `STIM_READ_D1_LENGTH, `STIM_READ_D2_LENGTH, `STIM_READ_D3_LENGTH, `STIM_READ_TOT_LENGTH}; + assign write_addr = '{`STIM_WRITE_BASE_ADDR, `STIM_WRITE_D0_STRIDE, `STIM_WRITE_D1_STRIDE, `STIM_WRITE_D2_STRIDE, `STIM_WRITE_D3_STRIDE, `STIM_WRITE_D4_STRIDE, `STIM_WRITE_D0_LENGTH, `STIM_WRITE_D1_LENGTH, `STIM_WRITE_D2_LENGTH, `STIM_WRITE_D3_LENGTH, `STIM_WRITE_TOT_LENGTH}; + + assign transp_mode = `STIM_TRANSP_MODE; + // assign transp_len = `STIM_TRANSP_LEN; + assign tensor_size_m = `STIM_TENSOR_SIZE_M; + assign tensor_size_n = `STIM_TENSOR_SIZE_N; + assign read_dim_enable = `STIM_READ_DIM_ENABLE; + assign write_dim_enable = `STIM_WRITE_DIM_ENABLE; + assign num_channels = `STIM_NUM_CHANNELS; + assign total_elements = `STIM_TOTAL_ELEMENTS; datamover_top_wrap #( @@ -217,8 +240,9 @@ import tb_package::*; int error_status; initial begin : main_execution - logic [31:0] len0_reg; - logic [31:0] len1_reg; + logic [31:0] ctrl_engine_reg; + logic [31:0] tensor_dim_reg; + logic [31:0] channels_reg; $info("Start execution...\n"); @@ -233,61 +257,74 @@ import tb_package::*; $readmemh(STIMULI_PATH, tb_datamover_top_wrap.i_testbench_memory.memory); // soft clear - periph_write(datamover_package::DATAMOVER_SOFT_CLEAR, datamover_package::HWPE_REGISTER_OFFS, 32'habcdefab, clk_i, periph_bus); + periph_write(datamover_package::DATAMOVER_SOFT_CLEAR, datamover_package::HWPE_REGISTER_OFFS, 32'habcdefab, clk_i, periph_bus); #(100*TCP); + $info("[%0t] Acquiring job...\n", $time); // acquire job $info("Acquiring job...\n"); - while(status !== 32'h00) - periph_read(datamover_package::DATAMOVER_ACQUIRE, datamover_package::HWPE_REGISTER_OFFS, status, clk_i, periph_bus); + while(status != 32'h00) + periph_read(datamover_package::DATAMOVER_ACQUIRE, datamover_package::HWPE_REGISTER_OFFS, status, clk_i, periph_bus); $info("Job acquired, configuring datamover...\n"); - + periph_write(datamover_package::DATAMOVER_REG_IN_PTR, datamover_package::DATAMOVER_REGISTER_OFFS, read_addr.base_addr, clk_i, periph_bus); periph_write(datamover_package::DATAMOVER_REG_OUT_PTR, datamover_package::DATAMOVER_REGISTER_OFFS, write_addr.base_addr, clk_i, periph_bus); - + // Configure packed length registers (see datamover_package.sv) - len0_reg = {read_addr.d1_length[7:0], read_addr.d0_length[11:0], read_addr.tot_length[11:0]}; - len1_reg = {4'b0, read_addr.d1_length[11:8], write_addr.d1_length[11:0], write_addr.d0_length[11:0]}; + ctrl_engine_reg = {16'b0, write_dim_enable[3:0], read_dim_enable[3:0], 5'b0, transp_mode[2:0]}; + tensor_dim_reg = {tensor_size_n[15:0], tensor_size_m[15:0]}; + channels_reg = {total_elements[20:0], num_channels[10:0]}; + // Make sure tot_length is the same for read and write assert (read_addr.tot_length == write_addr.tot_length) else $fatal("Read and write total lengths do not match!"); - periph_write(datamover_package::DATAMOVER_REG_LEN0, datamover_package::DATAMOVER_REGISTER_OFFS, len0_reg, clk_i, periph_bus); - periph_write(datamover_package::DATAMOVER_REG_LEN1, datamover_package::DATAMOVER_REGISTER_OFFS, len1_reg, clk_i, periph_bus); - - periph_write(datamover_package::DATAMOVER_REG_IN_D0_STRIDE, datamover_package::DATAMOVER_REGISTER_OFFS, read_addr.d0_stride, clk_i, periph_bus); - periph_write(datamover_package::DATAMOVER_REG_IN_D1_STRIDE, datamover_package::DATAMOVER_REGISTER_OFFS, read_addr.d1_stride, clk_i, periph_bus); - periph_write(datamover_package::DATAMOVER_REG_IN_D2_STRIDE, datamover_package::DATAMOVER_REGISTER_OFFS, 32'h0, clk_i, periph_bus); - - periph_write(datamover_package::DATAMOVER_REG_OUT_D0_STRIDE, datamover_package::DATAMOVER_REGISTER_OFFS, write_addr.d0_stride, clk_i, periph_bus); - periph_write(datamover_package::DATAMOVER_REG_OUT_D1_STRIDE, datamover_package::DATAMOVER_REGISTER_OFFS, write_addr.d1_stride, clk_i, periph_bus); - periph_write(datamover_package::DATAMOVER_REG_OUT_D2_STRIDE, datamover_package::DATAMOVER_REGISTER_OFFS, 32'h4, clk_i, periph_bus); - - // Transposition mode (LSB: 000=none, 001=1 elem, 010=2 elem, 100=4 elem) - periph_write(datamover_package::DATAMOVER_REG_TRANSP_MODE, datamover_package::DATAMOVER_REGISTER_OFFS, {29'b0, TRANSP_MODE}, clk_i, periph_bus); + periph_write(datamover_package::DATAMOVER_REG_TOT_LEN, datamover_package::DATAMOVER_REGISTER_OFFS, read_addr.tot_length, clk_i, periph_bus); + periph_write(datamover_package::DATAMOVER_REG_IN_D0, datamover_package::DATAMOVER_REGISTER_OFFS, {read_addr.d0_stride, read_addr.d0_length}, clk_i, periph_bus); + periph_write(datamover_package::DATAMOVER_REG_IN_D1, datamover_package::DATAMOVER_REGISTER_OFFS, {read_addr.d1_stride, read_addr.d1_length}, clk_i, periph_bus); + periph_write(datamover_package::DATAMOVER_REG_IN_D2, datamover_package::DATAMOVER_REGISTER_OFFS, {read_addr.d2_stride, read_addr.d2_length}, clk_i, periph_bus); + periph_write(datamover_package::DATAMOVER_REG_IN_D3, datamover_package::DATAMOVER_REGISTER_OFFS, {read_addr.d3_stride, read_addr.d3_length}, clk_i, periph_bus); + periph_write(datamover_package::DATAMOVER_REG_OUT_D0, datamover_package::DATAMOVER_REGISTER_OFFS, {write_addr.d0_stride, write_addr.d0_length}, clk_i, periph_bus); + periph_write(datamover_package::DATAMOVER_REG_OUT_D1, datamover_package::DATAMOVER_REGISTER_OFFS, {write_addr.d1_stride, write_addr.d1_length}, clk_i, periph_bus); + periph_write(datamover_package::DATAMOVER_REG_OUT_D2, datamover_package::DATAMOVER_REGISTER_OFFS, {write_addr.d2_stride, write_addr.d2_length}, clk_i, periph_bus); + periph_write(datamover_package::DATAMOVER_REG_OUT_D3, datamover_package::DATAMOVER_REGISTER_OFFS, {write_addr.d3_stride, write_addr.d3_length}, clk_i, periph_bus); + periph_write(datamover_package::DATAMOVER_REG_IN_OUT_D4_STRIDE, datamover_package::DATAMOVER_REGISTER_OFFS, {write_addr.d4_stride, read_addr.d4_stride}, clk_i, periph_bus); + periph_write(datamover_package::DATAMOVER_REG_MATRIX_DIM, datamover_package::DATAMOVER_REGISTER_OFFS, tensor_dim_reg, clk_i, periph_bus); + periph_write(datamover_package::DATAMOVER_REG_CHANNELS, datamover_package::DATAMOVER_REGISTER_OFFS, channels_reg, clk_i, periph_bus); + periph_write(datamover_package::DATAMOVER_REG_CTRL_ENGINE, datamover_package::DATAMOVER_REGISTER_OFFS, ctrl_engine_reg, clk_i, periph_bus); periph_write(datamover_package::DATAMOVER_COMMIT_AND_TRIGGER, datamover_package::HWPE_REGISTER_OFFS, 32'h0, clk_i, periph_bus); - while(status === 32'h00) - periph_read(datamover_package::DATAMOVER_STATUS, datamover_package::HWPE_REGISTER_OFFS, status, clk_i, periph_bus); + while(status == 32'h00) + periph_read(datamover_package::DATAMOVER_STATUS, datamover_package::HWPE_REGISTER_OFFS, status, clk_i, periph_bus); // ToDo(cdurrer): Why STATUS and not FINISHED register? $info("Datamover working...\n"); - - while(status !== 32'h00) + + while(status != 32'h00) periph_read(datamover_package::DATAMOVER_STATUS, datamover_package::HWPE_REGISTER_OFFS, status, clk_i, periph_bus); $info("Datamover finished transfer. Checking output...\n"); check_output( GOLDEN_PATH, // File containing golden reference data - 32'h0, // Start address in memory + 32'h0, // Start address in memory MEMORY_SIZE, // Number of entries to check tb_datamover_top_wrap.i_testbench_memory.memory, // Reference to memory array error_status ); + // Check if there were any errors and fail the simulation if so + if (error_status != 0) begin + $error("Test FAILED: Output mismatch detected (error_status = %0d)", error_status); + $display("DATAMOVER_TEST_FAILED"); + $stop(1); + end else begin + $info("Test PASSED: All output verification checks successful"); + $display("DATAMOVER_TEST_PASSED"); + end + $finish; - + end : main_execution -endmodule // tb_datamover_top_wrap \ No newline at end of file +endmodule // tb_datamover_top_wrap diff --git a/verif/tb/tb_package.sv b/verif/tb/tb_package.sv index 391549f..49a8c4b 100644 --- a/verif/tb/tb_package.sv +++ b/verif/tb/tb_package.sv @@ -1,8 +1,8 @@ -/* +/* * tb_package.sv * Arpan Suravi Prasad * - * Copyright (C) 2018-2023 ETH Zurich, University of Bologna + * Copyright (C) 2018-2026 ETH Zurich, University of Bologna * Copyright and related rights are licensed under the Solderpad Hardware * License, Version 0.51 (the "License"); you may not use this file except in * compliance with the License. You may obtain a copy of the License at @@ -13,8 +13,14 @@ * specific language governing permissions and limitations under the License. */ package tb_package; - /* Configuration */ + // ATI timing parameters + timeunit 1ps; + timeprecision 1ps; + localparam TCP = 1.0ns; // clock period, 1 GHz clock + localparam TA = 0.2ns; // application time + localparam TT = 0.8ns; // test time + /* Configuration */ localparam int ADDR_WIDTH = 32; localparam int PERIPH_ID = 10; localparam int MEMORY_SIZE= `STIM_MEM_SIZE; @@ -35,13 +41,6 @@ package tb_package; localparam int unsigned WORD_WIDTH = NUM_ELEM_WORD * ELEM_WIDTH; // should correspond to bank width localparam int unsigned BANDWIDTH_WORDS = BANDWIDTH / WORD_WIDTH; - // // ATI timing parameters. - timeunit 1ps; - timeprecision 1ps; - localparam TCP = 1.0ns; // clock period, 1 GHz clock - localparam TA = 0.2ns; // application time - localparam TT = 0.8ns; // test time - typedef struct{ logic req; logic gnt; @@ -161,7 +160,7 @@ package tb_package; status = 0; // Assume pass initially - + // Open the golden reference file for reading file = $fopen(golden_fname, "r"); if (file == 0) begin @@ -189,8 +188,8 @@ package tb_package; if (read_data !== golden_data) begin status = 1; $display("MISMATCH at address %0d: Expected %h, Actual %h", start_addr + i, golden_data, read_data); - end else begin - $display("MATCH at address %0d: %h", start_addr + i, read_data); + // end else begin + // $display("MATCH at address %0d: %h", start_addr + i, read_data); end end @@ -204,6 +203,6 @@ package tb_package; end endtask - -endpackage \ No newline at end of file + +endpackage