redmule verilator simulation

.gitignore

@@ -0,0 +1,17 @@
+.bender/
+bin/
+install/
+manifest.flist
+install
+sw/build/
+golden-model/FP16/net_parameters.h
+golden-model/FP16/scripts/__pycache__/
+golden-model/gemm/
+golden-model/venv/
+sw/inc/
+verilator_tb.vcd
+vsim/
+log/
+*.tar.gz
+verilator/
+simulation_results/

Makefile

@@ -7,29 +7,30 @@
 # Top-level Makefile
 
 # Paths to folders
-mkfile_path    := $(dir $(abspath $(firstword $(MAKEFILE_LIST))))
-SW             ?= $(mkfile_path)sw
+mkfile_path    := $(shell git rev-parse --show-toplevel)
+SW             ?= $(mkfile_path)/sw
 BUILD_DIR      ?= $(SW)/build
-VSIM_DIR       ?= $(mkfile_path)vsim
+VSIM_DIR       ?= $(mkfile_path)/vsim
 QUESTA         ?= questa-2023.4
 BENDER_DIR     ?= .
 BENDER         ?= bender
 ISA            ?= riscv
 ARCH           ?= rv
 XLEN           ?= 32
-XTEN           ?= imc
+XTEN           ?= imc_zicsr
+
 
 ifeq ($(REDMULE_COMPLEX),1)
 	TEST_SRCS := $(SW)/redmule_complex.c
 else
 	TEST_SRCS := $(SW)/redmule.c
 endif
 
-compile_script ?= $(mkfile_path)scripts/compile.tcl
-compile_script_synth ?= $(mkfile_path)scripts/synth_compile.tcl
+compile_script ?= $(mkfile_path)/scripts/compile.tcl
+compile_script_synth ?= $(mkfile_path)/scripts/synth_compile.tcl
 compile_flag   ?= +acc -permissive -suppress 2583 -suppress 13314
 
-INI_PATH  = $(mkfile_path)modelsim.ini
+INI_PATH  = $(mkfile_path)/modelsim.ini
 WORK_PATH = $(VSIM_DIR)/work
 
 # Useful Parameters
@@ -53,11 +54,13 @@ INC += -I$(SW)/utils
 BOOTSCRIPT := $(SW)/kernel/crt0.S
 LINKSCRIPT := $(SW)/kernel/link.ld
 
-CC=$(ISA)$(XLEN)-unknown-elf-gcc
-LD=$(CC)
-OBJDUMP=$(ISA)$(XLEN)-unknown-elf-objdump
-CC_OPTS=-march=$(ARCH)$(XLEN)$(XTEN) -mabi=ilp32 -D__$(ISA)__ -O2 -g -Wextra -Wall -Wno-unused-parameter -Wno-unused-variable -Wno-unused-function -Wundef -fdata-sections -ffunction-sections -MMD -MP
-LD_OPTS=-march=$(ARCH)$(XLEN)$(XTEN) -mabi=ilp32 -D__$(ISA)__ -MMD -MP -nostartfiles -nostdlib -Wl,--gc-sections
+# Setup toolchain (from SDK) and options
+RV_CC=$(ISA)$(XLEN)-unknown-elf-gcc
+RV_LD=$(ISA)$(XLEN)-unknown-elf-gcc
+RV_OBJDUMP=$(ISA)$(XLEN)-unknown-elf-objdump
+RV_CC_OPTS=-march=$(ARCH)$(XLEN)$(XTEN) -mabi=ilp32 -D__$(ISA)__ -O2 -g -Wextra -Wall -Wno-unused-parameter -Wno-unused-variable -Wno-unused-function -Wundef -fdata-sections -ffunction-sections -MMD -MP
+RV_LD_OPTS=-march=$(ARCH)$(XLEN)$(XTEN) -mabi=ilp32 -D__$(ISA)__ -MMD -MP -nostartfiles -nostdlib -Wl,--gc-sections
+
 
 # Setup build object dirs
 CRT=$(BUILD_DIR)/crt0.o
@@ -70,25 +73,25 @@ STIM_DATA=$(VSIM_DIR)/stim_data.txt
 # Build implicit rules
 $(STIM_INSTR) $(STIM_DATA): $(BIN)
 	objcopy --srec-len 1 --output-target=srec $(BIN) $(BIN).s19
-	scripts/parse_s19.pl $(BIN).s19 > $(BIN).txt
-	python scripts/s19tomem.py $(BIN).txt $(STIM_INSTR) $(STIM_DATA)
+	scripts/parse_s19.pl $(BIN).s19 > $(BIN).txt 2>$(BUILD_DIR)/parse_s19.pl.log
+	python scripts/s19tomem.py $(BIN).txt $(STIM_INSTR) $(STIM_DATA) 
 
 $(BIN): $(CRT) $(OBJ)
-	$(LD) $(LD_OPTS) -o $(BIN) $(CRT) $(OBJ) -T$(LINKSCRIPT)
+	$(RV_LD) $(RV_LD_OPTS) -o $(BIN) $(CRT) $(OBJ) -T$(LINKSCRIPT)
 
 $(CRT): $(BUILD_DIR)
-	$(CC) $(CC_OPTS) -c $(BOOTSCRIPT) -o $(CRT)
+	$(RV_CC) $(CC_OPTS) -c $(BOOTSCRIPT) -o $(CRT)
 
 $(OBJ): $(TEST_SRCS)
-	$(CC) $(CC_OPTS) -c $(TEST_SRCS) $(FLAGS) $(INC) -o $(OBJ)
+	$(RV_CC) $(CC_OPTS) -c $(TEST_SRCS) $(FLAGS) $(INC) -o $(OBJ)
 
 $(BUILD_DIR):
 	mkdir -p $(BUILD_DIR)
 
 SHELL := /bin/bash
 
 # Generate instructions and data stimuli
-sw-build: $(STIM_INSTR) $(STIM_DATA) dis
+sw-build: $(VSIM_DIR) $(STIM_INSTR) $(STIM_DATA) dis
 
 # Run the simulation
 run: $(CRT)
@@ -118,8 +121,9 @@ bender:
 include bender_common.mk
 include bender_sim.mk
 include bender_synth.mk
+include Makefile.verilator
 
-WAVES := $(mkfile_path)scripts/wave.tcl
+WAVES := $(mkfile_path)/scripts/wave.tcl
 
 ifeq ($(REDMULE_COMPLEX),1)
 	tb := redmule_complex_tb
@@ -149,12 +153,14 @@ synth-ips:
 
 sw-clean:
 	rm -rf $(BUILD_DIR)
+	@rm -vf $(STIM_INSTR)
+	@rm -vf $(STIM_DATA)
 
 hw-clean:
 	rm -rf $(VSIM_DIR)
 
 dis:
-	$(OBJDUMP) -d $(BIN) > $(DUMP)
+	$(RV_OBJDUMP) -d $(BIN) > $(DUMP)
 
 OP     ?= gemm
 fp_fmt ?= FP16
@@ -169,7 +175,8 @@ golden-clean:
 	$(MAKE) -C golden-model golden-clean
 
 clean-all: hw-clean sw-clean
-	rm -rf $(mkfile_path).bender
+	rm -rf $(mkfile_path)/.bender
+	rm -rf $(mkfile_path)/Bender.lock
 	rm -rf $(compile_script)
 
 hw-build: $(VSIM_DIR)
@@ -179,3 +186,4 @@ hw-build: $(VSIM_DIR)
 sw-all: sw-clean sw-build
 
 hw-all: hw-clean hw-build
+

Makefile.tools

@@ -0,0 +1,77 @@
+
+prj_path := $(shell git rev-parse --show-toplevel)
+num_cores := $(shell nproc)
+num_cores_half := $(shell echo "$$(($(num_cores) / 2))")
+CXX := g++-10
+
+
+INSTALL_PREFIX          ?= install
+INSTALL_DIR             ?= $(prj_path)/${INSTALL_PREFIX}
+GCC_INSTALL_DIR         ?= ${INSTALL_DIR}/riscv-gcc
+LLVM_INSTALL_DIR        ?= ${INSTALL_DIR}/riscv-llvm
+ISA_SIM_INSTALL_DIR     ?= ${INSTALL_DIR}/riscv-isa-sim
+ISA_SIM_MOD_INSTALL_DIR ?= ${INSTALL_DIR}/riscv-isa-sim-mod
+VERIL_SRC_ROOT           ?= $(prj_path)/vendor
+VERIL_INSTALL_DIR       ?= ${INSTALL_DIR}/verilator
+VERIL_VERSION           ?= v5.028
+
+all: verilator riscv32-gcc 
+# Verilator
+verilator: ${VERIL_INSTALL_DIR}/bin/verilator
+
+${VERIL_INSTALL_DIR}/bin/verilator:
+	rm -fr $(VERIL_SRC_ROOT)/verilator
+	cd $(VERIL_SRC_ROOT) && git clone https://github.com/verilator/verilator.git
+	# Checkout the right version
+	cd $(VERIL_SRC_ROOT)/verilator  && git reset --hard && git fetch && git checkout ${VERIL_VERSION}
+	# Compile verilator
+	cd $(VERIL_SRC_ROOT)/verilator  && git clean -xfdf && autoconf && \
+	./configure --prefix=$(VERIL_INSTALL_DIR) CXX=g++-10  && make -j$(num_cores_half)  && make install
+	touch ${VERIL_INSTALL_DIR}/bin/verilator
+
+
+
+.PHONY: cores
+
+cores:
+	@num_cores=$$(nproc); \
+	num_cores=$$((num_cores / 2)); \
+	echo "Number of cores available on this machine (divided by 2): $$num_cores"
+
+riscv32-gcc: $(GCC_INSTALL_DIR)
+
+$(GCC_INSTALL_DIR): vendor/riscv32-elf-gcc.url
+	rm -fr $(GCC_INSTALL_DIR)
+	mkdir -p $(INSTALL_DIR)
+	cd vendor && \
+	wget  `cat $(CURDIR)/$<` -O riscv.tar.gz && \
+	tar -xzvf riscv.tar.gz -C  $(INSTALL_DIR)/ riscv	
+	mv $(INSTALL_DIR)/riscv $(GCC_INSTALL_DIR)
+	touch  $(GCC_INSTALL_DIR)
+
+
+riscv32-llvm: $(LLVM_INSTALL_DIR)
+
+vendor/riscv32-elf-llvm.tar.gz: vendor/riscv32-elf-llvm.url
+	cd vendor && \
+	wget `cat $(CURDIR)/$<` -O riscv32-elf-llvm.tar.gz && \
+	touch riscv32-elf-llvm.tar.gz
+
+$(LLVM_INSTALL_DIR): vendor/riscv32-elf-llvm.tar.gz
+	rm -fr $(LLVM_INSTALL_DIR)
+	mkdir -p $(INSTALL_DIR)
+	cd vendor && \
+	tar -xzvf riscv32-elf-llvm.tar.gz -C  $(INSTALL_DIR)/ riscv	
+	mv $(INSTALL_DIR)/riscv $(LLVM_INSTALL_DIR)
+	touch  $(LLVM_INSTALL_DIR)
+
+riscv32-llvm-patch:
+	@cd $(prj_path)/util/isolde &&   \
+	tar -xzvf tools.tar.gz     &&   \
+	mv $(prj_path)/util/isolde/riscv32-unknown-elf-objcopy $(LLVM_INSTALL_DIR)/bin  && \
+	mv $(prj_path)/util/isolde/riscv32-unknown-elf-objdump  $(LLVM_INSTALL_DIR)/bin  && \
+	echo "REPLACED riscv32-unknown-elf- objcopy/objdump"
+
+dev-dep:
+	sudo apt-get install libelf-dev
+	sudo apt-get install srecord

Makefile.verilator

@@ -0,0 +1,86 @@
+#############
+# Verilator #
+#############
+
+VLT_TOP_MODULE ?= redmule_tb
+#VLT_TOP_MODULE = tb_top_verilator
+
+VERI_LOG_DIR      ?= $(mkfile_path)/log/$(VLT_TOP_MODULE)
+SIM_TEST_INPUTS   ?= $(mkfile_path)/vsim
+TEST              ?= redmule
+BIN_DIR           = $(mkfile_path)/bin/$(VLT_TOP_MODULE)
+VERI_FLAGS        +=
+
+
+
+.PHONY: veri-clean 
+
+# Clean all build directories and temporary files for Questasim simulation
+veri-clean: 
+	rm -f manifest.flist
+	make -C sim/core -f Makefile.verilator CV_CORE_MANIFEST=${CURDIR}/manifest.flist  SIM_RESULTS=$(BIN_DIR) $@
+
+verilate: $(BIN_DIR)/verilator_executable
+
+manifest.flist: Bender.yml
+	$(BENDER) script verilator $(common_targs) $(VLT_BENDER)  >$@
+	touch $@
+
+$(BIN_DIR)/verilator_executable: manifest.flist
+	mkdir -p $(dir $@)
+	make -C sim/core -f Makefile.verilator CV_CORE_MANIFEST=${CURDIR}/manifest.flist  SIM_RESULTS=$(BIN_DIR) VLT_TOP_MODULE=$(VLT_TOP_MODULE) verilate
+
+sanity-veri-run: VLT_TOP_MODULE := tb_top_verilator
+sanity-veri-run:
+	mkdir -p $(VERI_LOG_DIR)
+	rm -f $(VERI_LOG_DIR)/verilator_tb.vcd
+	make -C sim/core -f Makefile.verilator CV_CORE_MANIFEST=${CURDIR}/manifest.flist  SIM_RESULTS=$(BIN_DIR) VLT_TOP_MODULE=$(VLT_TOP_MODULE) TEST=hello-world run-test
+	mv sim/core/verilator_tb.vcd $(VERI_LOG_DIR)/
+
+.PHONY: run-test
+run-test: $(BIN_DIR)/verilator_executable 
+	@echo "$(BANNER)"
+	@echo "* Running with Verilator: "
+	@echo "*                         logfile in $(VERI_LOG_DIR)/$(TEST).log"
+	@echo "*                         *.vcd   in $(VERI_LOG_DIR)"
+	@echo "$(BANNER)"
+	mkdir -p $(VERI_LOG_DIR)
+	rm -f $(VERI_LOG_DIR)/verilator_tb.vcd
+	$(BIN_DIR)/verilator_executable  \
+		$(VERI_FLAGS) \
+		"+firmware=$(SIM_TEST_INPUTS)/stim_instr.txt" \
+		"+simdata=$(SIM_TEST_INPUTS)/stim_data.txt" \
+		| tee $(VERI_LOG_DIR)/$(TEST).log
+	mv verilator_tb.vcd $(VERI_LOG_DIR)/
+
+
+.PHONY: veri-test
+veri-test: $(BIN_DIR)/verilator_executable 
+	@echo "$(BANNER)"
+	@echo "* Running with Verilator: "
+	@echo "*                         logfile in $(VERI_LOG_DIR)/$(TEST).log"
+	@echo "*                         *.vcd   in $(VERI_LOG_DIR)"
+	@echo "$(BANNER)"
+	mkdir -p $(VERI_LOG_DIR)
+	rm -f $(VERI_LOG_DIR)/verilator_tb.vcd
+	$(BIN_DIR)/verilator_executable  \
+		$(VERI_FLAGS) \
+		"+firmware=$(SIM_TEST_INPUTS)/stim_instr.txt" \
+		"+simdata=$(SIM_TEST_INPUTS)/stim_data.txt" \
+		| tee $(VERI_LOG_DIR)/$(TEST).log
+	mv verilator_tb.vcd $(VERI_LOG_DIR)/	
+
+
+.PHONY: help
+
+help:
+	@echo "verilator related available targets:"
+	@echo verilate                                 -- builds verilator simulation, available here: $(BIN_DIR)/verilator_executable
+	@echo run-test                                 -- runs the test
+	@echo veri-clean                               -- gets a clean slate for simulation
+	@echo verilate VLT_TOP_MODULE=tb_top_verilator
+	@echo sanity-veri-run                          -- smoke test for $(BIN_DIR)/verilator_executable, only with  VLT_TOP_MODULE=tb_top_verilator
+
+
+
+

README.md

@@ -1,3 +1,57 @@
+# ISOLDE
+First time, install toolchain
+
+```sh
+make -f Makefile.tools
+```
+otherwise:  
+```sh
+. ./eth.sh 
+```
+
+## build simulation
+top module can be configured in cmd line,VLT_TOP_MODULE=<top module name>, it defaults to **redmule_tb**, see [Makefile.verilator](Makefile.verilator)
+```sh
+make verilate
+```
+### get a clean slate
+```sh
+make veri-clean
+```
+## test simulation
+only works if VLT_TOP_MODULE == tb_top_verilator
+```sh
+make verilate VLT_TOP_MODULE=tb_top_verilator
+make sanity-veri-run
+```
+Output similar to:    
+```
+HELLO WORLD!!!
+This is the OpenHW Group CV32E40P CORE-V processor core.
+CV32E40P is a RISC-V ISA compliant core with the following attributes:
+        mvendorid = 0x602
+        marchid   = 0x4
+        mimpid    = 0x0
+        misa      = 0x40001104
+        XLEN is 32-bits
+        Supported Instructions Extensions: MIC
+
+TOP.tb_top_verilator @ 130110: EXIT SUCCESS
+- /home/uic52463/hdd2/isolde-project/redmule/tb/core/tb_top_verilator.sv:83: Verilog $finish
+```
+## build sw
+get a clean slate
+```sh
+make sw-clean
+```
+```sh
+make golden
+make sw-build
+```
+the following files have been generated:  
+* vsim/stim_instr.txt
+* vsim/stim_data.txt
+
 # RedMulE
 RedMulE (**Red**uced-Precision Matrix **Mul**tiplication **E**ngine) is an open-source hardware accelerator based on the [HWPE](https://hwpe-doc.readthedocs.io/en/latest/index.html) template. It is designed to accelerate General Matrix-Matrix Operations (GEMM-Ops) on Floating-Point (FP) FP16 and FP8 input matrices. The keyword GEMM-Ops includes all the matrix operations of the kind **Z = (X op1 W) op2 Z**. The operators *op1* and *op2* can be any of those grouped in the following table:
 
@@ -21,7 +75,7 @@ If you want to use RedMulE for academic purposes, please cite it as:
 
 ```
 @article{TORTORELLA2023122,
-title = {RedMule: A mixed-precision matrix�matrix operation engine for flexible and energy-efficient on-chip linear algebra and TinyML training acceleration},
+title = {RedMule: A mixed-precision matrix�matrix operation engine for flexible and energy-efficient on-chip linear algebra and TinyML training acceleration},
 journal = {Future Generation Computer Systems},
 volume = {149},
 pages = {122-135},

bender_sim.mk

@@ -8,8 +8,12 @@
 sim_targs += -t rtl
 sim_targs += -t test
 
+VLT_BENDER += -t rtl
+
 ifeq ($(REDMULE_COMPLEX),1)
-	sim_targs += -t redmule_test_complex
+	sim_targs  += -t redmule_test_complex
+	VLT_BENDER += -t redmule_test_complex
 else
-	sim_targs += -t redmule_test_hwpe
+	sim_targs  += -t redmule_test_hwpe
+	VLT_BENDER += -t redmule_test_hwpe
 endif