Merge branch 'main' into hls4ml-optimization-api-part-2

.pre-commit-config.yaml

@@ -2,15 +2,15 @@ exclude: (^hls4ml\/templates\/(vivado|quartus)\/(ap_types|ac_types)\/|^test/pyte
 
 repos:
 - repo: https://github.com/psf/black
-  rev: 24.8.0
+  rev: 24.10.0
   hooks:
   - id: black
     language_version: python3
     args: ['--line-length=125',
            '--skip-string-normalization']
 
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.6.0
+  rev: v5.0.0
   hooks:
   - id: check-added-large-files
   - id: check-case-conflict
@@ -30,13 +30,13 @@ repos:
     args: ["--profile", "black", --line-length=125]
 
 - repo: https://github.com/asottile/pyupgrade
-  rev: v3.17.0
+  rev: v3.18.0
   hooks:
   - id: pyupgrade
     args: ["--py36-plus"]
 
 - repo: https://github.com/asottile/setup-cfg-fmt
-  rev: v2.5.0
+  rev: v2.7.0
   hooks:
   - id: setup-cfg-fmt
 
@@ -50,7 +50,7 @@ repos:
            '--extend-ignore=E203,T201']  # E203 is not PEP8 compliant
 
 - repo: https://github.com/mgedmin/check-manifest
-  rev: "0.49"
+  rev: "0.50"
   hooks:
   - id: check-manifest
     stages: [manual]

Jenkinsfile

@@ -16,7 +16,7 @@ pipeline {
           sh '''#!/bin/bash --login
               conda activate hls4ml-py310
               conda install -y jupyterhub pydot graphviz pytest pytest-cov
-              pip install pytest-randomly jupyter onnx>=1.4.0 matplotlib pandas seaborn pydigitalwavetools==1.1 pyyaml tensorflow==2.14 qonnx torch git+https://github.com/google/qkeras.git pyparsing
+              pip install pytest-randomly jupyter onnx>=1.4.0 matplotlib pandas seaborn pydigitalwavetools==1.1 pyyaml tensorflow==2.14 qonnx torch git+https://github.com/jmitrevs/qkeras.git@qrecurrent_unstack pyparsing
               pip install -U ../ --user
               ./convert-keras-models.sh -x -f keras-models.txt
               pip uninstall hls4ml -y'''

hls4ml/backends/fpga/fpga_backend.py

@@ -1,6 +1,6 @@
 import math
-import os
 import re
+import subprocess
 from bisect import bisect_left
 from collections.abc import Iterable
 
@@ -131,19 +131,22 @@ def compile(self, model):
         Returns:
             string: Returns the name of the compiled library.
         """
-        curr_dir = os.getcwd()
-        os.chdir(model.config.get_output_dir())
 
         lib_name = None
-        try:
-            ret_val = os.system('bash build_lib.sh')
-            if ret_val != 0:
-                raise Exception(f'Failed to compile project "{model.config.get_project_name()}"')
-            lib_name = '{}/firmware/{}-{}.so'.format(
-                model.config.get_output_dir(), model.config.get_project_name(), model.config.get_config_value('Stamp')
-            )
-        finally:
-            os.chdir(curr_dir)
+        ret_val = subprocess.run(
+            ['./build_lib.sh'],
+            shell=True,
+            text=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            cwd=model.config.get_output_dir(),
+        )
+        if ret_val.returncode != 0:
+            print(ret_val.stdout)
+            raise Exception(f'Failed to compile project "{model.config.get_project_name()}"')
+        lib_name = '{}/firmware/{}-{}.so'.format(
+            model.config.get_output_dir(), model.config.get_project_name(), model.config.get_config_value('Stamp')
+        )
 
         return lib_name
 

hls4ml/templates/quartus/build_lib.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -e
 
 CC=g++
 if [[ "$OSTYPE" == "linux-gnu" ]]; then

hls4ml/templates/vivado/build_lib.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -e
 
 CC=g++
 if [[ "$OSTYPE" == "linux-gnu" ]]; then
@@ -10,8 +11,9 @@ LDFLAGS=
 INCFLAGS="-Ifirmware/ap_types/"
 PROJECT=myproject
 LIB_STAMP=mystamp
+WEIGHTS_DIR="\"weights\""
 
-${CC} ${CFLAGS} ${INCFLAGS} -c firmware/${PROJECT}.cpp -o ${PROJECT}.o
-${CC} ${CFLAGS} ${INCFLAGS} -c ${PROJECT}_bridge.cpp -o ${PROJECT}_bridge.o
+${CC} ${CFLAGS} ${INCFLAGS} -D WEIGHTS_DIR=${WEIGHTS_DIR} -c firmware/${PROJECT}.cpp -o ${PROJECT}.o
+${CC} ${CFLAGS} ${INCFLAGS} -D WEIGHTS_DIR=${WEIGHTS_DIR} -c ${PROJECT}_bridge.cpp -o ${PROJECT}_bridge.o
 ${CC} ${CFLAGS} ${INCFLAGS} -shared ${PROJECT}.o ${PROJECT}_bridge.o -o firmware/${PROJECT}-${LIB_STAMP}.so
 rm -f *.o

hls4ml/writer/catapult_writer.py

@@ -1,7 +1,9 @@
 import glob
 import os
+import stat
 import tarfile
 from collections import OrderedDict
+from pathlib import Path
 from shutil import copyfile, copytree, rmtree
 
 import numpy as np
@@ -749,55 +751,50 @@ def write_build_script(self, model):
             model (ModelGraph): the hls4ml model.
         """
 
-        filedir = os.path.dirname(os.path.abspath(__file__))
+        filedir = Path(__file__).parent
 
         # build_prj.tcl
-        srcpath = os.path.join(filedir, '../templates/catapult/build_prj.tcl')
-        dstpath = f'{model.config.get_output_dir()}/build_prj.tcl'
-        # copyfile(srcpath, dstpath)
-        f = open(srcpath)
-        fout = open(dstpath, 'w')
-        for line in f.readlines():
-            indent = line[: len(line) - len(line.lstrip())]
-            line = line.replace('myproject', model.config.get_project_name())
-            line = line.replace('CATAPULT_DIR', model.config.get_project_dir())
-            if '#hls-fpga-machine-learning insert techlibs' in line:
-                if model.config.get_config_value('Technology') is None:
-                    if model.config.get_config_value('Part') is not None:
-                        line = indent + 'setup_xilinx_part {{{}}}\n'.format(model.config.get_config_value('Part'))
-                    elif model.config.get_config_value('ASICLibs') is not None:
-                        line = indent + 'setup_asic_libs {{{}}}\n'.format(model.config.get_config_value('ASICLibs'))
-                else:
-                    if model.config.get_config_value('Technology') == 'asic':
-                        line = indent + 'setup_asic_libs {{{}}}\n'.format(model.config.get_config_value('ASICLibs'))
+        srcpath = (filedir / '../templates/catapult/build_prj.tcl').resolve()
+        dstpath = Path(f'{model.config.get_output_dir()}/build_prj.tcl').resolve()
+        with open(srcpath) as src, open(dstpath, 'w') as dst:
+            for line in src.readlines():
+                indent = line[: len(line) - len(line.lstrip())]
+                line = line.replace('myproject', model.config.get_project_name())
+                line = line.replace('CATAPULT_DIR', model.config.get_project_dir())
+                if '#hls-fpga-machine-learning insert techlibs' in line:
+                    if model.config.get_config_value('Technology') is None:
+                        if model.config.get_config_value('Part') is not None:
+                            line = indent + 'setup_xilinx_part {{{}}}\n'.format(model.config.get_config_value('Part'))
+                        elif model.config.get_config_value('ASICLibs') is not None:
+                            line = indent + 'setup_asic_libs {{{}}}\n'.format(model.config.get_config_value('ASICLibs'))
                     else:
-                        line = indent + 'setup_xilinx_part {{{}}}\n'.format(model.config.get_config_value('Part'))
-            elif '#hls-fpga-machine-learning insert invoke_args' in line:
-                tb_in_file = model.config.get_config_value('InputData')
-                tb_out_file = model.config.get_config_value('OutputPredictions')
-                invoke_args = '$sfd/firmware/weights'
-                if tb_in_file is not None:
-                    invoke_args = invoke_args + f' $sfd/tb_data/{tb_in_file}'
-                if tb_out_file is not None:
-                    invoke_args = invoke_args + f' $sfd/tb_data/{tb_out_file}'
-                line = indent + f'flow package option set /SCVerify/INVOKE_ARGS "{invoke_args}"\n'
-            elif 'set hls_clock_period 5' in line:
-                line = indent + 'set hls_clock_period {}\n'.format(model.config.get_config_value('ClockPeriod'))
-            fout.write(line)
-        f.close()
-        fout.close()
+                        if model.config.get_config_value('Technology') == 'asic':
+                            line = indent + 'setup_asic_libs {{{}}}\n'.format(model.config.get_config_value('ASICLibs'))
+                        else:
+                            line = indent + 'setup_xilinx_part {{{}}}\n'.format(model.config.get_config_value('Part'))
+                elif '#hls-fpga-machine-learning insert invoke_args' in line:
+                    tb_in_file = model.config.get_config_value('InputData')
+                    tb_out_file = model.config.get_config_value('OutputPredictions')
+                    invoke_args = '$sfd/firmware/weights'
+                    if tb_in_file is not None:
+                        invoke_args = invoke_args + f' $sfd/tb_data/{tb_in_file}'
+                    if tb_out_file is not None:
+                        invoke_args = invoke_args + f' $sfd/tb_data/{tb_out_file}'
+                    line = indent + f'flow package option set /SCVerify/INVOKE_ARGS "{invoke_args}"\n'
+                elif 'set hls_clock_period 5' in line:
+                    line = indent + 'set hls_clock_period {}\n'.format(model.config.get_config_value('ClockPeriod'))
+                dst.write(line)
 
         # build_lib.sh
-        f = open(os.path.join(filedir, '../templates/catapult/build_lib.sh'))
-        fout = open(f'{model.config.get_output_dir()}/build_lib.sh', 'w')
-
-        for line in f.readlines():
-            line = line.replace('myproject', model.config.get_project_name())
-            line = line.replace('mystamp', model.config.get_config_value('Stamp'))
-
-            fout.write(line)
-        f.close()
-        fout.close()
+        build_lib_src = (filedir / '../templates/catapult/build_lib.sh').resolve()
+        build_lib_dst = Path(f'{model.config.get_output_dir()}/build_lib.sh').resolve()
+        with open(build_lib_src) as src, open(build_lib_dst, 'w') as dst:
+            for line in src.readlines():
+                line = line.replace('myproject', model.config.get_project_name())
+                line = line.replace('mystamp', model.config.get_config_value('Stamp'))
+
+                dst.write(line)
+        build_lib_dst.chmod(build_lib_dst.stat().st_mode | stat.S_IEXEC)
 
     def write_nnet_utils(self, model):
         """Copy the nnet_utils, AP types headers and any custom source to the project output directory

hls4ml/writer/quartus_writer.py

@@ -1,7 +1,9 @@
 import glob
 import os
+import stat
 import tarfile
 from collections import OrderedDict
+from pathlib import Path
 from shutil import copyfile, copytree, rmtree
 
 import numpy as np
@@ -877,32 +879,30 @@ def write_build_script(self, model):
             model (ModelGraph): the hls4ml model.
         """
 
-        # Makefile
-        filedir = os.path.dirname(os.path.abspath(__file__))
-        f = open(os.path.join(filedir, '../templates/quartus/Makefile'))
-        fout = open(f'{model.config.get_output_dir()}/Makefile', 'w')
+        filedir = Path(__file__).parent
 
-        for line in f.readlines():
-            line = line.replace('myproject', model.config.get_project_name())
+        # Makefile
+        makefile_src = (filedir / '../templates/quartus/Makefile').resolve()
+        makefile_dst = Path(f'{model.config.get_output_dir()}/Makefile').resolve()
+        with open(makefile_src) as src, open(makefile_dst, 'w') as dst:
+            for line in src.readlines():
+                line = line.replace('myproject', model.config.get_project_name())
 
-            if 'DEVICE   :=' in line:
-                line = 'DEVICE   := {}\n'.format(model.config.get_config_value('Part'))
+                if 'DEVICE   :=' in line:
+                    line = 'DEVICE   := {}\n'.format(model.config.get_config_value('Part'))
 
-            fout.write(line)
-        f.close()
-        fout.close()
+                dst.write(line)
 
         # build_lib.sh
-        f = open(os.path.join(filedir, '../templates/quartus/build_lib.sh'))
-        fout = open(f'{model.config.get_output_dir()}/build_lib.sh', 'w')
-
-        for line in f.readlines():
-            line = line.replace('myproject', model.config.get_project_name())
-            line = line.replace('mystamp', model.config.get_config_value('Stamp'))
-
-            fout.write(line)
-        f.close()
-        fout.close()
+        build_lib_src = (filedir / '../templates/quartus/build_lib.sh').resolve()
+        build_lib_dst = Path(f'{model.config.get_output_dir()}/build_lib.sh').resolve()
+        with open(build_lib_src) as src, open(build_lib_dst, 'w') as dst:
+            for line in src.readlines():
+                line = line.replace('myproject', model.config.get_project_name())
+                line = line.replace('mystamp', model.config.get_config_value('Stamp'))
+
+                dst.write(line)
+        build_lib_dst.chmod(build_lib_dst.stat().st_mode | stat.S_IEXEC)
 
     def write_nnet_utils(self, model):
         """Copy the nnet_utils, AP types headers and any custom source to the project output directory

hls4ml/writer/symbolic_writer.py

@@ -1,5 +1,7 @@
 import glob
 import os
+import stat
+from pathlib import Path
 from shutil import copyfile, copytree, rmtree
 
 from hls4ml.backends import get_backend
@@ -56,49 +58,48 @@ def write_build_script(self, model):
             model (ModelGraph): the hls4ml model.
         """
 
-        filedir = os.path.dirname(os.path.abspath(__file__))
-
-        # build_prj.tcl
-        f = open(f'{model.config.get_output_dir()}/project.tcl', 'w')
-        f.write('variable project_name\n')
-        f.write(f'set project_name "{model.config.get_project_name()}"\n')
-        f.write('variable backend\n')
-        f.write('set backend "vivado"\n')
-        f.write('variable part\n')
-        f.write('set part "{}"\n'.format(model.config.get_config_value('Part')))
-        f.write('variable clock_period\n')
-        f.write('set clock_period {}\n'.format(model.config.get_config_value('ClockPeriod')))
-        f.write('variable clock_uncertainty\n')
-        f.write('set clock_uncertainty {}\n'.format(model.config.get_config_value('ClockUncertainty', '0%')))
-        f.write('variable version\n')
-        f.write('set version "{}"\n'.format(model.config.get_config_value('Version', '1.0.0')))
-        f.close()
+        filedir = Path(__file__).parent
+
+        # project.tcl
+        prj_tcl_dst = Path(f'{model.config.get_output_dir()}/project.tcl')
+        with open(prj_tcl_dst, 'w') as f:
+            f.write('variable project_name\n')
+            f.write(f'set project_name "{model.config.get_project_name()}"\n')
+            f.write('variable backend\n')
+            f.write('set backend "vivado"\n')
+            f.write('variable part\n')
+            f.write('set part "{}"\n'.format(model.config.get_config_value('Part')))
+            f.write('variable clock_period\n')
+            f.write('set clock_period {}\n'.format(model.config.get_config_value('ClockPeriod')))
+            f.write('variable clock_uncertainty\n')
+            f.write('set clock_uncertainty {}\n'.format(model.config.get_config_value('ClockUncertainty', '0%')))
+            f.write('variable version\n')
+            f.write('set version "{}"\n'.format(model.config.get_config_value('Version', '1.0.0')))
 
         # build_prj.tcl
-        srcpath = os.path.join(filedir, '../templates/vivado/build_prj.tcl')
+        srcpath = (filedir / '../templates/vivado/build_prj.tcl').resolve()
         dstpath = f'{model.config.get_output_dir()}/build_prj.tcl'
         copyfile(srcpath, dstpath)
 
         # vivado_synth.tcl
-        srcpath = os.path.join(filedir, '../templates/vivado/vivado_synth.tcl')
+        srcpath = (filedir / '../templates/vivado/vivado_synth.tcl').resolve()
         dstpath = f'{model.config.get_output_dir()}/vivado_synth.tcl'
         copyfile(srcpath, dstpath)
 
         # build_lib.sh
-        f = open(os.path.join(filedir, '../templates/symbolic/build_lib.sh'))
-        fout = open(f'{model.config.get_output_dir()}/build_lib.sh', 'w')
-
-        for line in f.readlines():
-            line = line.replace('myproject', model.config.get_project_name())
-            line = line.replace('mystamp', model.config.get_config_value('Stamp'))
-            line = line.replace('mylibspath', model.config.get_config_value('HLSLibsPath'))
-
-            if 'LDFLAGS=' in line and not os.path.exists(model.config.get_config_value('HLSLibsPath')):
-                line = 'LDFLAGS=\n'
-
-            fout.write(line)
-        f.close()
-        fout.close()
+        build_lib_src = (filedir / '../templates/symbolic/build_lib.sh').resolve()
+        build_lib_dst = Path(f'{model.config.get_output_dir()}/build_lib.sh').resolve()
+        with open(build_lib_src) as src, open(build_lib_dst, 'w') as dst:
+            for line in src.readlines():
+                line = line.replace('myproject', model.config.get_project_name())
+                line = line.replace('mystamp', model.config.get_config_value('Stamp'))
+                line = line.replace('mylibspath', model.config.get_config_value('HLSLibsPath'))
+
+                if 'LDFLAGS=' in line and not os.path.exists(model.config.get_config_value('HLSLibsPath')):
+                    line = 'LDFLAGS=\n'
+
+                dst.write(line)
+        build_lib_dst.chmod(build_lib_dst.stat().st_mode | stat.S_IEXEC)
 
     def write_hls(self, model):
         print('Writing HLS project')