From 44507194d49bf7180966e79acc259d838893767d Mon Sep 17 00:00:00 2001 From: Tiago Oliveira Date: Mon, 5 Aug 2024 20:27:21 +0100 Subject: [PATCH] libjade multirepo support : take 1 --- scripts/releaser/extract-libjade | 97 ++++++++++++++++++++++++++++++++ scripts/releaser/jpp | 36 +++++++++--- src/.gitignore | 7 ++- src/Makefile | 80 ++++++++++++++++++++++---- src/Makefile.common | 42 ++++++++++++-- 5 files changed, 234 insertions(+), 28 deletions(-) create mode 100755 scripts/releaser/extract-libjade mode change 100644 => 100755 scripts/releaser/jpp diff --git a/scripts/releaser/extract-libjade b/scripts/releaser/extract-libjade new file mode 100755 index 0000000..3fa1234 --- /dev/null +++ b/scripts/releaser/extract-libjade @@ -0,0 +1,97 @@ +#!/usr/bin/env bash + +# pro user tip (because this repository shares the same structure as in libjade); assumes libjade is next to formosa-25519 +# +# ./scripts/releaser/extract-libjade --list-implementations | while read implementation; do ./scripts/releaser/extract-libjade --gen-implementation $implementation ../libjade/$(realpath --relative-to="$(pwd)" $implementation); done +# + +call=$0 +script_dir=$(cd "$(dirname "$0")" ; pwd -P) +top_dir=$(cd "$(dirname "$0")/../../" ; pwd -P) + +print_usage() +{ + echo "usage:" + echo " $ $call --list-implementations" + echo " $ $call --gen-implementation IMPLEMENTATION DIRECTORY" +} + +# if there are no arguments, print usage +if [ $# -eq 0 ]; then + print_usage + exit 1 +fi + +# check if --list-implementations +if [ "$1" == "--list-implementations" ]; then + make --no-print-directory -C $top_dir/src print-available-implementations + exit 1 +fi + +# check if --gen-implementation +if [ "$1" == "--gen-implementation" ]; then + # if we are generating an implementation then there should be 3 args + if [ $# -eq 3 ]; then + + # start by realpath them (useful to run make) + implementation=$(realpath $2) + directory=$(realpath $3) + + # test if IMPLEMENTATION directory exists + if [ ! -d "$implementation" ]; then + echo " IMPLEMENTATION: $implementation does not exist." + exit 1; + fi + + # test if libjade DIRECTORY exists + if [ ! -d "$directory" ]; then + echo " DIRECTORY: $directory does not exist." + exit 1; + fi + + relative_implementation=$(realpath --relative-to="$top_dir/src" "$implementation") + make --no-print-directory -C $top_dir/src/ $relative_implementation/preprocess-inplace + + ############################################################################# + # copy the preprocessed files + + jazz_files=$(find $implementation -name '*.jazz') + for file in $jazz_files; do + cp $file $directory/ + done + + # setup the Makefile + echo -n "SRCS := " > $directory/Makefile + for file in $jazz_files; do + echo -n $(basename $file) >> $directory/Makefile + done + echo "" >> $directory/Makefile + + # NOTE: the following line will need change (or be deleted) once multi-repo libjade is stable + echo "include ../../../../Makefile.common" >> $directory/Makefile + + # NOTE: the following command will change once there is a PR in libjade to move api.h files out of include/ directories + cp $implementation/include/api.h $directory/include/api.h + + # + ############################################################################# + + # restore implementation state + make --no-print-directory -C $top_dir/src/ $relative_implementation/revert-preprocess-inplace + + exit 1; + + else + echo "error: --gen-implementation : number of required arguments 3 : provided $#" + print_usage + exit 1 + fi +fi + +# with 'good' options this should be unreachable, hence, print usage +print_usage +exit 1 + + + + diff --git a/scripts/releaser/jpp b/scripts/releaser/jpp old mode 100644 new mode 100755 index dcb1983..598ac4a --- a/scripts/releaser/jpp +++ b/scripts/releaser/jpp @@ -41,6 +41,8 @@ sub jpp_main() my $dry_run = 0; # if 1, do not write to any file my $safe_reverse = 1; # if reverse, ask the user if the changes are ok + my $bor_prefix = undef; # path to be removed from the full path in REQUIRE_ENTRY_TAG + # get the command-line arguments GetOptions( 'in=s' => \$in, @@ -51,7 +53,8 @@ sub jpp_main() 'reverse' => \$reverse, 'help|h' => \$help, 'dry-run' => \$dry_run, - 'check-diff!' => \$safe_reverse + 'check-diff!' => \$safe_reverse, + 'bor-prefix|b=s' => \$bor_prefix ); # check -help or -h @@ -66,7 +69,7 @@ sub jpp_main() if($reverse) { - my ($reverse_hr) = jpp_reverse($in, $out, $dry_run, $safe_reverse, $verbose); + my ($reverse_hr) = jpp_reverse($in, $out, $dry_run, $safe_reverse, $verbose, $bor_prefix); } else { # return variables description: @@ -75,7 +78,7 @@ sub jpp_main() # visited_hr: hash (reference) intuitively, 'children' -> '[parents]' # for each file (canonical path) that was visited, visited_hr contains the list of files that have it required (in the context of preprocessing in) # preprocessed_ar: array (reference) with the contents of the preprocessed file (if out is defined, then contents for preprocessed_ar were printed to the output file) - my ($preprocessed_ar, $visited_hr, $include_hr) = jpp($in, $out, \@include_a, $with_reverse_info, $dry_run, $verbose); + my ($preprocessed_ar, $visited_hr, $include_hr) = jpp($in, $out, \@include_a, $with_reverse_info, $dry_run, $verbose, $bor_prefix); } } @@ -89,6 +92,7 @@ sub jpp() my $with_reverse_info = shift; # with_reverse_info (-with_reverse_info) my $dry_run = shift; # do not write to files (-dry-run) my $verbose = shift; # verbose (-v) + my $bor_prefix = shift; my %visited_h = (); # 'children' -> '[parents]' @@ -110,7 +114,7 @@ sub jpp() # - it receives as arguments: # -- $in (first file); $include_hr (hash ref. to "-I"'s); \%visited_h (hash ref to visited files, initially empty); # -- verbose (to print some debug info); level (to keep track of current search depth and do indentation, initially 0); - my ($preprocessed_ar, $visited_hr) = preprocess($in, "entry", $include_hr, \%visited_h, $with_reverse_info, $verbose, 0); + my ($preprocessed_ar, $visited_hr) = preprocess($in, "entry", $include_hr, \%visited_h, $with_reverse_info, $verbose, 0, $bor_prefix); store($out, $preprocessed_ar, $dry_run); print_visited($visited_hr) if($verbose); @@ -127,6 +131,8 @@ sub jpp_reverse() my $dry_run = shift; # do not write to files (-dry-run), print to stdout instead my $safe_reverse = shift; # ask the user about changes in files my $verbose = shift; # verbose (-v) + my $bor_prefix = shift; + my %reverse_h = (); # 'files' -> '[code]' (final step is to update files with the code) # get the absolute path and load the contents of the file @@ -136,7 +142,7 @@ sub jpp_reverse() # if out is not defined, update inplace (write the output in $in) $out = (defined $out) ? realpath($out) : $in; - my ($line_position, $reverse_hr) = preprocess_reverse($in_lines_ar, 0, \%reverse_h, $out, $verbose); + my ($line_position, $reverse_hr) = preprocess_reverse($in_lines_ar, 0, \%reverse_h, $out, $verbose, $bor_prefix); if( check_changes_reverse($in, $out, $reverse_hr, $safe_reverse) == 1 ) { store_reverse($reverse_hr, $dry_run); } @@ -193,6 +199,7 @@ sub preprocess my $with_reverse_info = shift; # include information to undo preprocessing my $verbose = shift; # print some internal information my $level = shift; # current level of recursion + my $bor_prefix = shift; # if file was already visited, then return an empty list if (defined $visited_hr->{$in}) @@ -225,12 +232,18 @@ sub preprocess # otherwise concat with base from in $file = realpath($file); + my $file_path_to_print = $file; + + # remove from file some prefix + if(defined $bor_prefix) + { $file_path_to_print =~ s/^\Q$bor_prefix\E//; } + ### if ($with_reverse_info) { my $r_string = $REQUIRE_ENTRY_TAG . $line . $REQUIRE_SEPARATOR_TAG . - $file; + $file_path_to_print; push (@preprocessed, $r_string); } # NOTE: this includes repetead files (it is not a problem; in the event that one would like to remove these from the code, some quick patch might be able to do it;) ### @@ -239,7 +252,7 @@ sub preprocess # avoid calling preprocess for a file that was already required if (not defined $visited_hr->{$file}) - { my ($sublines_ar, $visited_hr) = preprocess($file, $in, $include_hr, $visited_hr, $with_reverse_info, $verbose, $level+1); # fetch contents of other files + { my ($sublines_ar, $visited_hr) = preprocess($file, $in, $include_hr, $visited_hr, $with_reverse_info, $verbose, $level+1, $bor_prefix); # fetch contents of other files push @preprocessed, @$sublines_ar; # add to the array the lines from the requires } @@ -359,7 +372,7 @@ sub store_reverse sub preprocess_reverse { - my ($lines_ar, $line_position, $reverse_hr, $current_file_name, $verbose) = @_; + my ($lines_ar, $line_position, $reverse_hr, $current_file_name, $verbose, $bor_prefix) = @_; # initialize array for code (if not defined) if (not defined $reverse_hr->{$current_file_name}) @@ -375,8 +388,13 @@ sub preprocess_reverse # if ENTRY_tag, call preprocess_reverse if($line =~ /^$REQUIRE_ENTRY_TAG/) { my ($tag, $require_statement, $file_name) = split($REQUIRE_SEPARATOR_TAG, $line); + + # add to file some prefix + if(defined $bor_prefix) + { $file_name = $bor_prefix . $file_name; } + push @{$reverse_hr->{$current_file_name}}, $require_statement; - ($line_position, $reverse_hr) = preprocess_reverse($lines_ar, ($line_position+1), $reverse_hr, $file_name, $verbose); + ($line_position, $reverse_hr) = preprocess_reverse($lines_ar, ($line_position+1), $reverse_hr, $file_name, $verbose, $bor_prefix); next; } diff --git a/src/.gitignore b/src/.gitignore index 2b6043a..9200e0e 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -1,3 +1,4 @@ +*.jpp *.s *.safety *.safety_* @@ -6,11 +7,11 @@ *.o *.a _build/ -libjade.a -libjade.h +libformosa25519.a +libformosa25519.h .deps .ec .log .error -.jflags +.JFLAGS .ci/ diff --git a/src/Makefile b/src/Makefile index e37193d..a44c095 100644 --- a/src/Makefile +++ b/src/Makefile @@ -11,6 +11,7 @@ CFLAGS ?= -O3 -Wall -Wextra -Wpedantic -Wvla -Werror -std=c99 \ AR ?= ar RANLIB ?= ranlib + # ----------------------------------------------------------------------------- SRC := . @@ -18,12 +19,14 @@ PROOF := $(abspath ../proof) FILTER ?= $(SRC)/crypto_% EXCLUDE ?= -JASMIN_IMPLEMENTATIONS ?= $(filter $(FILTER), $(filter-out $(EXCLUDE), $(sort $(dir $(shell find $(SRC) -name '*.jazz'))))) +IMPLEMENTATIONS_FILES ?= $(filter $(FILTER), $(filter-out $(EXCLUDE), $(sort $(shell find $(SRC)/ -name '*.jazz')))) +IMPLEMENTATIONS_DIRECTORIES := $(dir $(IMPLEMENTATIONS_FILES)) -ASSEMBLY_FILES ?= $(shell find $(SRC) -name '*.s') +ASSEMBLY_FILES ?= $(IMPLEMENTATIONS_FILES:%.jazz=%.s) OBJECT_FILES := $(ASSEMBLY_FILES:%.s=%.o) C_HEADER_FILES ?= $(addsuffix include/api.h, $(dir $(ASSEMBLY_FILES))) + # ----------------------------------------------------------------------------- # when CI=1, information about compilation/checking properties of the files # will be stored under .ci/ folders in the implementations directories. Then @@ -34,22 +37,24 @@ CI_REMOVE_OK_LOGS ?= "1" export CI + # ----------------------------------------------------------------------------- # setting the default rule as 'all': compiles, and checks safety and sct for # all implementations default: all + # ----------------------------------------------------------------------------- # compile Jasmin implementations to assembly and create libformosa25519.a -.PHONY: libformosa25519.a libformosa25519.h $(JASMIN_IMPLEMENTATIONS) +.PHONY: libformosa25519.a libformosa25519.h $(ASSEMBLY_FILES) .INTERMEDIATE: $(OBJECT_FILES) -just_compile_jasmin_implementations: $(JASMIN_IMPLEMENTATIONS) +just_compile: $(ASSEMBLY_FILES) -libformosa25519.a: $(JASMIN_IMPLEMENTATIONS) +libformosa25519.a: $(ASSEMBLY_FILES) $(MAKE) __libformosa25519 __libformosa25519: $(OBJECT_FILES) @@ -57,13 +62,14 @@ __libformosa25519: $(OBJECT_FILES) $(RANLIB) libformosa25519.a echo "" | cat - $(C_HEADER_FILES) > libformosa25519.h -$(JASMIN_IMPLEMENTATIONS): - $(MAKE) -C $@ +$(ASSEMBLY_FILES): + $(MAKE) -C $(@D) $(@F) + # ----------------------------------------------------------------------------- # safety checking -CHECK_SAFETY ?= $(addsuffix check-safety, $(JASMIN_IMPLEMENTATIONS)) +CHECK_SAFETY ?= $(addsuffix check-safety, $(IMPLEMENTATIONS_DIRECTORIES)) CHECK_SAFETY_TIMEOUT ?= 10m export CHECK_SAFETY_TIMEOUT @@ -75,10 +81,11 @@ check-safety: $(CHECK_SAFETY) $(CHECK_SAFETY): $(MAKE) -C $(@D) $(@F) + # ----------------------------------------------------------------------------- # speculative "constant-time" checking / "secret-independence" checking -CHECK_SCT ?= $(addsuffix check-sct, $(JASMIN_IMPLEMENTATIONS)) +CHECK_SCT ?= $(addsuffix check-sct, $(IMPLEMENTATIONS_DIRECTORIES)) .PHONY: check-sct $(CHECK_SCT) @@ -87,10 +94,11 @@ check-sct: $(CHECK_SCT) $(CHECK_SCT): $(MAKE) -C $(@D) $(@F) + # ----------------------------------------------------------------------------- # extraction to easycrypt (updates proof/ directory) -EXTRACT_TO_EASYCRYPT ?= $(addsuffix extract-to-easycrypt, $(JASMIN_IMPLEMENTATIONS)) +EXTRACT_TO_EASYCRYPT ?= $(addsuffix extract-to-easycrypt, $(IMPLEMENTATIONS_DIRECTORIES)) .PHONY: extract-to-easycrypt $(EXTRACT_TO_EASYCRYPT) @@ -99,6 +107,30 @@ extract-to-easycrypt: $(EXTRACT_TO_EASYCRYPT) $(EXTRACT_TO_EASYCRYPT): $(MAKE) -C $(@D) $(@F) + +# ----------------------------------------------------------------------------- +# preprocess-inplace and revert-preprocess-inplace + +PREPROCESS_INPLACE ?= $(addsuffix preprocess-inplace, $(IMPLEMENTATIONS_DIRECTORIES)) + +.PHONY: preprocess-inplace $(PREPROCESS_INPLACE) + +preprocess-inplace: $(PREPROCESS_INPLACE) + +$(PREPROCESS_INPLACE): + $(MAKE) -C $(@D) $(@F) + + +REVERT_PREPROCESS_INPLACE ?= $(addsuffix revert-preprocess-inplace, $(IMPLEMENTATIONS_DIRECTORIES)) + +.PHONY: preprocess-inplace $(REVERT_PREPROCESS_INPLACE) + +revert-preprocess-inplace: $(REVERT_PREPROCESS_INPLACE) + +$(REVERT_PREPROCESS_INPLACE): + $(MAKE) -C $(@D) $(@F) + + # ----------------------------------------------------------------------------- # reporter: this section defines rules for reporting the current implementation # status @@ -134,6 +166,7 @@ ifneq ($(words $(CI_ERROR_FILES)),0) $(error $(CI_ERROR_FILES)) endif + # ----------------------------------------------------------------------------- # to run 'everything': $ make -j$(nproc) all .PHONY: all @@ -149,6 +182,19 @@ all: $(MAKE) reporter $(MAKE) err +all-with-preprocess-inplace: CI=1 +all-with-preprocess-inplace: + $(MAKE) distclean + $(MAKE) preprocess-inplace + $(MAKE) check-safety + $(MAKE) check-sct + $(MAKE) extract-to-easycrypt + $(MAKE) libformosa25519.a + $(MAKE) libformosa25519.h + $(MAKE) reporter + $(MAKE) err + + # ----------------------------------------------------------------------------- # clean rules @@ -157,11 +203,20 @@ all: clean: rm -f libformosa25519.a libformosa25519.h rm -f $(LOGS) - for i in $(JASMIN_IMPLEMENTATIONS); do $(MAKE) -C $$i clean; done + for i in $(IMPLEMENTATIONS_DIRECTORIES); do $(MAKE) -C $$i clean; done rm -f $(PROOF)/arrays/*Array*.ec distclean: clean + +# ----------------------------------------------------------------------------- +# printing section +.PHONHY: print-available-implementations + +print-available-implementations: + @for i in $(IMPLEMENTATIONS_DIRECTORIES); do (cd $$i && pwd); done + + # ----------------------------------------------------------------------------- # debug/print rules: helps to check the effects of FILTER or EXCLUDE @@ -171,7 +226,8 @@ debug-print-variables: @echo " PROOF: $(PROOF)\n" @echo " FILTER: $(FILTER)\n" @echo " EXCLUDE: $(EXCLUDE)\n" - @echo " JASMIN_IMPLEMENTATIONS: $(JASMIN_IMPLEMENTATIONS)\n" + @echo " IMPLEMENTATIONS_DIRECTORIES: $(IMPLEMENTATIONS_DIRECTORIES)\n" + @echo " IMPLEMENTATIONS_FILES: $(IMPLEMENTATIONS_FILES)\n" @echo " ASSEMBLY_FILES: $(ASSEMBLY_FILES)\n" @echo " OBJECT_FILES: $(OBJECT_FILES)\n" diff --git a/src/Makefile.common b/src/Makefile.common index a2f92b4..ac175e6 100644 --- a/src/Makefile.common +++ b/src/Makefile.common @@ -51,6 +51,14 @@ JASMIN_COMPILE = ($(JASMIN) $(JFLAGS) $(JINCLUDE) -o $@ $<) $(CI_CMD) ASSEMBLY_FILES = $(SRCS:%.$(JEXT)=%.s) + +# preprocessing vars + +JPP ?= $(PROJECT_DIR)/scripts/releaser/jpp +PREPROCESSED_FILES := $(SRCS:%.$(JEXT)=%.jpp) +JASMIN_PREPROCESS = ($(JPP) $(JINCLUDE) -b $(PROJECT_DIR) -out $@ -in $<) $(CI_CMD) + + # ----------------------------------------------------------------------------- # extraction vars @@ -62,6 +70,7 @@ JASMIN_EXTRACT = (cd $(EXTRACT_DIR) && $(JASMIN) $(JINCLUDE) $(EFLAGS) -oecarray EASYCRYPT_FILES = $(SRCS:%.$(JEXT)=$(EXTRACT_DIR)/%_s.ec) + # ----------------------------------------------------------------------------- # dependencies vars @@ -69,12 +78,30 @@ DEPS_DIR ?= .deps DEPS = ((printf "$@: "; printf "$< "; $(JASMIN) $(JINCLUDE) -print-dependencies $<) > $(DEPS_DIR)/$(@F).d) $(CI_CMD) DEPFILES ?= + # ----------------------------------------------------------------------------- # default rule default: compile +# ----------------------------------------------------------------------------- +# preprocess rules + +preprocess: $(PREPROCESSED_FILES) + @true + +$(PREPROCESSED_FILES): +%.jpp: %.$(JEXT) $(DEPS_DIR)/%.s.d | $(DEPS_DIR) $(CI_DIR) + $(DEPS) + $(JASMIN_PREPROCESS) + +preprocess-inplace: preprocess + for file in $(SRCS); do mv $${file%.$(JEXT)}.jpp $$file; done + +revert-preprocess-inplace: + for file in $(SRCS); do git checkout $$file; done + # ----------------------------------------------------------------------------- # compilation rules @@ -130,16 +157,18 @@ include $(wildcard $(DEPFILES)) # the following rule records the flags used to compile the implementation # these are used by the release script -.PHONY: .jflags -.jflags: - @echo -n "$(JFLAGS)" > .jflags +.PHONY: .JFLAGS +.JFLAGS: + @echo -n "$(JFLAGS)" > .JFLAGS # -------------------------------------------------------------------- .PHONY: clean distclean clean: - rm -fr $(CI_DIR) $(DEPS_DIR) *.s *.safety* *.sct* *.o .jflags + rm -fr $(CI_DIR) $(DEPS_DIR) *.jpp *.s *.safety* *.sct* *.o .JFLAGS + +distclean: clean rm -fr $(EASYCRYPT_FILES) # -------------------------------------------------------------------- @@ -171,6 +200,11 @@ debug-print-variables: @echo " ASSEMBLY_FILES: $(ASSEMBLY_FILES)" @echo " JASMIN_COMPILE: not printed, it is rule dependent." @echo "" + @echo "Preprocess variables:" + @echo " JPP: $(JPP)" + @echo " PREPROCESSED_FILES: $(PREPROCESSED_FILES)" + @echo " JASMIN_PREPROCESS: not printed, it is rule dependent." + @echo "" @echo "Extraction variables:" @echo " EXTRACT_DIR: $(EXTRACT_DIR)" @echo " EXTRACT_ARRAY_DIR: $(EXTRACT_ARRAY_DIR)"