Update upstream and refactor project structure

.gitignore

@@ -53,3 +53,4 @@ dkms.conf
 
 .vscode
 
+build

.gitmodules

@@ -1,6 +1,6 @@
-[submodule "deps/ckb-c-stdlib"]
-	path = deps/ckb-c-stdlib
-	url = https://github.com/nervosnetwork/ckb-c-stdlib.git
 [submodule "deps/compiler-rt-builtins-riscv"]
 	path = deps/compiler-rt-builtins-riscv
 	url = https://github.com/nervosnetwork/compiler-rt-builtins-riscv.git
+[submodule "deps/ckb-c-stdlib"]
+	path = deps/ckb-c-stdlib
+	url = https://github.com/nervosnetwork/ckb-c-stdlib

Makefile

@@ -13,68 +13,116 @@ ifeq ($(UNAME), Darwin)
 	RANLIB := llvm-ranlib
 endif
 
-CFLAGS := --target=riscv64 -march=rv64imc_zba_zbb_zbc_zbs
-CFLAGS += -g -Oz \
-		-Wall -Werror -Wno-nonnull -Wno-unused-function \
-		-fno-builtin-printf -fno-builtin-memcmp \
-		-nostdinc -nostdlib \
-		-fdata-sections -ffunction-sections
-
-CFLAGS += -I deps/ckb-c-stdlib/libc -I deps/ckb-c-stdlib
-CFLAGS += -I include -I include/c-stdlib
-CFLAGS += -I deps/compiler-rt-builtins-riscv/compiler-rt/lib/builtins
-
-CFLAGS += -Wextra -Wno-sign-compare -Wno-missing-field-initializers -Wundef -Wuninitialized \
-		  -Wunused -Wno-unused-parameter -Wchar-subscripts -funsigned-char -Wno-unused-function \
-		  -DCONFIG_VERSION=\"2021-03-27-CKB\"
-CFLAGS += -Wno-incompatible-library-redeclaration -Wno-implicit-const-int-float-conversion -Wno-invalid-noreturn
-
-CFLAGS += -DCKB_DECLARATION_ONLY
-CFLAGS += -D__BYTE_ORDER=1234 -D__LITTLE_ENDIAN=1234 -D__ISO_C_VISIBLE=1999 -D__GNU_VISIBLE
-CFLAGS += -DCKB_MALLOC_DECLARATION_ONLY -DCKB_PRINTF_DECLARATION_ONLY -DCONFIG_BIGNUM -DCONFIG_STACK_CHECK
-# uncomment to dump memory usage
-# CFLAGS += -DMEMORY_USAGE
+CFLAGS_TARGET = --target=riscv64 -march=rv64imc_zba_zbb_zbc_zbs
+CFLAGS_OPTIMIZE = -g -Oz -fdata-sections -ffunction-sections
+CFLAGS_WARNNING = -Wno-incompatible-library-redeclaration -Wno-invalid-noreturn -Wno-implicit-const-int-float-conversion
+CFLAGS_NO_BUILTIN = -fno-builtin-printf -fno-builtin-memcmp
+CFLAGS_DEFINE = -D__BYTE_ORDER=1234 \
+	-D__LITTLE_ENDIAN=1234 \
+	-D__ISO_C_VISIBLE=1999 \
+	-D__GNU_VISIBLE \
+	-DCKB_MALLOC_DECLARATION_ONLY \
+	-DCKB_PRINTF_DECLARATION_ONLY
+CFLAGS_BASE = $(CFLAGS_TARGET) $(CFLAGS_OPTIMIZE) $(CFLAGS_WARNNING) $(CFLAGS_NO_BUILTIN) $(CFLAGS_DEFINE)
+CFLAGS_BASE_CKB_C_STDLIB = $(CFLAGS_BASE) \
+	-I deps/ckb-c-stdlib/libc \
+	-I deps/ckb-c-stdlib
+CFLAGS_BASE_LIBC = $(CFLAGS_BASE) \
+	-I libc \
+	-I deps/ckb-c-stdlib/libc \
+	-I deps/ckb-c-stdlib \
+	-DCKB_DECLARATION_ONLY
+CFLAGS_BASE_NNCP = $(CFLAGS_BASE) \
+	-I libc \
+	-I deps/ckb-c-stdlib/libc \
+	-I deps/ckb-c-stdlib \
+	-DCKB_DECLARATION_ONLY
+CFLAGS_BASE_SRC = $(CFLAGS_BASE) \
+	-I libc \
+	-I deps/ckb-c-stdlib/libc \
+	-I deps/ckb-c-stdlib \
+	-I deps/nncp \
+	-I deps/quickjs \
+	-DCKB_DECLARATION_ONLY \
+	-DCONFIG_BIGNUM
+CFLAGS_BASE_QUICKJS = $(CFLAGS_BASE) \
+	-I libc \
+	-I deps/ckb-c-stdlib/libc \
+	-I deps/ckb-c-stdlib \
+	-DCKB_DECLARATION_ONLY \
+	-DCONFIG_BIGNUM \
+	-DEMSCRIPTEN \
+	-DCONFIG_STACK_CHECK \
+	-DCONFIG_VERSION=\"2024-01-13-CKB\"
 
 LDFLAGS := -static --gc-sections
 LDFLAGS += -Ldeps/compiler-rt-builtins-riscv/build -lcompiler-rt
 
-OBJDIR=build
+all: out build/ckb-js-vm
 
-QJS_OBJS=$(OBJDIR)/qjs.o $(OBJDIR)/quickjs.o $(OBJDIR)/libregexp.o $(OBJDIR)/libunicode.o \
-		$(OBJDIR)/cutils.o $(OBJDIR)/mocked.o $(OBJDIR)/std_module.o $(OBJDIR)/ckb_module.o $(OBJDIR)/ckb_cell_fs.o \
-		$(OBJDIR)/libbf.o $(OBJDIR)/cmdopt.o
-
-STD_OBJS=$(OBJDIR)/string_impl.o $(OBJDIR)/malloc_impl.o $(OBJDIR)/math_impl.o \
-		$(OBJDIR)/math_log_impl.o $(OBJDIR)/math_pow_impl.o $(OBJDIR)/printf_impl.o $(OBJDIR)/stdio_impl.o \
-		$(OBJDIR)/locale_impl.o
-
-
-all: build/ckb-js-vm
+out:
+	mkdir -p build
+	mkdir -p build/bytecode
+	mkdir -p build/ckb-c-stdlib
+	mkdir -p build/libc
+	mkdir -p build/nncp
+	mkdir -p build/src
+	mkdir -p build/quickjs
 
 deps/compiler-rt-builtins-riscv/build/libcompiler-rt.a:
 	cd deps/compiler-rt-builtins-riscv && make
 
-build/ckb-js-vm: $(STD_OBJS) $(QJS_OBJS) $(OBJDIR)/impl.o deps/compiler-rt-builtins-riscv/build/libcompiler-rt.a
+build/ckb-js-vm: build/ckb-c-stdlib/impl.o \
+                 build/libc/ckb_cell_fs.o \
+                 build/libc/ctype.o \
+                 build/libc/fenv.o \
+                 build/libc/locale.o \
+                 build/libc/malloc.o \
+                 build/libc/math.o \
+                 build/libc/math_log.o \
+                 build/libc/math_pow.o \
+                 build/libc/printf.o \
+                 build/libc/stdio.o \
+                 build/libc/stdlib.o \
+                 build/libc/string.o \
+                 build/libc/sys_time.o \
+                 build/libc/time.o \
+                 build/nncp/cmdopt.o \
+                 build/quickjs/quickjs.o \
+                 build/quickjs/libregexp.o \
+                 build/quickjs/libunicode.o \
+                 build/quickjs/cutils.o \
+                 build/quickjs/libbf.o \
+                 build/quickjs/repl.o \
+                 build/quickjs/qjscalc.o \
+                 build/src/ckb_module.o \
+                 build/src/qjs.o \
+                 build/src/std_module.o \
+                 deps/compiler-rt-builtins-riscv/build/libcompiler-rt.a
 	$(LD) $(LDFLAGS) -o $@ $^
 	cp $@ [email protected]
 	$(OBJCOPY) --strip-debug --strip-all $@
 	ls -lh build/ckb-js-vm
 
-$(OBJDIR)/%.o: quickjs/%.c
+build/ckb-c-stdlib/%.o: deps/ckb-c-stdlib/libc/src/%.c
+	@echo build $<
+	@$(CC) $(CFLAGS_BASE_CKB_C_STDLIB) -c -o $@ $<
+
+build/libc/%.o: libc/src/%.c
 	@echo build $<
-	@$(CC) $(CFLAGS) -c -o $@ $<
+	@$(CC) $(CFLAGS_BASE_LIBC) -c -o $@ $<
 
-$(OBJDIR)/%.o: include/c-stdlib/src/%.c
+build/nncp/%.o: deps/nncp/%.c
 	@echo build $<
-	@$(CC) $(CFLAGS) -c -o $@ $<
+	@$(CC) $(CFLAGS_BASE_NNCP) -c -o $@ $<
 
-$(OBJDIR)/%.o: include/%.c
+build/src/%.o: src/%.c
 	@echo build $<
-	@$(CC) $(CFLAGS) -c -o $@ $<
+	@$(CC) $(CFLAGS_BASE_SRC) -c -o $@ $<
 
-$(OBJDIR)/impl.o: deps/ckb-c-stdlib/libc/src/impl.c
+build/quickjs/%.o: deps/quickjs/%.c
 	@echo build $<
-	@$(CC) $(filter-out -DCKB_DECLARATION_ONLY, $(CFLAGS)) -c -o $@ $<
+	@$(CC) $(CFLAGS_BASE_QUICKJS) -c -o $@ $<
 
 test:
 	make -f tests/examples/Makefile
@@ -85,22 +133,18 @@ benchmark:
 	make -f tests/benchmark/Makefile
 
 clean:
-	rm -f build/*.o
-	rm -f build/ckb-js-vm
-	rm -f build/ckb-js-vm.debug
+	rm -rf build
 	cd tests/ckb_js_tests && make clean
 	make -C deps/compiler-rt-builtins-riscv clean
 
 STYLE := "{BasedOnStyle: Google, TabWidth: 4, IndentWidth: 4, UseTab: Never, SortIncludes: false, ColumnLimit: 120}"
 fmt:
 	clang-format-18 -i -style=$(STYLE) \
-		quickjs/ckb_module.c \
-		quickjs/ckb_module.h \
-		quickjs/mocked.c \
-		quickjs/mocked.h \
-		quickjs/qjs.c \
-		quickjs/std_module.c \
-		quickjs/std_module.h
+		libc/*.h \
+		libc/internal/*.h \
+		libc/src/*.c \
+		libc/sys/*.h \
+		src/*
 
 install:
 	wget 'https://github.com/nervosnetwork/ckb-standalone-debugger/releases/download/v0.119.0/ckb-debugger-linux-x64.tar.gz'

deps/nncp/Changelog

@@ -0,0 +1,37 @@
+2024-06-05:
+
+- Windows fixes
+- single libnc_cuda library for cuda 11 and 12
+
+2023-10-21:
+
+- use a larger transformer model
+
+2021-06-01:
+
+- more hyperparameter tuning
+- preprocessor fixes
+
+2021-04-24:
+
+- use new LibNC library with CUDA and BF16 support
+- merged Transformer, LSTM and preprocessor into nncp
+- Transformer: added GEGLU activation
+- use integer-only operations in the preprocessor
+
+2019-11-16:
+
+- nncp: added the "-seg_len" option to specify the interval in symbols
+  between each parameter update
+- nncp: use rms_norm instead of layer_norm
+- trfcp: improved the positional encoding
+- trfcp: use GeLU instead of ReLU
+
+2019-06-29:
+
+- improved arithmetic coder
+
+2019-05-08:
+
+- nncp: added -n_embed_out parameter
+- preprocess: suppressed "/tmp/word1.txt" file

deps/nncp/Makefile

@@ -0,0 +1,39 @@
+#
+# Makefile for nncp
+# Copyright (c) 2018-2021 Fabrice Bellard
+#
+#CONFIG_WIN32=y
+
+ifdef CONFIG_WIN32
+CROSS_PREFIX=x86_64-w64-mingw32-
+EXE=.exe
+LIBEXT=.lib
+DLLEXT=.dll
+else
+LIBEXT=.a
+DLLEXT=.so
+endif
+
+HOST_CC=gcc
+CC=$(CROSS_PREFIX)gcc
+CXX=$(CROSS_PREFIX)g++
+AR=$(CROSS_PREFIX)ar
+CFLAGS_VERSION:=-DCONFIG_VERSION=\"$(shell cat VERSION)\"
+CFLAGS=-O3 -Wall -Wpointer-arith -g -fno-math-errno -fno-trapping-math -MMD -Wno-format-truncation $(CFLAGS_VERSION) -DLIBNC_CONFIG_FULL
+LDFLAGS=-Wl,-rpath='$$ORIGIN/'
+PROGS=nncp$(EXE)
+LIBS+=-lm -lpthread
+
+all: $(PROGS)
+
+clean:
+	rm -f *.o *.d $(PROGS)
+
+nncp$(EXE): nncp.o cmdopt.o cp_utils.o arith.o preprocess.o cutils.o \
+        libnc$(DLLEXT)
+	$(CC) $(LDFLAGS) -o $@ $^ -lz $(LIBS)
+
+%.o: %.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+
+-include $(wildcard *.d)

deps/nncp/VERSION

@@ -0,0 +1 @@
+2024-06-05