Merge pull request #5 from ice-blockchain/ion-sync-upstream

Ion sync upstream

.github/workflows/build-ion-linux-x86-64-shared.yml

@@ -7,7 +7,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, ubuntu-22.04]
+        os: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04]
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -21,7 +21,7 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev
 
-    - name: Install clang-16
+    - if: matrix.os != 'ubuntu-24.04'
       run: |
         wget https://apt.llvm.org/llvm.sh
         chmod +x llvm.sh

.github/workflows/build-ion-wasm-emscripten.yml

@@ -19,7 +19,7 @@ jobs:
 
     - name: Build ION WASM artifacts
       run: |
-        cd assembly/wasm
+        cp assembly/wasm/fift-func-wasm-build-ubuntu.sh .
         chmod +x fift-func-wasm-build-ubuntu.sh
         ./fift-func-wasm-build-ubuntu.sh -a
 

.github/workflows/docker-ubuntu-branch-image.yml

@@ -0,0 +1,48 @@
+name: Docker Ubuntu 22.04 branch image
+
+on:
+  workflow_dispatch:
+  push:
+    branches-ignore:
+      - master
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get tag as branch name
+        id: tag
+        run: |
+          echo "TAG=${GITHUB_REF##*/}" >> $GITHUB_OUTPUT
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v6
+        with:
+          platforms: linux/amd64
+          push: true
+          context: ./
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}

.github/workflows/docker-ubuntu-image.yml

@@ -20,22 +20,49 @@ jobs:
           submodules: 'recursive'
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Build and export to Docker
+        uses: docker/build-push-action@v6
+        with:
+          load: true
+          context: ./
+          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test
+
+      - name: Test
+        run: |
+          docker run --rm -e "TEST=1" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test
+
+      - name: Get next tag
+        id: tag
+        run: |
+          git fetch --all --tags
+          git tag -l
+          NEW_TAG=v$(date +'%Y.%m')
+          FOUND=$(git tag -l | grep $NEW_TAG | wc -l)
+          if [ $FOUND -eq 0 ]; then
+            echo "TAG=$NEW_TAG" >> $GITHUB_OUTPUT
+          else
+            echo "TAG=$NEW_TAG-$FOUND" >> $GITHUB_OUTPUT
+          fi
+
       - name: Build and push
         id: docker_build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
+          platforms: linux/amd64,linux/arm64
           push: true
           context: ./
-          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }}

.github/workflows/ion-x86-64-windows.yml

@@ -9,7 +9,7 @@ defaults:
 jobs:
   build:
 
-    runs-on: windows-2022
+    runs-on: windows-2019
 
     steps:
       - name: Get Current OS version
@@ -23,9 +23,9 @@ jobs:
 
       - name: Build TON
         run: |
-          copy assembly\native\build-windows-github.bat .
-          copy assembly\native\build-windows.bat .
-          build-windows-github.bat Enterprise
+          copy assembly\native\build-windows-github-2019.bat .
+          copy assembly\native\build-windows-2019.bat .
+          build-windows-github-2019.bat Enterprise
 
       - name: Upload artifacts
         uses: actions/upload-artifact@master

CMake/FindSodium.cmake

@@ -37,12 +37,14 @@ if (NOT DEFINED SODIUM_USE_STATIC_LIBS)
   option(SODIUM_USE_STATIC_LIBS "enable to statically link against sodium" OFF)
 endif()
 if(NOT (SODIUM_USE_STATIC_LIBS EQUAL SODIUM_USE_STATIC_LIBS_LAST))
-  unset(sodium_LIBRARY CACHE)
-  unset(SODIUM_LIBRARY_DEBUG CACHE)
-  unset(SODIUM_LIBRARY_RELEASE CACHE)
-  unset(sodium_DLL_DEBUG CACHE)
-  unset(sodium_DLL_RELEASE CACHE)
-  set(SODIUM_USE_STATIC_LIBS_LAST ${SODIUM_USE_STATIC_LIBS} CACHE INTERNAL "internal change tracking variable")
+  if (NOT SODIUM_LIBRARY_RELEASE)
+    unset(sodium_LIBRARY CACHE)
+    unset(SODIUM_LIBRARY_DEBUG CACHE)
+    unset(SODIUM_LIBRARY_RELEASE CACHE)
+    unset(sodium_DLL_DEBUG CACHE)
+    unset(sodium_DLL_RELEASE CACHE)
+    set(SODIUM_USE_STATIC_LIBS_LAST ${SODIUM_USE_STATIC_LIBS} CACHE INTERNAL "internal change tracking variable")
+  endif()
 endif()
 
 
@@ -295,4 +297,4 @@ else()
         )
     endif()
   endif()
-endif()
+endif()

CMakeLists.txt

@@ -539,6 +539,9 @@ target_link_libraries(test-ton-collator overlay tdutils tdactor adnl tl_api dht
 add_executable(test-http test/test-http.cpp)
 target_link_libraries(test-http PRIVATE tonhttp)
 
+add_executable(test-emulator test/test-td-main.cpp emulator/test/emulator-tests.cpp)
+target_link_libraries(test-emulator PRIVATE emulator)
+
 get_directory_property(HAS_PARENT PARENT_DIRECTORY)
 if (HAS_PARENT)
   set(ALL_TEST_SOURCE
@@ -570,6 +573,7 @@ add_test(test-cells test-cells ${TEST_OPTIONS})
 add_test(test-smartcont test-smartcont)
 add_test(test-net test-net)
 add_test(test-actors test-tdactor)
+add_test(test-emulator test-emulator)
 
 #BEGIN tonlib
 add_test(test-tdutils test-tdutils)

Changelog.md

@@ -1,3 +1,20 @@
+## 2024.08 Update
+
+1. Introduction of dispatch queues, message envelopes with transaction chain metadata, and explicitly stored msg_queue size, which will be activated by `Config8.version >= 8` and new `Config8.capabilities` bits: `capStoreOutMsgQueueSize`, `capMsgMetadata`, `capDeferMessages`. 
+2. A number of changes to transcation executor which will activated for `Config8.version >= 8`:
+    - Check mode on invalid `action_send_msg`. Ignore action if `IGNORE_ERROR` (+2) bit is set, bounce if `BOUNCE_ON_FAIL` (+16) bit is set.
+    - Slightly change random seed generation to fix mix of `addr_rewrite` and `addr`.
+    - Fill in `skipped_actions` for both invalid and valid messages with `IGNORE_ERROR` mode that can't be sent.
+    - Allow unfreeze through external messages.
+    - Don't use user-provided `fwd_fee` and `ihr_fee` for internal messages.
+3. A few issues with broadcasts were fixed: stop on receiving last piece, response to AdnlMessageCreateChannel
+4. A number of fixes and improvements for emulator and tonlib: correct work with config_addr, not accepted externals, bounces, debug ops gas consumption, added version and c5 dump, fixed tonlib crashes
+5. Added new flags and commands to the node, in particular `--fast-state-serializer`, `getcollatoroptionsjson`, `setcollatoroptionsjson`
+
+Besides the work of the core team, this update is based on the efforts of @krigga (emulator), stonfi team, in particular @dbaranovstonfi and @hey-researcher (emulator), and  @loeul, @xiaoxianBoy, @simlecode (typos in comments and docs).
+
+
+
 ## 2024.06 Update
 
 1. Make Jemalloc default allocator

Dockerfile

@@ -1,10 +1,15 @@
-FROM ubuntu:22.04 as builder
+FROM ubuntu:22.04 AS builder
 RUN apt-get update && \
-	DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git ninja-build libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev pkg-config autoconf automake libtool libjemalloc-dev && \
-	rm -rf /var/lib/apt/lists/*
-ENV CC clang
-ENV CXX clang++
-ENV CCACHE_DISABLE 1
+        DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git ninja-build libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev pkg-config autoconf automake libtool libjemalloc-dev lsb-release software-properties-common gnupg
+
+RUN wget https://apt.llvm.org/llvm.sh && \
+    chmod +x llvm.sh && \
+    ./llvm.sh 16 all && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV CC=/usr/bin/clang-16
+ENV CXX=/usr/bin/clang++-16
+ENV CCACHE_DISABLE=1
 
 WORKDIR /
 RUN mkdir ion
@@ -13,27 +18,30 @@ WORKDIR /ion
 COPY ./ ./
 
 RUN mkdir build && \
-	cd build && \
-	cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DTON_USE_JEMALLOC=ON .. && \
-	ninja storage-daemon storage-daemon-cli tonlibjson fift func validator-engine validator-engine-console generate-random-id dht-server lite-client
+        cd build && \
+        cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DTON_USE_JEMALLOC=ON .. && \
+        ninja storage-daemon storage-daemon-cli tonlibjson fift func validator-engine validator-engine-console generate-random-id dht-server lite-client
 
 FROM ubuntu:22.04
 RUN apt-get update && \
-    apt-get install -y wget libatomic1 openssl libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev libjemalloc-dev && \
+    apt-get install -y wget curl libatomic1 openssl libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev libjemalloc-dev htop net-tools netcat iptraf-ng jq tcpdump pv plzip && \
     rm -rf /var/lib/apt/lists/*
 
-RUN mkdir -p /var/ion-work/db && \
-    mkdir -p /var/ion-work/db/static
+RUN mkdir -p /var/ion-work/db /var/ion-work/scripts /usr/share/ion/smartcont/ /usr/lib/fift/
 
 COPY --from=builder /ion/build/storage/storage-daemon/storage-daemon /usr/local/bin/
 COPY --from=builder /ion/build/storage/storage-daemon/storage-daemon-cli /usr/local/bin/
 COPY --from=builder /ion/build/lite-client/lite-client /usr/local/bin/
 COPY --from=builder /ion/build/validator-engine/validator-engine /usr/local/bin/
 COPY --from=builder /ion/build/validator-engine-console/validator-engine-console /usr/local/bin/
 COPY --from=builder /ion/build/utils/generate-random-id /usr/local/bin/
+COPY --from=builder /ion/build/crypto/fift /usr/local/bin/
+COPY --from=builder /ion/build/crypto/func /usr/local/bin/
+COPY --from=builder /ion/crypto/smartcont/* /usr/share/ion/smartcont/
+COPY --from=builder /ion/crypto/fift/lib/* /usr/lib/fift/
 
 WORKDIR /var/ion-work/db
-COPY ./docker/init.sh ./docker/control.template ./
-RUN chmod +x init.sh
+COPY ./docker/init.sh ./docker/control.template /var/ion-work/scripts/
+RUN chmod +x /var/ion-work/scripts/init.sh
 
-ENTRYPOINT ["/var/ion-work/db/init.sh"]
+ENTRYPOINT ["/var/ion-work/scripts/init.sh"]

adnl/adnl-channel.cpp

@@ -112,16 +112,16 @@ void AdnlChannelImpl::send_message(td::uint32 priority, td::actor::ActorId<AdnlN
 }
 
 void AdnlChannelImpl::receive(td::IPAddress addr, td::BufferSlice data) {
-  auto P = td::PromiseCreator::lambda(
-      [peer = peer_pair_, channel_id = channel_in_id_, addr, id = print_id()](td::Result<AdnlPacket> R) {
-        if (R.is_error()) {
-          VLOG(ADNL_WARNING) << id << ": dropping IN message: can not decrypt: " << R.move_as_error();
-        } else {
-          auto packet = R.move_as_ok();
-          packet.set_remote_addr(addr);
-          td::actor::send_closure(peer, &AdnlPeerPair::receive_packet_from_channel, channel_id, std::move(packet));
-        }
-      });
+  auto P = td::PromiseCreator::lambda([peer = peer_pair_, channel_id = channel_in_id_, addr, id = print_id(),
+                                       size = data.size()](td::Result<AdnlPacket> R) {
+    if (R.is_error()) {
+      VLOG(ADNL_WARNING) << id << ": dropping IN message: can not decrypt: " << R.move_as_error();
+    } else {
+      auto packet = R.move_as_ok();
+      packet.set_remote_addr(addr);
+      td::actor::send_closure(peer, &AdnlPeerPair::receive_packet_from_channel, channel_id, std::move(packet), size);
+    }
+  });
 
   decrypt(std::move(data), std::move(P));
 }

adnl/adnl-ext-server.cpp

@@ -91,7 +91,7 @@ td::Status AdnlInboundConnection::process_custom_packet(td::BufferSlice &data, b
     auto F = fetch_tl_object<ton_api::tcp_authentificate>(data.clone(), true);
     if (F.is_ok()) {
       if (nonce_.size() > 0 || !remote_id_.is_zero()) {
-        return td::Status::Error(ErrorCode::protoviolation, "duplicate authentificate");
+        return td::Status::Error(ErrorCode::protoviolation, "duplicate authenticate");
       }
       auto f = F.move_as_ok();
       nonce_ = td::SecureString{f->nonce_.size() + 256};