Initial commit

.github/workflows/cd.yml

@@ -0,0 +1,91 @@
+name: Build and upload releases
+
+on:
+  # For pushes to main, build binaries and store them as artifacts (also upload Docker images)
+  # For pushes to main with tags, also make a GitHub release.
+  push:
+    branches:
+      - main
+    tags:
+      - "v*"
+
+jobs:
+  build_binary:
+    name: Build the binaries
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        build: [linux-x86_64, macos-arm64]
+        include:
+          - build: linux-x86_64
+            os: ubuntu-latest
+            target: x86_64-unknown-linux-gnu
+          - build: macos-arm64
+            os: macos-14
+            target: arm64-apple-darwin
+
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v4
+
+      - run: |
+          rustup toolchain install nightly --profile minimal
+          rustup default nightly
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: "true"
+
+      - name: Build the release binary
+        shell: bash
+        run: |
+          cargo build --release
+
+      - name: Test invoking the binary
+        shell: bash
+        run: |
+          ./target/release/lakehouse-loader --help || exit 1
+
+      - name: Prepare artifact name
+        shell: bash
+        run: |
+          echo "ARTIFACT=lakehouse-loader-nightly-${{ matrix.target }}" >> $GITHUB_ENV
+          echo "SOURCE=target/release/lakehouse-loader" >> $GITHUB_ENV
+
+      - name: Upload binaries as artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.ARTIFACT }}
+          path: ${{ env.SOURCE }}
+
+  github_release:
+    name: Perform GitHub release
+    needs: build_binary
+    runs-on: ubuntu-latest
+    if: startsWith(github.event.ref, 'refs/tags/v')
+    steps:
+      - name: Get the release version from the tag
+        shell: bash
+        if: env.RELEASE_VERSION == ''
+        run: |
+          echo "RELEASE_VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
+          echo "version is: ${{ env.RELEASE_VERSION }}"
+
+      - name: Get artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: artifacts
+
+      - name: Package artifacts
+        run: |
+          chmod +x artifacts/lakehouse-loader-nightly-x86_64-unknown-linux-gnu/lakehouse-loader
+          tar -C artifacts/lakehouse-loader-nightly-x86_64-unknown-linux-gnu -czf lakehouse-loader-${{ env.RELEASE_VERSION }}-x86_64-unknown-linux-gnu.tar.gz lakehouse-loader
+          tar -C artifacts/lakehouse-loader-nightly-arm64-apple-darwin -czf lakehouse-loader-${{ env.RELEASE_VERSION }}-arm64-apple-darwin.tar.gz lakehouse-loader
+
+      - name: Upload release archive
+        uses: softprops/action-gh-release@v1
+        with:
+          files: |
+            lakehouse-loader-${{ env.RELEASE_VERSION }}-x86_64-unknown-linux-gnu.tar.gz
+            lakehouse-loader-${{ env.RELEASE_VERSION }}-arm64-apple-darwin.tar.gz

.github/workflows/ci.yml

@@ -0,0 +1,54 @@
+name: CI
+
+on:
+  push:
+
+jobs:
+  CI:
+    name: Lint, build, test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install pre-commit
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y pre-commit
+
+      - name: Configure pre-commit cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pre-commit
+          key: pre-commit-${{ runner.os }}-pre-commit-${{ hashFiles('**/.pre-commit-config.yaml') }}
+
+      - name: Install minimal nightly with clippy and rustfmt
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: nightly
+          components: rustfmt, clippy
+
+      - name: Install tool for formatting Cargo.toml files
+        run: cargo install cargo-tomlfmt
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: "true"
+
+      - name: Check pre-commit hooks (formatting and Clippy)
+        run: pre-commit run --all
+
+      - name: Build in debug mode
+        run: |
+          cargo build
+
+      - name: Spin up the test object store and postgres server
+        run: docker compose up -d --wait || true
+
+      - name: Run tests
+        run: |
+          cargo test
+        env:
+          AWS_ALLOW_HTTP: "true"
+          AWS_ENDPOINT: http://localhost:9000
+          AWS_ACCESS_KEY_ID: minioadmin
+          AWS_SECRET_ACCESS_KEY: minioadmin

.gitignore

@@ -0,0 +1 @@
+/target

.pre-commit-config.yaml

@@ -0,0 +1,29 @@
+repos:
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v2.7.1
+    hooks:
+      - id: prettier
+        files: \.(json|markdown|md|yaml|yml)$
+        exclude: tests/data
+  - repo: local
+    hooks:
+      - id: fmt
+        name: fmt
+        entry: cargo fmt
+        language: system
+        types: [rust]
+        args: ["--"]
+
+      - id: tomlfmt
+        name: tomlfmt
+        entry: find . -name 'Cargo.toml' -exec cargo tomlfmt -p {} \;
+        language: system
+        pass_filenames: false
+
+      # Run Clippy (lint), which also runs cargo check (check the code compiles)
+      - id: clippy
+        name: clippy
+        entry: ./ci/clippy.sh
+        language: system
+        types: [rust]
+        pass_filenames: false