From 84c1790d9b4ee1705bdbee5908107ae6d6f25414 Mon Sep 17 00:00:00 2001 From: AlexMikhalev Date: Wed, 31 Jan 2024 15:54:22 +0000 Subject: [PATCH 1/2] Embed default config Signed-off-by: AlexMikhalev --- .github/workflows/ci.yml | 31 +----------- .../{publish.yml => publish-tauri.yml} | 0 .github/workflows/test-on-pr-tauri.yml | 31 ++++++++++++ .github/workflows/test-on-pr.yml | 47 ++++++++----------- Earthfile | 39 +++++++++++---- crates/settings/default/settings.toml | 19 +------- crates/settings/default/settings_full.toml | 29 ++++++++++++ crates/settings/src/lib.rs | 9 ++-- default/settings.toml | 14 ++++++ default/settings_default_desktop.toml | 29 ++++++++++++ desktop/Earthfile | 6 +-- terraphim_server/default/settings.toml | 19 +------- .../default/settings_default_server.toml | 29 ++++++++++++ terraphim_server/src/main.rs | 4 +- 14 files changed, 200 insertions(+), 106 deletions(-) rename .github/workflows/{publish.yml => publish-tauri.yml} (100%) create mode 100644 .github/workflows/test-on-pr-tauri.yml create mode 100644 crates/settings/default/settings_full.toml create mode 100644 default/settings.toml create mode 100644 default/settings_default_desktop.toml create mode 100644 terraphim_server/default/settings_default_server.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7de31613..cda59804 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ env: CARGO_TERM_COLOR: always jobs: - build: + build-and-test: runs-on: ubuntu-latest env: EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} @@ -28,31 +28,4 @@ jobs: - name: Docker Login run: docker login --username "$DOCKERHUB_USERNAME" --password "$DOCKERHUB_TOKEN" - name: Run build - run: earthly --org applied-knowledge-systems --sat my-satellite --ci --push +docker-all - - build-and-test: - name: Build and Test - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Install system dependencies - run: sudo apt-get update && sudo apt-get install -y libgtk-3-dev libsoup2.4-dev libwebkit2gtk-4.0-dev - - - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt, clippy - - - uses: Swatinem/rust-cache@v2 - - - name: cargo check - run: cargo check --workspace - - - name: cargo fmt - run: cargo fmt --all -- --check - - - name: cargo clippy - run: cargo clippy --workspace --all-targets --all-features -- -D warnings - - - name: cargo test - run: cargo test --workspace --exclude terraphim_ai --exclude app --all-targets --all-features + run: earthly --org applied-knowledge-systems --sat my-satellite --ci --push +pipeline \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish-tauri.yml similarity index 100% rename from .github/workflows/publish.yml rename to .github/workflows/publish-tauri.yml diff --git a/.github/workflows/test-on-pr-tauri.yml b/.github/workflows/test-on-pr-tauri.yml new file mode 100644 index 00000000..399490f7 --- /dev/null +++ b/.github/workflows/test-on-pr-tauri.yml @@ -0,0 +1,31 @@ +name: "test-on-pr" +on: [pull_request] + +jobs: + test-tauri: + strategy: + fail-fast: false + matrix: + platform: [macos-latest, ubuntu-latest, windows-latest] + + runs-on: ${{ matrix.platform }} + steps: + - uses: actions/checkout@v2 + - name: setup node + uses: actions/setup-node@v1 + with: + node-version: 16 + - name: install Rust stable + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + - name: install dependencies (ubuntu only) + if: matrix.platform == 'ubuntu-latest' + run: | + sudo apt-get update + sudo apt-get install -y libgtk-3-dev webkit2gtk-4.0 libappindicator3-dev librsvg2-dev patchelf + - name: install app dependencies and build it + run: yarn && yarn build + - uses: tauri-apps/tauri-action@v0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test-on-pr.yml b/.github/workflows/test-on-pr.yml index 399490f7..624e7d84 100644 --- a/.github/workflows/test-on-pr.yml +++ b/.github/workflows/test-on-pr.yml @@ -1,31 +1,24 @@ name: "test-on-pr" -on: [pull_request] +on: + [pull_request] -jobs: - test-tauri: - strategy: - fail-fast: false - matrix: - platform: [macos-latest, ubuntu-latest, windows-latest] +env: + CARGO_TERM_COLOR: always - runs-on: ${{ matrix.platform }} +jobs: + build-and-test: + runs-on: ubuntu-latest + env: + EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + FORCE_COLOR: 1 steps: - - uses: actions/checkout@v2 - - name: setup node - uses: actions/setup-node@v1 - with: - node-version: 16 - - name: install Rust stable - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - name: install dependencies (ubuntu only) - if: matrix.platform == 'ubuntu-latest' - run: | - sudo apt-get update - sudo apt-get install -y libgtk-3-dev webkit2gtk-4.0 libappindicator3-dev librsvg2-dev patchelf - - name: install app dependencies and build it - run: yarn && yarn build - - uses: tauri-apps/tauri-action@v0 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: earthly/actions-setup@v1 + with: + version: v0.7.23 + - uses: actions/checkout@v2 + - name: Docker Login + run: docker login --username "$DOCKERHUB_USERNAME" --password "$DOCKERHUB_TOKEN" + - name: Run build + run: earthly --org applied-knowledge-systems --sat my-satellite --ci --push +pipeline \ No newline at end of file diff --git a/Earthfile b/Earthfile index b72ff7e9..9fdaae25 100644 --- a/Earthfile +++ b/Earthfile @@ -19,6 +19,7 @@ WORKDIR /code pipeline: BUILD desktop+build + BUILD +build-debug BUILD +fmt BUILD +lint BUILD +test @@ -46,20 +47,23 @@ install: RUN update-ca-certificates RUN rustup component add clippy RUN rustup component add rustfmt - RUN cargo install cross DO rust+INIT --keep_fingerprints=true + RUN cargo install cross + RUN cargo install orogene + source: FROM +install + WORKDIR /code COPY --keep-ts Cargo.toml Cargo.lock ./ - COPY --keep-ts --dir terraphim-server desktop crates terraphim_types ./ - COPY desktop+build/dist /code/terraphim-server/dist + COPY --keep-ts --dir terraphim_server desktop crates terraphim_types ./ DO rust+CARGO --args=fetch cross-build: FROM +source ARG --required TARGET DO rust+SET_CACHE_MOUNTS_ENV + COPY desktop+build/dist /code/terraphim-server/dist WITH DOCKER RUN --mount=$EARTHLY_RUST_CARGO_HOME_CACHE --mount=$EARTHLY_RUST_TARGET_CACHE cross build --target $TARGET --release END @@ -69,20 +73,32 @@ cross-build: build: FROM +source + DO rust+SET_CACHE_MOUNTS_ENV + COPY desktop+build/dist /code/terraphim-server/dist DO rust+CARGO --args="build --offline --release" --output="release/[^/\.]+" RUN ./target/release/terraphim_server --version SAVE ARTIFACT ./target/release/terraphim_server AS LOCAL artifact/bin/terraphim_server-$TARGET -test: +build-debug: FROM +source - DO rust+CARGO --args="test" + DO rust+SET_CACHE_MOUNTS_ENV + DO rust+CARGO --args="build --offline" --output="debug/[^/\.]+" + RUN ./target/debug/terraphim_server --version + SAVE ARTIFACT ./target/debug/terraphim_server AS LOCAL artifact/bin/terraphim_server-debug + +test: + FROM +build-debug + DO rust+SET_CACHE_MOUNTS_ENV + COPY --chmod=0755 +build-debug/terraphim_server /terraphim_server_debug + RUN --mount=$EARTHLY_RUST_CARGO_HOME_CACHE --mount=$EARTHLY_RUST_TARGET_CACHE nohup /terraphim_server_debug & sleep 2 & cargo test --offline; + #DO rust+CARGO --args="test --offline" fmt: - FROM +source + FROM +build-debug DO rust+CARGO --args="fmt --check" lint: - FROM +source + FROM +build-debug DO rust+CARGO --args="clippy --no-deps --all-features --all-targets" docker-musl: @@ -121,4 +137,11 @@ docker-slim: COPY +build/terraphim_server terraphim_server EXPOSE 8000 ENTRYPOINT ["./terraphim_server"] - SAVE IMAGE aks/terraphim_server:buster \ No newline at end of file + SAVE IMAGE aks/terraphim_server:buster + +docker-sratch: + FROM scratch + COPY +build/terraphim_server terraphim_server + EXPOSE 8000 + ENTRYPOINT ["./terraphim_server"] + SAVE IMAGE aks/terraphim_server:scratch \ No newline at end of file diff --git a/crates/settings/default/settings.toml b/crates/settings/default/settings.toml index 0553ba69..d569903b 100644 --- a/crates/settings/default/settings.toml +++ b/crates/settings/default/settings.toml @@ -1,17 +1,9 @@ server_hostname = "127.0.0.1:8000" api_endpoint="http://localhost:8000/api" -[profiles.s3] -type = "s3" -bucket = "test" -region = "${TERRAPHIM_PROFILE_S3_REGION:-us-east-1}" -endpoint = "${TERRAPHIM_PROFILE_S3_ENDPOINT:-http://rpi4node3:8333/}" -access_key_id = "${AWS_ACCESS_KEY_ID}" -secret_access_key = "${AWS_SECRET_ACCESS_KEY}" - [profiles.sled] type = "sled" -datadir= "/tmp/opendal/sled" +datadir= "/tmp/sled" [profiles.dash] type = "dashmap" @@ -19,11 +11,4 @@ root = "/tmp/dashmaptest" [profiles.rock] type = "rocksdb" -datadir = "/tmp/opendal/rocksdb" - -[profiles.atomicserver] -endpoint = "${TERRAPHIM_PROFILE_ATOMICSERVER}" -type = "atomicserver" -private_key = "${TERRAPHIM_PROFILE_ATOMICSERVER_PRIVATE_KEY}" -public_key = "${TERRAPHIM_PROFILE_ATOMICSERVER_PUBLIC_KEY}" -parent_resource_id="${TERRAPHIM_PROFILE_ATOMICSERVER}" \ No newline at end of file +datadir = "/tmp/rocksdb" \ No newline at end of file diff --git a/crates/settings/default/settings_full.toml b/crates/settings/default/settings_full.toml new file mode 100644 index 00000000..0553ba69 --- /dev/null +++ b/crates/settings/default/settings_full.toml @@ -0,0 +1,29 @@ +server_hostname = "127.0.0.1:8000" +api_endpoint="http://localhost:8000/api" + +[profiles.s3] +type = "s3" +bucket = "test" +region = "${TERRAPHIM_PROFILE_S3_REGION:-us-east-1}" +endpoint = "${TERRAPHIM_PROFILE_S3_ENDPOINT:-http://rpi4node3:8333/}" +access_key_id = "${AWS_ACCESS_KEY_ID}" +secret_access_key = "${AWS_SECRET_ACCESS_KEY}" + +[profiles.sled] +type = "sled" +datadir= "/tmp/opendal/sled" + +[profiles.dash] +type = "dashmap" +root = "/tmp/dashmaptest" + +[profiles.rock] +type = "rocksdb" +datadir = "/tmp/opendal/rocksdb" + +[profiles.atomicserver] +endpoint = "${TERRAPHIM_PROFILE_ATOMICSERVER}" +type = "atomicserver" +private_key = "${TERRAPHIM_PROFILE_ATOMICSERVER_PRIVATE_KEY}" +public_key = "${TERRAPHIM_PROFILE_ATOMICSERVER_PUBLIC_KEY}" +parent_resource_id="${TERRAPHIM_PROFILE_ATOMICSERVER}" \ No newline at end of file diff --git a/crates/settings/src/lib.rs b/crates/settings/src/lib.rs index b230d46c..47fc6488 100644 --- a/crates/settings/src/lib.rs +++ b/crates/settings/src/lib.rs @@ -41,7 +41,7 @@ impl Settings { } } }; - + println!("Reading config_file: {:?}", config_file); Ok(Settings::with_layers(&[ Layer::Toml(config_file), Layer::Env(Some(String::from("TERRAPHIM_"))), @@ -54,13 +54,16 @@ fn create_config_folder(path: &PathBuf) -> Result { std::fs::create_dir_all(path)?; } let filename = path.join("settings.toml"); - + let default_config = include_str!("../default/settings.toml"); if filename.exists() { log::warn!("File exists"); log::warn!("{:?}", filename); } else { log::warn!("File does not exist"); - std::fs::copy("default/settings.toml", &filename)?; + // FIXME: the logic of settings that each app - desktop or server have their own default settings.toml + // this is a quickfix to make testing sane + println!("Writing default config to: {:?}", filename); + std::fs::write( &filename,default_config)?; } Ok(filename) } diff --git a/default/settings.toml b/default/settings.toml new file mode 100644 index 00000000..8e577a43 --- /dev/null +++ b/default/settings.toml @@ -0,0 +1,14 @@ +server_hostname = "${TERRAPHIM_SERVER_HOSTNAME}-127.0.0.1:8000" +api_endpoint="${TERRAPHIM_SERVER_API_ENDPOINT}-http://localhost:8000/api" + +[profiles.sled] +type = "sled" +datadir= "/tmp/sled" + +[profiles.dash] +type = "dashmap" +root = "/tmp/dashmaptest" + +[profiles.rock] +type = "rocksdb" +datadir = "/tmp/rocksdb" \ No newline at end of file diff --git a/default/settings_default_desktop.toml b/default/settings_default_desktop.toml new file mode 100644 index 00000000..47b2a4b8 --- /dev/null +++ b/default/settings_default_desktop.toml @@ -0,0 +1,29 @@ +server_hostname = "${TERRAPHIM_SERVER_HOSTNAME}-127.0.0.1:8000" +api_endpoint="${TERRAPHIM_SERVER_API_ENDPOINT}-http://localhost:8000/api" + +[profiles.s3] +type = "s3" +bucket = "test" +region = "${TERRAPHIM_PROFILE_S3_REGION:-us-east-1}" +endpoint = "${TERRAPHIM_PROFILE_S3_ENDPOINT:-http://rpi4node3:8333/}" +access_key_id = "${AWS_ACCESS_KEY_ID}" +secret_access_key = "${AWS_SECRET_ACCESS_KEY}" + +[profiles.sled] +type = "sled" +datadir= "/tmp/opendal/sled" + +[profiles.dash] +type = "dashmap" +root = "/tmp/dashmaptest" + +[profiles.rock] +type = "rocksdb" +datadir = "/tmp/opendal/rocksdb" + +[profiles.atomicserver] +endpoint = "${TERRAPHIM_PROFILE_ATOMICSERVER}" +type = "atomicserver" +private_key = "${TERRAPHIM_PROFILE_ATOMICSERVER_PRIVATE_KEY}" +public_key = "${TERRAPHIM_PROFILE_ATOMICSERVER_PUBLIC_KEY}" +parent_resource_id="${TERRAPHIM_PROFILE_ATOMICSERVER}" \ No newline at end of file diff --git a/desktop/Earthfile b/desktop/Earthfile index cfb7aae9..0d398ac7 100644 --- a/desktop/Earthfile +++ b/desktop/Earthfile @@ -5,9 +5,9 @@ WORKDIR frontend deps: # COPY package.json tsconfig.json yarn.lock vite.config.ts tsconfig.node.json index.html ./ - COPY . . - COPY --dir src src - COPY --dir public public + COPY --keep-ts . . + # COPY --dir src src + # COPY --dir public public RUN yarn build: diff --git a/terraphim_server/default/settings.toml b/terraphim_server/default/settings.toml index 47b2a4b8..8e577a43 100644 --- a/terraphim_server/default/settings.toml +++ b/terraphim_server/default/settings.toml @@ -1,17 +1,9 @@ server_hostname = "${TERRAPHIM_SERVER_HOSTNAME}-127.0.0.1:8000" api_endpoint="${TERRAPHIM_SERVER_API_ENDPOINT}-http://localhost:8000/api" -[profiles.s3] -type = "s3" -bucket = "test" -region = "${TERRAPHIM_PROFILE_S3_REGION:-us-east-1}" -endpoint = "${TERRAPHIM_PROFILE_S3_ENDPOINT:-http://rpi4node3:8333/}" -access_key_id = "${AWS_ACCESS_KEY_ID}" -secret_access_key = "${AWS_SECRET_ACCESS_KEY}" - [profiles.sled] type = "sled" -datadir= "/tmp/opendal/sled" +datadir= "/tmp/sled" [profiles.dash] type = "dashmap" @@ -19,11 +11,4 @@ root = "/tmp/dashmaptest" [profiles.rock] type = "rocksdb" -datadir = "/tmp/opendal/rocksdb" - -[profiles.atomicserver] -endpoint = "${TERRAPHIM_PROFILE_ATOMICSERVER}" -type = "atomicserver" -private_key = "${TERRAPHIM_PROFILE_ATOMICSERVER_PRIVATE_KEY}" -public_key = "${TERRAPHIM_PROFILE_ATOMICSERVER_PUBLIC_KEY}" -parent_resource_id="${TERRAPHIM_PROFILE_ATOMICSERVER}" \ No newline at end of file +datadir = "/tmp/rocksdb" \ No newline at end of file diff --git a/terraphim_server/default/settings_default_server.toml b/terraphim_server/default/settings_default_server.toml new file mode 100644 index 00000000..47b2a4b8 --- /dev/null +++ b/terraphim_server/default/settings_default_server.toml @@ -0,0 +1,29 @@ +server_hostname = "${TERRAPHIM_SERVER_HOSTNAME}-127.0.0.1:8000" +api_endpoint="${TERRAPHIM_SERVER_API_ENDPOINT}-http://localhost:8000/api" + +[profiles.s3] +type = "s3" +bucket = "test" +region = "${TERRAPHIM_PROFILE_S3_REGION:-us-east-1}" +endpoint = "${TERRAPHIM_PROFILE_S3_ENDPOINT:-http://rpi4node3:8333/}" +access_key_id = "${AWS_ACCESS_KEY_ID}" +secret_access_key = "${AWS_SECRET_ACCESS_KEY}" + +[profiles.sled] +type = "sled" +datadir= "/tmp/opendal/sled" + +[profiles.dash] +type = "dashmap" +root = "/tmp/dashmaptest" + +[profiles.rock] +type = "rocksdb" +datadir = "/tmp/opendal/rocksdb" + +[profiles.atomicserver] +endpoint = "${TERRAPHIM_PROFILE_ATOMICSERVER}" +type = "atomicserver" +private_key = "${TERRAPHIM_PROFILE_ATOMICSERVER_PRIVATE_KEY}" +public_key = "${TERRAPHIM_PROFILE_ATOMICSERVER_PUBLIC_KEY}" +parent_resource_id="${TERRAPHIM_PROFILE_ATOMICSERVER}" \ No newline at end of file diff --git a/terraphim_server/src/main.rs b/terraphim_server/src/main.rs index 0265160e..cbf37d94 100644 --- a/terraphim_server/src/main.rs +++ b/terraphim_server/src/main.rs @@ -63,8 +63,8 @@ async fn main() -> Result<()> { let addr = server_hostname; let role = "system operator".to_string(); - // let automata_url = "https://system-operator.s3.eu-west-2.amazonaws.com/term_to_id.json"; - let automata_url = "./data/term_to_id.json"; + let automata_url = "https://system-operator.s3.eu-west-2.amazonaws.com/term_to_id.json"; + // let automata_url = "./data/term_to_id.json"; let rolegraph = RoleGraph::new(role.clone(), automata_url).await?; config_state.roles.insert( role, From e3f8ebd9ac4bea528e3370c4ec10c107ff972634 Mon Sep 17 00:00:00 2001 From: AlexMikhalev Date: Wed, 31 Jan 2024 16:53:46 +0000 Subject: [PATCH 2/2] Correct Earthfile to build earthly Signed-off-by: AlexMikhalev --- Earthfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Earthfile b/Earthfile index 9fdaae25..f984f9b1 100644 --- a/Earthfile +++ b/Earthfile @@ -49,7 +49,7 @@ install: RUN rustup component add rustfmt DO rust+INIT --keep_fingerprints=true RUN cargo install cross - RUN cargo install orogene + #RUN cargo install orogene source: @@ -82,6 +82,7 @@ build: build-debug: FROM +source DO rust+SET_CACHE_MOUNTS_ENV + COPY desktop+build/dist /code/terraphim-server/dist DO rust+CARGO --args="build --offline" --output="debug/[^/\.]+" RUN ./target/debug/terraphim_server --version SAVE ARTIFACT ./target/debug/terraphim_server AS LOCAL artifact/bin/terraphim_server-debug @@ -139,7 +140,7 @@ docker-slim: ENTRYPOINT ["./terraphim_server"] SAVE IMAGE aks/terraphim_server:buster -docker-sratch: +docker-scratch: FROM scratch COPY +build/terraphim_server terraphim_server EXPOSE 8000