add buck2 RE rust example (#361)

- Adding Buck2 RE rust example
- Adding example to CI run
- Fixing a couple of typos in other READMEs
-  Fix minor issue with go toolchain with latest version of Buck2

buck2/rust/.buckconfig

@@ -0,0 +1,32 @@
+[cells]
+root = .
+prelude = prelude
+toolchains = toolchains
+none = none
+
+[cell_aliases]
+config = prelude
+fbcode = none
+fbsource = none
+buck = none
+
+[external_cells]
+prelude = bundled
+
+[parser]
+target_platform_detector_spec = target:root//...->root//platforms:remote_platform
+
+[buck2]
+digest_algorithms = SHA256
+
+[buck2_re_client]
+engine_address       = <CLUSTER_NAME>.cluster.engflow.com
+action_cache_address = <CLUSTER_NAME>.cluster.engflow.com
+cas_address          = <CLUSTER_NAME>.cluster.engflow.com
+http_headers         = <AUTH_HTTP_HEADERS>
+
+[build]
+execution_platforms = root//platforms:remote_platform
+
+[project]
+ignore = .git

buck2/rust/.gitignore

@@ -0,0 +1 @@
+/buck-out

buck2/rust/BUCK

@@ -0,0 +1,42 @@
+# Copyright 2022 EngFlow Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+rust_library(
+    name = "library",
+    srcs = glob(
+        ["src/**/*.rs"],
+    ),
+)
+
+rust_binary(
+    name = "main",
+    srcs = glob(
+        ["bin/**/*.rs"],
+    ),    
+    crate_root = "bin/main.rs",
+    deps = [":library"],
+)
+
+rust_test(
+    name = "test",
+    srcs = glob(
+        ["test/**/*.rs"],
+    ),
+    deps = [":library"],
+    # TODO: remove these once https://github.com/facebook/buck2/pull/826 gets merged.
+    remote_execution_action_key_providers = select({
+        "//platforms:engflow": "//platforms:remote_execution_action_keys",
+        "DEFAULT": None,
+    }),
+)

buck2/rust/README.md

@@ -0,0 +1,60 @@
+# EngFlow RE + Buck2 Rust example
+
+This example demonstrates use of EngFlow RE for a simple Rust project built with [Buck2](https://github.com/facebook/buck2) using the prelude.
+
+It is based on two existing samples in the Buck2 upstream repo:
+
+* Simple rust project with prelude - https://github.com/facebook/buck2/tree/main/examples/with_prelude/rust
+* Buck2 remote execution integration with EngFlow - https://github.com/facebook/buck2/tree/main/examples/remote_execution/engflow
+
+### Example structure
+
+In the `platforms` cell we specify:
+In the `platforms` cell we specify:
+* The platform used for remote execution in this project `root//platforms:remote_platform`, which includes the definition of the Docker image used for remote execution, and that defines constraints for targets to run in the remote execution environment. This platform provides an `ExecutionPlatformRegistrationInfo` a `ConfigurationInfo` and a `PlatformInfo` to be able to be used in the `.buckconfig`.
+* The action keys `root//platforms:remote_execution_action_keys`, which provides a default `BuildModeInfo` that is needed for RE of tests to function properly.
+* The platform `image` configured in `platforms/defs.bzl`, notably, uses a different image than other Buck2 samples in this repo. Specifically, it uses a public AWS image that has `rust` preinstalled. This is because Buck2 rust rules do not include a hermetic rust toolchain. We use a bookworm image with rust pre-installed. Image details can be found in https://gallery.ecr.aws/docker/library/rust. The Dockerfile can be found in https://github.com/rust-lang/docker-rust/blob/700c4f146427808cfb1e07a646e4afabbe99da4f/stable/bullseye/Dockerfile. If a different version of `rust` or other tools is needed you should create your own image and publish it to a repo that is accessible from the cluster.
+
+In the `toolchains` cell we specify:
+
+* The remote rust toolchain `root//toolchains:remote_rust_toolchain` which is compatible with the remote execution environment. This toolchain is configured with the `rustc_target_triple` to match the remote execution environment, and the `compiler`, `clippy_driver` and `rustdoc` attrs are set to point to the location of the `rustc` binary in the image used for remote execution.
+* The c++ toolchain `root//toolchains:cxx_tools_info_toolchain` that is compatible with the remote execution environment.
+* The clang tools, `root//toolchains:path_clang_tools`, which is used by the c++ toolchain, and specifies the tools installed in the Docker image.
+* The remote test execution toolchain, `root//toolchains:remote_test_execution_toolchain`. This toolchain defines platform options in the form of `capabilities`. Critically these include the `container-image`. This toolchain is identical to the one in the `buck2/cpp` sample in this repo.
+
+The `src`, `bin` and `test` cells:
+
+* Contain a copied version of https://github.com/facebook/buck2/tree/main/examples/with_prelude/rust that works with Buck2 and RE as configured in this sample project.
+
+* Key changes for remote execution in the top level `BUCK` file include setting environment variables to select the pre-installed rust toolchain in the container and set a custom value for `HOME`. This custom value is needed as `rustp` attempts to create a directory in the `HOME` location and we prefer this happen inside the `buck-out/` directory to guarantee actions do not modify any content outside of their scratch directory.
+
+To test the project with RE run (after setting up `.buckconfig` as indicated below):
+
+```
+buck2 test --remote-only //:test
+```
+
+You can also build the `main` for this sample by running:
+
+```
+buck2 build --remote-only //:main
+```
+
+Note the use of `--remote-only` to indicate remote execution should be used for the build / test command.
+
+### Relevant configs in `.buckconfig`
+
+The EngFlow endpoint and certificate should be configured as the
+following:
+
+```ini
+[buck2_re_client]
+engine_address       = <CLUSTER_NAME>.cluster.engflow.com
+action_cache_address = <CLUSTER_NAME>.cluster.engflow.com
+cas_address          = <CLUSTER_NAME>.cluster.engflow.com
+http_headers         = <AUTH_HTTP_HEADERS>
+ ```
+
+To obtain the value of `<AUTH_HTTP_HEADERS>`, log into https://<CLUSTER_NAME>.cluster.engflow.com/gettingstarted and obtain the value of `x-engflow-auth-token` in section `Method 2: JWT`, take note of this value. Then set `AUTH_HTTP_HEADERS` with the value `x-engflow-auth-method:jwt-v0,x-engflow-auth-token:<JWT_TOKEN_FROM_GETTINGSTARTED_PAGE>.
+
+Note for CI runs, the auth method used is [Github Tokens](https://docs.engflow.com/re/config/authentication.html#github-tokens).

buck2/rust/bin/main.rs

@@ -0,0 +1,3 @@
+fn main() {
+    library::print_hello();
+}

buck2/rust/platforms/BUCK

@@ -0,0 +1,42 @@
+# Copyright 2022 EngFlow Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load(":defs.bzl", "platforms")
+load(":defs.bzl", "action_keys")
+
+constraint_setting(
+  name = "re_provider"
+)
+
+constraint_value(
+  name = "engflow",
+  constraint_setting = ":re_provider",
+)
+
+# This platform configures details of remote execution.
+platforms(
+    name = "remote_platform",
+    cpu_configuration = "config//cpu:x86_64",
+    os_configuration = "config//os:linux",
+    re_provider = ":engflow",
+)
+
+# This action_key provides a default BuildModeInfo that is needed for RE of tests to function properly.
+# The values in `cell` and `mode` can be used, in practice, to create cache silos. Any values can be given to these attributes.
+action_keys(
+    name = "remote_execution_action_keys",
+    cell = "standard",
+    mode = "standard",
+    visibility = ["PUBLIC"],
+)

buck2/rust/platforms/defs.bzl

@@ -0,0 +1,77 @@
+# Copyright 2022 EngFlow Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This platform is essentially the same as the one provided in https://github.com/facebook/buck2/blob/804d62242214455d51787f7c8c96a1e12c75ec32/examples/remote_execution/engflow/platforms/defs.bzl
+# The main difference is we enable passing CPU and OS constraints and we use the sample EngFlow RE image.
+load("@prelude//:build_mode.bzl", "BuildModeInfo")
+
+def _platforms(ctx):
+    constraints = dict()
+    constraints.update(ctx.attrs.cpu_configuration[ConfigurationInfo].constraints)
+    constraints.update(ctx.attrs.os_configuration[ConfigurationInfo].constraints)
+    constraints.update(ctx.attrs.re_provider[ConfigurationInfo].constraints)
+    configuration = ConfigurationInfo(
+        constraints = constraints,
+        values = {},
+    )
+
+    # A bookworm image with rust pre-installed. Image details can be found in https://gallery.ecr.aws/docker/library/rust.
+    # Dockerfile can be found in https://github.com/rust-lang/docker-rust/blob/700c4f146427808cfb1e07a646e4afabbe99da4f/stable/bullseye/Dockerfile
+    image = "docker://public.ecr.aws/docker/library/rust:1.83.0-bullseye@sha256:24118f76a7da011b22a25b8e9dbdbb549ed29c1eba635d6aa4a9c9f5ed545066"
+    name = ctx.label.raw_target()
+    platform = ExecutionPlatformInfo(
+        label = ctx.label.raw_target(),
+        configuration = configuration,
+        executor_config = CommandExecutorConfig(
+            local_enabled = False,
+            remote_enabled = True,
+            use_limited_hybrid = False,
+            remote_execution_properties = {
+                "container-image": image,
+            },
+            remote_execution_use_case = "buck2-default",
+            # TODO: Use output_paths
+            remote_output_paths = "strict",
+        ),
+    )
+
+    return [
+        DefaultInfo(),
+        ExecutionPlatformRegistrationInfo(platforms = [platform]),
+        configuration,
+        PlatformInfo(label = str(name), configuration = configuration),
+    ]
+
+def _action_keys(ctx):
+    return [
+        DefaultInfo(),
+        BuildModeInfo(cell = ctx.attrs.cell, mode = ctx.attrs.mode),
+    ]
+
+platforms = rule(
+    attrs = {
+        "cpu_configuration": attrs.dep(providers = [ConfigurationInfo]),
+        "os_configuration": attrs.dep(providers = [ConfigurationInfo]),
+        "re_provider": attrs.dep(providers = [ConfigurationInfo]),
+    }, 
+    impl = _platforms
+)
+
+action_keys = rule(
+     attrs = {
+        "cell": attrs.string(),
+        "mode": attrs.string(),
+     },
+     impl = _action_keys
+)

buck2/rust/src/lib.rs

@@ -0,0 +1,3 @@
+pub fn print_hello() {
+    println!("hello world from rust toolchain");
+}

buck2/rust/test/test.rs

@@ -0,0 +1,4 @@
+#[test]
+fn test_it_doesnt_crash() {
+    library::print_hello();
+}

buck2/rust/toolchains/BUCK

@@ -0,0 +1,72 @@
+
+# Copyright 2022 EngFlow Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load("defs.bzl", "remote_rust_toolchain", "path_clang_tools")
+load("@prelude//toolchains:cxx.bzl", "cxx_tools_info_toolchain")
+load("@prelude//toolchains:python.bzl", "system_python_bootstrap_toolchain")
+load("@prelude//toolchains:remote_test_execution.bzl", "remote_test_execution_toolchain")
+load("@prelude//tests:test_toolchain.bzl", "noop_test_toolchain")
+
+remote_rust_toolchain(
+    name = "rust",
+    default_edition = "2021",
+    visibility = ["PUBLIC"],
+    rustdoc = "/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu/bin/rustc",
+    compiler = "/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu/bin/rustc",
+    clippy_driver = "/usr/local/rustup/toolchains/1.83.0-x86_64-unknown-linux-gnu/bin/rustc",
+    rustc_target_triple = "x86_64-unknown-linux-gnu",
+)
+
+# Python toolchain used in scripts that bootstrap other aspects of the Buck2 prelude.
+system_python_bootstrap_toolchain(
+    name = "python_bootstrap",
+    visibility = ["PUBLIC"],
+)
+
+# Custom clang tools that use g++ for linking.
+path_clang_tools(
+    name = "clang_tools",
+    visibility = ["PUBLIC"],
+)
+
+# Custom cpp toolchain that is compatible with the remote worker environment.
+cxx_tools_info_toolchain(
+    name = "cxx",
+    cxx_tools_info = ":clang_tools",
+    visibility = ["PUBLIC"],
+)
+
+# Default toolchain for remote execution of tests.
+# Note it defines a profile with a capability that defines the `container-image` that matches the one defined in //platforms:remote_platform. 
+# Capabilities are passed to the RE service to find workers that match them as Platform options.
+remote_test_execution_toolchain(
+    name = "remote_test_execution",
+    visibility = ["PUBLIC"],
+    default_profile = "cxx_re_toolchain",
+    profiles = {
+        "cxx_re_toolchain": {
+            "use_case": "cxx-testing",
+            "capabilities": {
+                "container-image" : "docker://public.ecr.aws/docker/library/rust:1.83.0-bullseye@sha256:24118f76a7da011b22a25b8e9dbdbb549ed29c1eba635d6aa4a9c9f5ed545066",
+            },
+        }
+    },
+)
+
+# In some cases the execution of test can fail looking for this `noop_test_toolchain`.
+noop_test_toolchain(
+    name = "test",
+    visibility = ["PUBLIC"],
+)