save

.gitignore

@@ -5,3 +5,4 @@ venv
 *.so
 *.log
 results-sf*
+*.tar

k8s/Dockerfile

@@ -1,4 +1,4 @@
-FROM rayproject/ray:2.9.0
+FROM rayproject/ray:2.37.0.cabc24-py312
 
 RUN sudo apt update && \
     sudo apt install -y curl build-essential
@@ -29,4 +29,6 @@ RUN source venv/bin/activate && \
     source /home/ray/.cargo/env && \
     maturin build --release
 
-RUN pip3 install /home/ray/target/wheels/datafusion_ray-0.6.0-cp38-abi3-manylinux_2_31_x86_64.whl
+FROM rayproject/ray:2.37.0.cabc24-py312
+COPY --from=0 /home/ray/target/wheels/datafusion_ray-0.6.0-cp38-abi3-manylinux_2_35_x86_64.whl /home/ray/datafusion_ray-0.6.0-cp38-abi3-manylinux_2_35_x86_64.whl
+RUN pip3 install /home/ray/datafusion_ray-0.6.0-cp38-abi3-manylinux_2_35_x86_64.whl

k8s/README.md

@@ -17,16 +17,32 @@ Run the following command to build the Docker image.
 docker build -t dfray -f k8s/Dockerfile .
 ```
 
-Import the image into k3s
+Either push the image to a repository that k3s can pull from (recommended), or copy the image directly into the nodes:
 
 ```shell
-docker save dfray | k3s ctr images import -
+docker save -o dfray.tar dfray
+scp dfray.tar username@node1:
+scp dfray.tar username@node2:
+```
+
+ssh into each node and run:
+
+```shell
+sudo k3s ctr images import dfray.tar
 ```
 
 ## Run the example
 
 From the `examples` directory:
 
 ```shell
-ray job submit --working-dir `pwd` --runtime-env ../k8s/runtime-env.yaml -- python3 tips.py
+ray job submit --runtime-env-json='{"container": {"image": "dfray"}}' -- python3 tips.py
 ```
+
+This fails with:
+
+```text
+ 'The 'container' field currently cannot be used together with other fields of runtime_env. Specified fields: dict_keys(['container', 'env_vars'])'
+```
+
+Possible fix: https://github.com/ray-project/ray/pull/42121

requirements-in.txt

@@ -6,6 +6,6 @@ mypy
 numpy
 pyarrow
 pytest
-ray==2.9.0
+ray==2.37.0
 toml
 importlib_metadata; python_version < "3.8"