From 0bb425adafcf53ccf3588e9e5a25292b1220cff7 Mon Sep 17 00:00:00 2001 From: luv-bansal Date: Wed, 13 Nov 2024 08:54:31 +0000 Subject: [PATCH 1/6] Nvidia NIM dockerfile --- .../dockerfile_template/Dockerfile.nim | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 clarifai/runners/dockerfile_template/Dockerfile.nim diff --git a/clarifai/runners/dockerfile_template/Dockerfile.nim b/clarifai/runners/dockerfile_template/Dockerfile.nim new file mode 100644 index 00000000..97aefdfe --- /dev/null +++ b/clarifai/runners/dockerfile_template/Dockerfile.nim @@ -0,0 +1,71 @@ +FROM nvcr.io/nim/meta/llama-3.1-8b-instruct:1.1.2 as build + +FROM gcr.io/distroless/python3-debian12:debug + + +COPY --from=build /bin/bash /bin/rbash +COPY --from=build /bin/sh /bin/sh +COPY --from=build /bin/rsh /bin/rsh + +# we have to overwrite the python3 binary that the distroless image uses +COPY --from=build /opt/nim/llm/.venv/bin/python3.10 /usr/bin/python3 +COPY --from=build /opt/nim/llm/.venv/bin/python3.10 /usr/local/bin/python3.10 + +# also copy in all the lib files for it. +COPY --from=build /lib /lib +COPY --from=build /lib64 /lib64 +COPY --from=build /usr/lib/ /usr/lib/ +COPY --from=build /usr/local/lib/ /usr/local/lib/ +# ldconfig is needed to update the shared library cache so system libraries (like CUDA) can be found +COPY --from=build /usr/sbin/ldconfig /sbin/ldconfig +COPY --from=build /usr/sbin/ldconfig.real /sbin/ldconfig.real +COPY --from=build /etc/ld.so.conf /etc/ld.so.conf +COPY --from=build /etc/ld.so.cache /etc/ld.so.cache +COPY --from=build /etc/ld.so.conf.d/ /etc/ld.so.conf.d/ + +# COPY NIM files +COPY --from=build /opt /opt +COPY --from=build /etc/nim /etc/nim + +# Set environment variables to use the nim libraries and python +ENV PYTHONPATH=${PYTHONPATH}:/opt/nim/llm/.venv/lib/python3.10/site-packages:/opt/nim/llm +ENV PATH="/opt/nim/llm/.venv/bin:/opt/hpcx/ucc/bin:/opt/hpcx/ucx/bin:/opt/hpcx/ompi/bin:$PATH" + +ENV LD_LIBRARY_PATH="/opt/hpcx/ucc/lib/ucc:/opt/hpcx/ucc/lib:/opt/hpcx/ucx/lib/ucx:/opt/hpcx/ucx/lib:/opt/hpcx/ompi/lib:/opt/hpcx/ompi/lib/openmpi:/opt/nim/llm/.venv/lib/python3.10/site-packages/tensorrt_llm/libs:/opt/nim/llm/.venv/lib/python3.10/site-packages/nvidia/cublas/lib:/opt/nim/llm/.venv/lib/python3.10/site-packages/tensorrt_libs:/opt/nim/llm/.venv/lib/python3.10/site-packages/nvidia/nccl/lib:$LD_LIBRARY_PATH" + +ENV LIBRARY_PATH=/opt/hpcx/ucc/lib:/opt/hpcx/ucx/lib:/opt/hpcx/ompi/lib:$LIBRARY_PATH + +ENV CPATH=/opt/hpcx/ompi/include:/opt/hpcx/ucc/include:/opt/hpcx/ucx/include:$CPATH +ENV LLM_PROJECT_DIR=/opt/nim/llm + +# Set environment variables for MPI +ENV OMPI_HOME=/opt/hpcx/ompi +ENV HPCX_MPI_DIR=/opt/hpcx/ompi +ENV MPIf_HOME=/opt/hpcx/ompi +ENV OPAL_PREFIX=/opt/hpcx/ompi + +# Set environment variables for UCC +ENV UCC_DIR=/opt/hpcx/ucc/lib/cmake/ucc +ENV UCC_HOME=/opt/hpcx/ucc +ENV HPCX_UCC_DIR=/opt/hpcx/ucc +ENV USE_UCC=1 +ENV USE_SYSTEM_UCC=1 + +# Set environment variables for HPC-X +ENV HPCX_DIR=/opt/hpcx +ENV HPCX_UCX_DIR=/opt/hpcx/ucx +ENV HPCX_MPI_DIR=/opt/hpcx/ompi + +# Set environment variables for UCX +ENV UCX_DIR=/opt/hpcx/ucx/lib/cmake/ucx +ENV UCX_HOME=/opt/hpcx/ucx + +ENV HOME=/opt/nim/llm + +# ln is needed to create symbolic links (needed by nvidia-container-runtime) +COPY --from=build /usr/bin/ln /usr/bin/ln + +# Run ldconfig in the build stage to update the library cache else CUDA libraries won't be found +RUN ldconfig -v + +SHELL ["/bin/rbash", "-c"] From 71dd021ad901fb623dbc47e836cd950817aa9244 Mon Sep 17 00:00:00 2001 From: Luv Bansal <70321430+luv-bansal@users.noreply.github.com> Date: Wed, 13 Nov 2024 14:49:45 +0530 Subject: [PATCH 2/6] Update Dockerfile.nim --- clarifai/runners/dockerfile_template/Dockerfile.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/clarifai/runners/dockerfile_template/Dockerfile.nim b/clarifai/runners/dockerfile_template/Dockerfile.nim index 97aefdfe..20bd4948 100644 --- a/clarifai/runners/dockerfile_template/Dockerfile.nim +++ b/clarifai/runners/dockerfile_template/Dockerfile.nim @@ -3,9 +3,8 @@ FROM nvcr.io/nim/meta/llama-3.1-8b-instruct:1.1.2 as build FROM gcr.io/distroless/python3-debian12:debug -COPY --from=build /bin/bash /bin/rbash +COPY --from=build /bin/bash /bin/bash COPY --from=build /bin/sh /bin/sh -COPY --from=build /bin/rsh /bin/rsh # we have to overwrite the python3 binary that the distroless image uses COPY --from=build /opt/nim/llm/.venv/bin/python3.10 /usr/bin/python3 @@ -65,7 +64,8 @@ ENV HOME=/opt/nim/llm # ln is needed to create symbolic links (needed by nvidia-container-runtime) COPY --from=build /usr/bin/ln /usr/bin/ln +# Now sure about below `ldconfig` command now, before CUDA libraries wasn't found without running `ldconfig` but not it seems to be working # Run ldconfig in the build stage to update the library cache else CUDA libraries won't be found RUN ldconfig -v -SHELL ["/bin/rbash", "-c"] +SHELL ["/bin/bash", "-c"] From 8515a7a9dc64ae33bbdefd154c2ab201880bc42d Mon Sep 17 00:00:00 2001 From: luv-bansal Date: Tue, 19 Nov 2024 08:37:54 -0500 Subject: [PATCH 3/6] Working NIM image --- .../dockerfile_template/Dockerfile.nim | 50 ++++++++++++------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/clarifai/runners/dockerfile_template/Dockerfile.nim b/clarifai/runners/dockerfile_template/Dockerfile.nim index 97aefdfe..3a24dd86 100644 --- a/clarifai/runners/dockerfile_template/Dockerfile.nim +++ b/clarifai/runners/dockerfile_template/Dockerfile.nim @@ -1,15 +1,36 @@ +ARG PYTHON_VERSION=3.10 +FROM public.ecr.aws/docker/library/python:${PYTHON_VERSION}-slim-bookworm as deps +ARG PYTHON_VERSION=3.10 +ENV DEBIAN_FRONTEND=noninteractive + + +RUN python${PYTHON_VERSION} -m venv /venv && \ + /venv/bin/pip install --disable-pip-version-check --upgrade pip setuptools wheel && \ + ln -sf /usr/bin/python${PYTHON_VERSION} /usr/bin/python3 && \ + apt-get clean && rm -rf /var/lib/apt/lists/*; + +# Use the NIM base image as another build stage FROM nvcr.io/nim/meta/llama-3.1-8b-instruct:1.1.2 as build +# Final image based on distroless FROM gcr.io/distroless/python3-debian12:debug +ARG PYTHON_VERSION=3.10 +# virtual env +COPY --from=deps /venv /venv +# we have to overwrite the python3 binary that the distroless image uses +COPY --from=deps /usr/local/bin/python${PYTHON_VERSION} /usr/bin/python3 +COPY --from=deps /usr/local/bin/python${PYTHON_VERSION} /usr/local/bin/python${PYTHON_VERSION} -COPY --from=build /bin/bash /bin/rbash -COPY --from=build /bin/sh /bin/sh -COPY --from=build /bin/rsh /bin/rsh +# Copy NIM files +COPY --from=build /opt /opt +COPY --from=build /etc/nim /etc/nim -# we have to overwrite the python3 binary that the distroless image uses -COPY --from=build /opt/nim/llm/.venv/bin/python3.10 /usr/bin/python3 -COPY --from=build /opt/nim/llm/.venv/bin/python3.10 /usr/local/bin/python3.10 +# Copy necessary binaries and libraries from the NIM base image +COPY --from=build /bin/bash /bin/bash +# COPY --from=build /bin/sh /bin/sh +COPY --from=build /bin/ssh /bin/ssh +COPY --from=build /usr/bin/ln /usr/bin/ln # also copy in all the lib files for it. COPY --from=build /lib /lib @@ -23,13 +44,10 @@ COPY --from=build /etc/ld.so.conf /etc/ld.so.conf COPY --from=build /etc/ld.so.cache /etc/ld.so.cache COPY --from=build /etc/ld.so.conf.d/ /etc/ld.so.conf.d/ -# COPY NIM files -COPY --from=build /opt /opt -COPY --from=build /etc/nim /etc/nim -# Set environment variables to use the nim libraries and python -ENV PYTHONPATH=${PYTHONPATH}:/opt/nim/llm/.venv/lib/python3.10/site-packages:/opt/nim/llm -ENV PATH="/opt/nim/llm/.venv/bin:/opt/hpcx/ucc/bin:/opt/hpcx/ucx/bin:/opt/hpcx/ompi/bin:$PATH" +# Set environment variables +ENV PYTHONPATH=/venv/lib/python3.10/site-packages:/opt/nim/llm/.venv/lib/python3.10/site-packages:/opt/nim/llm +ENV PATH="/usr/local/bin:/venv/bin:/opt/nim/llm/.venv/bin:/opt/hpcx/ucc/bin:/opt/hpcx/ucx/bin:/opt/hpcx/ompi/bin:$PATH" ENV LD_LIBRARY_PATH="/opt/hpcx/ucc/lib/ucc:/opt/hpcx/ucc/lib:/opt/hpcx/ucx/lib/ucx:/opt/hpcx/ucx/lib:/opt/hpcx/ompi/lib:/opt/hpcx/ompi/lib/openmpi:/opt/nim/llm/.venv/lib/python3.10/site-packages/tensorrt_llm/libs:/opt/nim/llm/.venv/lib/python3.10/site-packages/nvidia/cublas/lib:/opt/nim/llm/.venv/lib/python3.10/site-packages/tensorrt_libs:/opt/nim/llm/.venv/lib/python3.10/site-packages/nvidia/nccl/lib:$LD_LIBRARY_PATH" @@ -62,10 +80,4 @@ ENV UCX_HOME=/opt/hpcx/ucx ENV HOME=/opt/nim/llm -# ln is needed to create symbolic links (needed by nvidia-container-runtime) -COPY --from=build /usr/bin/ln /usr/bin/ln - -# Run ldconfig in the build stage to update the library cache else CUDA libraries won't be found -RUN ldconfig -v - -SHELL ["/bin/rbash", "-c"] +SHELL ["/bin/bash", "-c"] From 11f7c0a4e871472e64a4c1f0b8276bbf0d9c0ec5 Mon Sep 17 00:00:00 2001 From: luv-bansal Date: Tue, 19 Nov 2024 08:53:53 -0500 Subject: [PATCH 4/6] rupdate nim image --- .../dockerfile_template/Dockerfile.nim | 48 ++++++++++++------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/clarifai/runners/dockerfile_template/Dockerfile.nim b/clarifai/runners/dockerfile_template/Dockerfile.nim index 2fcaf9f4..0d161e85 100644 --- a/clarifai/runners/dockerfile_template/Dockerfile.nim +++ b/clarifai/runners/dockerfile_template/Dockerfile.nim @@ -1,14 +1,36 @@ +# Use an intermediate image to install pip and other dependencies +ARG PYTHON_VERSION=3.10 +FROM public.ecr.aws/docker/library/python:${PYTHON_VERSION}-slim-bookworm as deps +ARG PYTHON_VERSION=3.10 +ENV DEBIAN_FRONTEND=noninteractive + + +RUN python${PYTHON_VERSION} -m venv /venv && \ + /venv/bin/pip install --disable-pip-version-check --upgrade pip setuptools wheel && \ + ln -sf /usr/bin/python${PYTHON_VERSION} /usr/bin/python3 && \ + apt-get clean && rm -rf /var/lib/apt/lists/*; + +# Use the NIM base image as another build stage FROM nvcr.io/nim/meta/llama-3.1-8b-instruct:1.1.2 as build +# Final image based on distroless FROM gcr.io/distroless/python3-debian12:debug +ARG PYTHON_VERSION=3.10 +# virtual env +COPY --from=deps /venv /venv +# we have to overwrite the python3 binary that the distroless image uses +COPY --from=deps /usr/local/bin/python${PYTHON_VERSION} /usr/bin/python3 +COPY --from=deps /usr/local/bin/python${PYTHON_VERSION} /usr/local/bin/python${PYTHON_VERSION} -COPY --from=build /bin/bash /bin/bash -COPY --from=build /bin/sh /bin/sh +# Copy NIM files +COPY --from=build /opt /opt +COPY --from=build /etc/nim /etc/nim -# we have to overwrite the python3 binary that the distroless image uses -COPY --from=build /opt/nim/llm/.venv/bin/python3.10 /usr/bin/python3 -COPY --from=build /opt/nim/llm/.venv/bin/python3.10 /usr/local/bin/python3.10 +# Copy necessary binaries and libraries from the NIM base image +COPY --from=build /bin/bash /bin/bash +COPY --from=build /bin/ssh /bin/ssh +COPY --from=build /usr/bin/ln /usr/bin/ln # also copy in all the lib files for it. COPY --from=build /lib /lib @@ -22,13 +44,10 @@ COPY --from=build /etc/ld.so.conf /etc/ld.so.conf COPY --from=build /etc/ld.so.cache /etc/ld.so.cache COPY --from=build /etc/ld.so.conf.d/ /etc/ld.so.conf.d/ -# COPY NIM files -COPY --from=build /opt /opt -COPY --from=build /etc/nim /etc/nim -# Set environment variables to use the nim libraries and python -ENV PYTHONPATH=${PYTHONPATH}:/opt/nim/llm/.venv/lib/python3.10/site-packages:/opt/nim/llm -ENV PATH="/opt/nim/llm/.venv/bin:/opt/hpcx/ucc/bin:/opt/hpcx/ucx/bin:/opt/hpcx/ompi/bin:$PATH" +# Set environment variables +ENV PYTHONPATH=/venv/lib/python3.10/site-packages:/opt/nim/llm/.venv/lib/python3.10/site-packages:/opt/nim/llm +ENV PATH="/usr/local/bin:/venv/bin:/opt/nim/llm/.venv/bin:/opt/hpcx/ucc/bin:/opt/hpcx/ucx/bin:/opt/hpcx/ompi/bin:$PATH" ENV LD_LIBRARY_PATH="/opt/hpcx/ucc/lib/ucc:/opt/hpcx/ucc/lib:/opt/hpcx/ucx/lib/ucx:/opt/hpcx/ucx/lib:/opt/hpcx/ompi/lib:/opt/hpcx/ompi/lib/openmpi:/opt/nim/llm/.venv/lib/python3.10/site-packages/tensorrt_llm/libs:/opt/nim/llm/.venv/lib/python3.10/site-packages/nvidia/cublas/lib:/opt/nim/llm/.venv/lib/python3.10/site-packages/tensorrt_libs:/opt/nim/llm/.venv/lib/python3.10/site-packages/nvidia/nccl/lib:$LD_LIBRARY_PATH" @@ -61,11 +80,4 @@ ENV UCX_HOME=/opt/hpcx/ucx ENV HOME=/opt/nim/llm -# ln is needed to create symbolic links (needed by nvidia-container-runtime) -COPY --from=build /usr/bin/ln /usr/bin/ln - -# Now sure about below `ldconfig` command now, before CUDA libraries wasn't found without running `ldconfig` but not it seems to be working -# Run ldconfig in the build stage to update the library cache else CUDA libraries won't be found -RUN ldconfig -v - SHELL ["/bin/bash", "-c"] From 6948594852f2477cce07d1134e137f8f3989a372 Mon Sep 17 00:00:00 2001 From: luv-bansal Date: Wed, 20 Nov 2024 04:45:48 -0500 Subject: [PATCH 5/6] NIM integration --- ...Dockerfile.nim => Dockerfile.nim.template} | 49 +++++++++++-- clarifai/runners/models/model_upload.py | 69 +++++++++++++------ 2 files changed, 93 insertions(+), 25 deletions(-) rename clarifai/runners/dockerfile_template/{Dockerfile.nim => Dockerfile.nim.template} (65%) diff --git a/clarifai/runners/dockerfile_template/Dockerfile.nim b/clarifai/runners/dockerfile_template/Dockerfile.nim.template similarity index 65% rename from clarifai/runners/dockerfile_template/Dockerfile.nim rename to clarifai/runners/dockerfile_template/Dockerfile.nim.template index 0d161e85..f7a4a2cf 100644 --- a/clarifai/runners/dockerfile_template/Dockerfile.nim +++ b/clarifai/runners/dockerfile_template/Dockerfile.nim.template @@ -1,7 +1,5 @@ # Use an intermediate image to install pip and other dependencies -ARG PYTHON_VERSION=3.10 -FROM public.ecr.aws/docker/library/python:${PYTHON_VERSION}-slim-bookworm as deps -ARG PYTHON_VERSION=3.10 +FROM --platform=$TARGETPLATFORM public.ecr.aws/docker/library/python:${PYTHON_VERSION}-slim-bookworm as deps ENV DEBIAN_FRONTEND=noninteractive @@ -10,12 +8,14 @@ RUN python${PYTHON_VERSION} -m venv /venv && \ ln -sf /usr/bin/python${PYTHON_VERSION} /usr/bin/python3 && \ apt-get clean && rm -rf /var/lib/apt/lists/*; +# Install the NIM base image +ENV NGC_API_KEY=${NGC_API_KEY} + # Use the NIM base image as another build stage -FROM nvcr.io/nim/meta/llama-3.1-8b-instruct:1.1.2 as build +FROM --platform=$TARGETPLATFORM ${NIM_IMAGE} as build # Final image based on distroless FROM gcr.io/distroless/python3-debian12:debug -ARG PYTHON_VERSION=3.10 # virtual env COPY --from=deps /venv /venv @@ -81,3 +81,42 @@ ENV UCX_HOME=/opt/hpcx/ucx ENV HOME=/opt/nim/llm SHELL ["/bin/bash", "-c"] + +# These will be set by the templaing system. +ENV CLARIFAI_PAT=${CLARIFAI_PAT} +ENV CLARIFAI_USER_ID=${CLARIFAI_USER_ID} +ENV CLARIFAI_RUNNER_ID=${CLARIFAI_RUNNER_ID} +ENV CLARIFAI_NODEPOOL_ID=${CLARIFAI_NODEPOOL_ID} +ENV CLARIFAI_COMPUTE_CLUSTER_ID=${CLARIFAI_COMPUTE_CLUSTER_ID} +ENV CLARIFAI_API_BASE=${CLARIFAI_API_BASE} + +############################# +# User specific requirements +############################# +COPY requirements.txt . + +# Install requirements and clarifai package and cleanup before leaving this line. +# Note(zeiler): this could be in a future template as {{model_python_deps}} +RUN pip install --no-cache-dir -r requirements.txt && \ + pip install --no-cache-dir clarifai + +# Set the NUMBA cache dir to /tmp +ENV NUMBA_CACHE_DIR=/tmp/numba_cache +ENV LOCAL_NIM_CACHE=/tmp/nim_cache + + +# Set the working directory to /app +WORKDIR /app + +# Copy the current folder into /app/model_dir that the SDK will expect. +# Note(zeiler): would be nice to exclude checkpoints in case they were pre-downloaded. +COPY . /app/model_dir/${name} + +# Add the model directory to the python path. +ENV PYTHONPATH=${PYTHONPATH}:/app/model_dir/${name} + +ENTRYPOINT ["python", "-m", "clarifai.runners.server"] + +# Finally run the clarifai entrypoint to start the runner loop and local dev server. +# Note(zeiler): we may want to make this a clarifai CLI call. +CMD ["--model_path", "/app/model_dir/main"] diff --git a/clarifai/runners/models/model_upload.py b/clarifai/runners/models/model_upload.py index 014f8a20..f0228300 100644 --- a/clarifai/runners/models/model_upload.py +++ b/clarifai/runners/models/model_upload.py @@ -101,9 +101,10 @@ def _validate_config_checkpoints(self): assert "type" in self.config.get("checkpoints"), "No loader type specified in the config file" loader_type = self.config.get("checkpoints").get("type") + loader_type = loader_type.lower() if not loader_type: logger.info("No loader type specified in the config file for checkpoints") - assert loader_type == "huggingface", "Only huggingface loader supported for now" + if loader_type == "huggingface": assert "repo_id" in self.config.get("checkpoints"), "No repo_id specified in the config file" repo_id = self.config.get("checkpoints").get("repo_id") @@ -114,6 +115,15 @@ def _validate_config_checkpoints(self): else: hf_token = self.config.get("checkpoints").get("hf_token", None) return repo_id, hf_token + elif loader_type == "nvidia-nim" or loader_type == "nim": + assert "nim_image" in self.config.get( + "checkpoints"), "No nim_image specified in the config file" + assert "ngc_api_key" in self.config.get( + "checkpoints"), "No ngc_api_key specified in the config file" + + nim_image = self.config.get("checkpoints").get("nim_image") + ngc_api_key = self.config.get("checkpoints").get("ngc_api_key") + return nim_image, ngc_api_key @property def client(self): @@ -218,11 +228,21 @@ def _parse_requirements(self): return deendencies_version def create_dockerfile(self): - dockerfile_template = os.path.join( - os.path.dirname(os.path.dirname(__file__)), - 'dockerfile_template', - 'Dockerfile.template', - ) + loader_type = None + if self.config.get("checkpoints"): + loader_type = self.config.get("checkpoints").get("type") + if loader_type == "nvidia-nim" or loader_type == "nim": + dockerfile_template = os.path.join( + os.path.dirname(os.path.dirname(__file__)), + 'dockerfile_template', + 'Dockerfile.nim.template', + ) + else: + dockerfile_template = os.path.join( + os.path.dirname(os.path.dirname(__file__)), + 'dockerfile_template', + 'Dockerfile.template', + ) with open(dockerfile_template, 'r') as template_file: dockerfile_template = template_file.read() @@ -248,24 +268,32 @@ def create_dockerfile(self): base_image = self.PYTHON_BASE_IMAGE.format(python_version=python_version) - # Parse the requirements.txt file to determine the base image - dependencies = self._parse_requirements() - if 'torch' in dependencies and dependencies['torch']: - torch_version = dependencies['torch'] - - for image in self.AVAILABLE_TORCH_IMAGES: - if torch_version in image and f'py{python_version}' in image: - base_image = self.TORCH_BASE_IMAGE.format( - torch_version=torch_version, - python_version=python_version, - cuda_version=self.DEFAULT_CUDA_VERSION) - logger.info(f"Using Torch version {torch_version} base image to build the Docker image") - break + if loader_type == "nvidia-nim" or loader_type == "nim": + + base_image, ngc_api_key = self._validate_config_checkpoints() + else: + + # Parse the requirements.txt file to determine the base image + dependencies = self._parse_requirements() + if 'torch' in dependencies and dependencies['torch']: + torch_version = dependencies['torch'] + + for image in self.AVAILABLE_TORCH_IMAGES: + if torch_version in image and f'py{python_version}' in image: + base_image = self.TORCH_BASE_IMAGE.format( + torch_version=torch_version, + python_version=python_version, + cuda_version=self.DEFAULT_CUDA_VERSION) + logger.info( + f"Using Torch version {torch_version} base image to build the Docker image") + break # Replace placeholders with actual values dockerfile_content = dockerfile_template.safe_substitute( name='main', BASE_IMAGE=base_image, + NGC_API_KEY=ngc_api_key if loader_type == "nvidia-nim" or loader_type == "nim" else None, + PYTHON_VERSION=python_version, ) # Write Dockerfile @@ -363,7 +391,8 @@ def upload_model_version(self, download_checkpoints): logger.info( f"Model type {model_type_id} requires concepts to be specified in the config.yaml file.." ) - if self.config.get("checkpoints"): + if self.config.get("checkpoints") and self.config.get("checkpoints").get( + "type") == "huggingface": logger.info( "Checkpoints specified in the config.yaml file, will download the HF model's config.json file to infer the concepts." ) From dff62f8f1a6db0815f021ca71405c63797ac89a8 Mon Sep 17 00:00:00 2001 From: luv-bansal Date: Wed, 20 Nov 2024 04:50:14 -0500 Subject: [PATCH 6/6] NIM integration --- clarifai/runners/dockerfile_template/Dockerfile.nim.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clarifai/runners/dockerfile_template/Dockerfile.nim.template b/clarifai/runners/dockerfile_template/Dockerfile.nim.template index f7a4a2cf..7fe37a48 100644 --- a/clarifai/runners/dockerfile_template/Dockerfile.nim.template +++ b/clarifai/runners/dockerfile_template/Dockerfile.nim.template @@ -12,7 +12,7 @@ RUN python${PYTHON_VERSION} -m venv /venv && \ ENV NGC_API_KEY=${NGC_API_KEY} # Use the NIM base image as another build stage -FROM --platform=$TARGETPLATFORM ${NIM_IMAGE} as build +FROM --platform=$TARGETPLATFORM ${BASE_IMAGE} as build # Final image based on distroless FROM gcr.io/distroless/python3-debian12:debug