diff --git a/Dockerfile b/Dockerfile
index 7200533309c..2d69a5eeb74 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -174,7 +174,7 @@ COPY server/Makefile-flashinfer Makefile
 RUN make install-flashinfer
 
 # Text Generation Inference base image
-FROM nvidia/cuda:12.1.0-base-ubuntu22.04 AS base
+FROM nvidia/cuda:12.4.1-devel-ubuntu22.04 AS base
 
 # Conda env
 ENV PATH=/opt/conda/bin:$PATH \