DOC: update CI environment dependencies for doc build (#814)

Signed-off-by: Dawnfz <[email protected]>
xorbitsai · Sep 19, 2024 · dd46556 · dd46556
1 parent c765e57
commit dd46556
Show file tree

Hide file tree

Showing 3 changed files with 117 additions and 104 deletions.
diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml
@@ -129,7 +129,7 @@ jobs:
         minikube-version: 1.31.2
 
     - name: Install ucx dependencies
-      if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest')}}
+      if: ${{ (matrix.module != 'gpu') && (matrix.os == 'ubuntu-latest') && (matrix.module != 'doc-build') }}
       run: |
         conda install -c conda-forge -c rapidsai ucx-proc=*=cpu ucx ucx-py
     - name: Install libomp (macOS)
@@ -141,108 +141,114 @@ jobs:
         PYTHON: ${{ matrix.python-version }}
       if: ${{ matrix.module != 'gpu' }}
       run: |
-        pip install -e "git+https://github.com/xorbitsai/xoscar.git@main#subdirectory=python&egg=xoscar"
-        pip install "numpy<2.0.0" scipy cython pyftpdlib coverage flaky numexpr
+        if [[ "$MODULE" == "doc-build" ]]; then
+          pip install --upgrade --no-cache-dir pip setuptools
+          pip install --upgrade --no-cache-dir sphinx readthedocs-sphinx-ext
+          pip install --upgrade --upgrade-strategy only-if-needed --no-cache-dir ".[doc]"
+        else
+          pip install -e "git+https://github.com/xorbitsai/xoscar.git@main#subdirectory=python&egg=xoscar"
+          pip install "numpy<2.0.0" scipy cython pyftpdlib coverage flaky numexpr
 
-        if [[ "$MODULE" == "xorbits/pandas" ]]; then
-          pip install openpyxl
-        fi
-        if [[ "$MODULE" == "mars-core" ]]; then
-          pip install oss2
-        fi
-        if [[ "$MODULE" == "kubernetes" ]]; then
-          pip install kubernetes
-        fi
-        if [[ "$MODULE" == "kubernetes-juicefs" ]]; then
-          pip install kubernetes
-          if helm version --short >/dev/null 2>&1; then
-            echo "Helm installed. Skipping installation."
-          else
-            curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+          if [[ "$MODULE" == "xorbits/pandas" ]]; then
+            pip install openpyxl
           fi
-          if helm repo list | grep "juicefs" >/dev/null 2>&1; then
-            echo "Repo juicefs existed. Skipping adding to helm repos."
-          else
-            helm repo add juicefs https://juicedata.github.io/charts/
-            helm repo update
-            helm fetch --untar juicefs/juicefs-csi-driver
+          if [[ "$MODULE" == "mars-core" ]]; then
+            pip install oss2
           fi
-          if kubectl get pods -n kube-system | grep "juicefs-csi" >/dev/null 2>&1; then
-            echo "Juicefs-csi-driver installed. Skipping installation."
-          else
-            helm install juicefs-csi-driver juicefs/juicefs-csi-driver -n kube-system -f xorbits/deploy/kubernetes/external_storage/juicefs/tests/values.yaml
+          if [[ "$MODULE" == "kubernetes" ]]; then
+            pip install kubernetes
           fi
-          sleep 60
-          kubectl apply -f xorbits/deploy/kubernetes/external_storage/juicefs/tests/example-redis-config.yaml
-          sleep 60
-        fi
-        if [[ "$MODULE" == "hadoop" ]]; then
-          ../CI/install-hadoop.sh
-          echo "import coverage; coverage.process_startup()" > \
-            $(python -c "import site; print(site.getsitepackages()[-1])")/coverage.pth
-          conda install --quiet --yes conda-forge::libffi==3.4.2 conda-forge::skein==0.8.1 conda-forge::conda-pack==0.8.0 conda-forge::protobuf==3.20.1 conda-forge::grpcio==1.42.0
-        fi
-        if [[ "$MODULE" == "vineyard" ]]; then
-          pip install vineyard
-          mkdir -p /tmp/etcd-download-test
-          export ETCD_VER=v3.4.13
-          export ETCD_DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download
-          curl -L $ETCD_DOWNLOAD_URL/$ETCD_VER/etcd-$ETCD_VER-linux-amd64.tar.gz -o /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz
-          tar xzvf /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1
-          sudo mv /tmp/etcd-download-test/etcd /usr/local/bin/
-          sudo mv /tmp/etcd-download-test/etcdctl /usr/local/bin/
-          rm -fr /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz /tmp/etcd-download-test
-        fi
-        if [[ "$MODULE" == "external-storage" ]]; then
-          export CUR_DIR=$(pwd)
-          mkdir -p /tmp/alluxio-download-test
-          cd /tmp/alluxio-download-test
-          wget -q https://downloads.alluxio.io/downloads/files/2.9.3/alluxio-2.9.3-bin.tar.gz
-          tar -xzf alluxio-2.9.3-bin.tar.gz
-          cd alluxio-2.9.3
-          cp conf/alluxio-env.sh.template conf/alluxio-env.sh
-          cp conf/alluxio-site.properties.template conf/alluxio-site.properties
-          echo "alluxio.master.hostname=localhost" >> conf/alluxio-site.properties
-          cd $CUR_DIR
-          sudo mv /tmp/alluxio-download-test/alluxio-2.9.3 /usr/local/bin/
-          rm -R /tmp/alluxio-download-test
-          unset CUR_DIR
-          export ALLUXIO_HOME="/usr/local/bin/alluxio-2.9.3"
-          ${ALLUXIO_HOME}/bin/alluxio format
-          ${ALLUXIO_HOME}/bin/alluxio-start.sh local SudoMount
-          curl -sSL https://d.juicefs.com/install | sh -
-          curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg
-          echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list
-          sudo apt-get update
-          sudo apt-get install redis
-          export JUICEFS_HOME="/usr/local/bin/juicefs"
-        fi
-        if [[ "$PYTHON" == '3.9' ]]; then
-          pip install h5py zarr matplotlib fastparquet
-          conda install -n test --quiet --yes -c conda-forge python=$PYTHON \
-            "tiledb-py>=0.4.3,<0.6.0" "tiledb<2.0.0" || true
-        fi
-        if [[ "$MODULE" == "_mars/learn" ]]; then
-          pip install scikit-learn
-          pip install xgboost lightgbm keras tensorflow faiss-cpu\<1.7.3 torch torchvision \
-            statsmodels tsfresh dask[complete] mimesis\<9.0.0
-        fi
-        if [[ "$MODULE" == "learn" ]]; then
-          pip install xgboost lightgbm
-        fi
-        if [[ "$MODULE" == "compatibility" ]]; then
-          # test if compatible with older versions
-          pip install "pandas==1.5.3" "scipy<=1.10.1" "numpy<=1.24.1" "matplotlib<=3.7.0" "pyarrow<12.0.0" "sqlalchemy<2"
-        fi
-        if [[ "$MODULE" == "jax" ]]; then
-          # test jax
-          pip install "jax>=0.4.0" "jaxlib>=0.4.0"
-        fi
-        if [[ "$MODULE" == "datasets" ]]; then
-          # test huggingface datasets
-          pip install datasets torch
+          if [[ "$MODULE" == "kubernetes-juicefs" ]]; then
+            pip install kubernetes
+            if helm version --short >/dev/null 2>&1; then
+              echo "Helm installed. Skipping installation."
+            else
+              curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+            fi
+            if helm repo list | grep "juicefs" >/dev/null 2>&1; then
+              echo "Repo juicefs existed. Skipping adding to helm repos."
+            else
+              helm repo add juicefs https://juicedata.github.io/charts/
+              helm repo update
+              helm fetch --untar juicefs/juicefs-csi-driver
+            fi
+            if kubectl get pods -n kube-system | grep "juicefs-csi" >/dev/null 2>&1; then
+              echo "Juicefs-csi-driver installed. Skipping installation."
+            else
+              helm install juicefs-csi-driver juicefs/juicefs-csi-driver -n kube-system -f xorbits/deploy/kubernetes/external_storage/juicefs/tests/values.yaml
+            fi
+            sleep 60
+            kubectl apply -f xorbits/deploy/kubernetes/external_storage/juicefs/tests/example-redis-config.yaml
+            sleep 60
+          fi
+          if [[ "$MODULE" == "hadoop" ]]; then
+            ../CI/install-hadoop.sh
+            echo "import coverage; coverage.process_startup()" > \
+              $(python -c "import site; print(site.getsitepackages()[-1])")/coverage.pth
+            conda install --quiet --yes conda-forge::libffi==3.4.2 conda-forge::skein==0.8.1 conda-forge::conda-pack==0.8.0 conda-forge::protobuf==3.20.1 conda-forge::grpcio==1.42.0
+          fi
+          if [[ "$MODULE" == "vineyard" ]]; then
+            pip install vineyard
+            mkdir -p /tmp/etcd-download-test
+            export ETCD_VER=v3.4.13
+            export ETCD_DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download
+            curl -L $ETCD_DOWNLOAD_URL/$ETCD_VER/etcd-$ETCD_VER-linux-amd64.tar.gz -o /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz
+            tar xzvf /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1
+            sudo mv /tmp/etcd-download-test/etcd /usr/local/bin/
+            sudo mv /tmp/etcd-download-test/etcdctl /usr/local/bin/
+            rm -fr /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz /tmp/etcd-download-test
+          fi
+          if [[ "$MODULE" == "external-storage" ]]; then
+            export CUR_DIR=$(pwd)
+            mkdir -p /tmp/alluxio-download-test
+            cd /tmp/alluxio-download-test
+            wget -q https://downloads.alluxio.io/downloads/files/2.9.3/alluxio-2.9.3-bin.tar.gz
+            tar -xzf alluxio-2.9.3-bin.tar.gz
+            cd alluxio-2.9.3
+            cp conf/alluxio-env.sh.template conf/alluxio-env.sh
+            cp conf/alluxio-site.properties.template conf/alluxio-site.properties
+            echo "alluxio.master.hostname=localhost" >> conf/alluxio-site.properties
+            cd $CUR_DIR
+            sudo mv /tmp/alluxio-download-test/alluxio-2.9.3 /usr/local/bin/
+            rm -R /tmp/alluxio-download-test
+            unset CUR_DIR
+            export ALLUXIO_HOME="/usr/local/bin/alluxio-2.9.3"
+            ${ALLUXIO_HOME}/bin/alluxio format
+            ${ALLUXIO_HOME}/bin/alluxio-start.sh local SudoMount
+            curl -sSL https://d.juicefs.com/install | sh -
+            curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg
+            echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list
+            sudo apt-get update
+            sudo apt-get install redis
+            export JUICEFS_HOME="/usr/local/bin/juicefs"
+          fi
+          if [[ "$PYTHON" == '3.9' ]]; then
+            pip install h5py zarr matplotlib fastparquet
+            conda install -n test --quiet --yes -c conda-forge python=$PYTHON \
+              "tiledb-py>=0.4.3,<0.6.0" "tiledb<2.0.0" || true
+          fi
+          if [[ "$MODULE" == "_mars/learn" ]]; then
+            pip install scikit-learn
+            pip install xgboost lightgbm keras tensorflow faiss-cpu\<1.7.3 torch torchvision \
+              statsmodels tsfresh dask[complete] mimesis\<9.0.0
+          fi
+          if [[ "$MODULE" == "learn" ]]; then
+            pip install xgboost lightgbm
+          fi
+          if [[ "$MODULE" == "compatibility" ]]; then
+            # test if compatible with older versions
+            pip install "pandas==1.5.3" "scipy<=1.10.1" "numpy<=1.24.1" "matplotlib<=3.7.0" "pyarrow<12.0.0" "sqlalchemy<2"
+          fi
+          if [[ "$MODULE" == "jax" ]]; then
+            # test jax
+            pip install "jax>=0.4.0" "jaxlib>=0.4.0"
+          fi
+          if [[ "$MODULE" == "datasets" ]]; then
+            # test huggingface datasets
+            pip install datasets torch
+          fi
+          pip install -e ".[dev,extra,aws]"
         fi
-        pip install -e ".[dev,extra,aws]"
       working-directory: ./python
 
     - name: Install on JuiceFsCI

diff --git a/python/xorbits/lightgbm/__init__.py b/python/xorbits/lightgbm/__init__.py
@@ -22,7 +22,8 @@ def __dir__():
     try:
         import lightgbm
     except ImportError:  # pragma: no cover
-        raise AttributeError("lightgbm is required but not installed.")
+        # raise AttributeError("lightgbm is required but not installed.")
+        lightgbm = None
     from .mars_adapters import MARS_LIGHTGBM_CALLABLES
 
     return list(MARS_LIGHTGBM_CALLABLES.keys())
@@ -34,7 +35,8 @@ def __getattr__(name: str):
     try:
         import lightgbm
     except ImportError:  # pragma: no cover
-        raise AttributeError("lightgbm is required but not installed.")
+        # raise AttributeError("lightgbm is required but not installed.")
+        lightgbm = None
     from .mars_adapters import MARS_LIGHTGBM_CALLABLES
 
     if name in MARS_LIGHTGBM_CALLABLES:

diff --git a/python/xorbits/lightgbm/mars_adapters/core.py b/python/xorbits/lightgbm/mars_adapters/core.py
@@ -16,7 +16,10 @@
 import inspect
 from typing import Callable, Dict, List, Optional
 
-import lightgbm
+try:
+    import lightgbm
+except ImportError:  # pragma: no cover
+    lightgbm = None
 
 from ..._mars.learn.contrib import lightgbm as mars_lightgbm
 from ..._mars.learn.contrib.lightgbm.classifier import (
@@ -97,6 +100,8 @@ def _install_cls_members(module_callables: Dict[str, Callable]):
 
 MARS_LIGHTGBM_CALLABLES = _collect_module_callables(skip_members=["register_op"])
 _install_cls_members(MARS_LIGHTGBM_CALLABLES)
-attach_module_callable_docstring(LGBMClassifier, lightgbm, lightgbm.LGBMClassifier)
-attach_module_callable_docstring(LGBMRegressor, lightgbm, lightgbm.LGBMRegressor)
-attach_module_callable_docstring(LGBMRanker, lightgbm, lightgbm.LGBMRanker)
+
+if lightgbm is not None:
+    attach_module_callable_docstring(LGBMClassifier, lightgbm, lightgbm.LGBMClassifier)
+    attach_module_callable_docstring(LGBMRegressor, lightgbm, lightgbm.LGBMRegressor)
+    attach_module_callable_docstring(LGBMRanker, lightgbm, lightgbm.LGBMRanker)