Skip to content

Commit

Permalink
cicd: add pip mirror https://pypi.python.org/simple for bigdl related…
Browse files Browse the repository at this point in the history
… package (intel#6827)

* add pip mirror

* update

* typo
  • Loading branch information
liu-shaojun authored Dec 1, 2022
1 parent 3ee036b commit f2a0325
Show file tree
Hide file tree
Showing 31 changed files with 63 additions and 63 deletions.
2 changes: 1 addition & 1 deletion .github/actions/appTests/scala-appTests/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ runs:
source activate py37
pip install wheel
pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3
wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requrirements/requirements-friesian-python-ut-spark3.txt -O ${{ github.workspace }}/requirements-friesian-python-ut-spark3.txt
wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requirements/requirements-friesian-python-ut-spark3.txt -O ${{ github.workspace }}/requirements-friesian-python-ut-spark3.txt
pip uninstall -r ${{ github.workspace }}/requirements-friesian-python-ut-spark3.txt -y
pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} -r ${{ github.workspace }}/requirements-friesian-python-ut-spark3.txt
Expand Down
4 changes: 2 additions & 2 deletions .github/actions/colab/colab-python-py37-tf1/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ runs:
source activate py37
pip install wheel
pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3
wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requrirements/requirements-colab-python-tf1-spark3.txt -O ${{ github.workspace }}/requirements-colab-python-tf1-spark3.txt
wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requirements/requirements-colab-python-tf1-spark3.txt -O ${{ github.workspace }}/requirements-colab-python-tf1-spark3.txt
pip uninstall -r ${{ github.workspace }}/requirements-colab-python-tf1-spark3.txt -y
pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} -r ${{ github.workspace }}/requirements-colab-python-tf1-spark3.txt
Expand All @@ -48,7 +48,7 @@ runs:
chmod a+x python/orca/dev/colab-notebook/ipynb2py.sh
chmod a+x python/orca/dev/colab-notebook/run-notebook-tf1.sh
pip uninstall -y bigdl bigdl-dllib bigdl-orca
pip install --pre --upgrade bigdl-orca
pip install -i https://pypi.org/simple --pre --upgrade bigdl-orca
pip install jupyter
bash python/orca/dev/colab-notebook/run-notebook-tf1.sh
source deactivate
Expand Down
4 changes: 2 additions & 2 deletions .github/actions/colab/colab-python-py37-tf2/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ runs:
source activate py37
pip install wheel
pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3
wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requrirements/requirements-friesian-python-tf2-spark3.txt -O ${{ github.workspace }}/requirements-friesian-python-tf2-spark3.txt
wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requirements/requirements-friesian-python-tf2-spark3.txt -O ${{ github.workspace }}/requirements-friesian-python-tf2-spark3.txt
pip uninstall -r ${{ github.workspace }}/requirements-friesian-python-tf2-spark3.txt -y
pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} -r ${{ github.workspace }}/requirements-friesian-python-tf2-spark3.txt
Expand All @@ -51,7 +51,7 @@ runs:
pip uninstall -y bigdl bigdl-dllib bigdl-orca pyspark
# pip install --pre --upgrade bigdl-orca[all]
pip install jupyter argparse
pip install --pre --upgrade bigdl-orca-spark3[ray]
pip install -i https://pypi.org/simple --pre --upgrade bigdl-orca-spark3[ray]
bash python/orca/dev/colab-notebook/run-notebook-ray.sh
bash python/orca/dev/colab-notebook/run-notebook-tf2.sh
pip uninstall -y bigdl bigdl-dllib bigdl-orca
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ runs:
source python/friesian/dev/prepare_env.sh
export SPARK_LOCAL_HOSTNAME=localhost
pip install pyspark==3.1.3
pip install bigdl-math bigdl-tf
pip install -i https://pypi.org/simple bigdl-math bigdl-tf
cd python/friesian/dev/test
bash run-feature-example-tests.sh
bash run-example-tests-tf1.15.0.sh
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ runs:
pip install wheel
pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3
wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requrirements/requirements-friesian-python-ut-spark3.txt -O ${{ github.workspace }}/requirements-friesian-python-ut-spark3.txt
wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requirements/requirements-friesian-python-ut-spark3.txt -O ${{ github.workspace }}/requirements-friesian-python-ut-spark3.txt
pip uninstall -r ${{ github.workspace }}/requirements-friesian-python-ut-spark3.txt -y
pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} -r ${{ github.workspace }}/requirements-friesian-python-ut-spark3.txt
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ runs:
pip install wheel
pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3
wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requrirements/requirements-friesian-python-tf2-spark3.txt -O ${{ github.workspace }}/requirements-friesian-python-tf2-spark3.txt
wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requirements/requirements-friesian-python-tf2-spark3.txt -O ${{ github.workspace }}/requirements-friesian-python-tf2-spark3.txt
pip uninstall -r ${{ github.workspace }}/requirements-friesian-python-tf2-spark3.txt -y
pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} -r ${{ github.workspace }}/requirements-friesian-python-tf2-spark3.txt
pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} deepctr==0.9.2 --no-deps
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ runs:
export KERAS_BACKEND=tensorflow
chmod a+x python/dllib/examples/run-example-tests-pip.sh
pip install --pre --upgrade bigdl-dllib -i https://pypi.python.org/simple
pip install -i https://pypi.python.org/simple --pre --upgrade bigdl-dllib
python/dllib/examples/run-example-tests-pip.sh
source deactivate
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ runs:
# install orca
pip install -i https://pypi.org/simple python/orca/src/dist/bigdl_orca*-py3-none-macosx_10_11_x86_64.whl
# install friesian
pip install python/friesian/src/dist/bigdl_friesian*-py3-none-macosx_10_11_x86_64.whl
pip install -i https://pypi.python.org/simple python/friesian/src/dist/bigdl_friesian*-py3-none-macosx_10_11_x86_64.whl
cd python/friesian/dev/test
bash run-feature-example-tests.sh
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ runs:
pip uninstall -y bigdl-dllib bigdl-orca bigdl-chronos bigdl-orca-spark3 bigdl-dllib-spark3 bigdl-chronos-spark3 pyspark
pip install pyspark==3.1.3
pip install bigdl-tf bigdl-math
pip install -i https://pypi.python.org/simple bigdl-tf bigdl-math
echo "Running py37 tests"
cd python/friesian/dev/test
bash run-feature-example-tests.sh
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ runs:
# install orca
pip install -i https://pypi.org/simple python/orca/src/dist/bigdl_orca*-py3-none-macosx_10_11_x86_64.whl
# install friesian
pip install python/friesian/src/dist/bigdl_friesian*-py3-none-macosx_10_11_x86_64.whl
pip install -i https://pypi.python.org/simple python/friesian/src/dist/bigdl_friesian*-py3-none-macosx_10_11_x86_64.whl
cd python/friesian/dev/test
gsed -i "s/sed/gsed/g" run-example-tests-tf2.6.0.sh
Expand Down
4 changes: 2 additions & 2 deletions .github/actions/orca-ray-ctx-example-action/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ runs:
pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3
wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requrirements/requirements-orca-python-ray.txt -O ${{ github.workspace }}/requirements-orca-python-ray.txt
wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requirements/requirements-orca-python-ray.txt -O ${{ github.workspace }}/requirements-orca-python-ray.txt
pip uninstall -r ${{ github.workspace }}/requirements-orca-python-ray.txt -y
pip install -i https://pypi.tuna.tsinghua.edu.cn/simple -r ${{ github.workspace }}/requirements-orca-python-ray.txt
pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} pytest
Expand All @@ -33,7 +33,7 @@ runs:
HOROVOD_WITH_TENSORFLOW=1; HOROVOD_WITH_GLOO=1; pip install --no-cache-dir horovod==0.19.2
pip install ray==1.11.0
pip install --pre --upgrade bigdl-orca
pip install -i https://pypi.python.org/simple --pre --upgrade bigdl-orca
- name: Run Test
shell: bash
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ runs:
run: |
source activate py37
export SPARK_LOCAL_HOSTNAME=localhost
pip install --pre --upgrade bigdl-orca-spark3
pip install -i https://pypi.python.org/simple --pre --upgrade bigdl-orca-spark3
chmod a+x python/orca/dev/example/run-example-test-ray-horovod-tf.sh
python/orca/dev/example/run-example-test-ray-horovod-tf.sh
ray stop
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ runs:
run: |
source activate py37
export SPARK_LOCAL_HOSTNAME=localhost
pip install --pre --upgrade bigdl-orca-spark3
pip install -i https://pypi.python.org/simple --pre --upgrade bigdl-orca-spark3
chmod a+x python/orca/dev/example/run-example-test-ray.sh
python/orca/dev/example/run-example-test-ray.sh
source deactivate
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ runs:
run: |
source activate py37
export SPARK_LOCAL_HOSTNAME=localhost
pip install --pre --upgrade bigdl-orca-spark3
pip install -i https://pypi.org/simple --pre --upgrade bigdl-orca-spark3
chmod a+x python/orca/dev/test/run-pytests-horovod-tf.sh
python/orca/dev/test/run-pytests-horovod-tf.sh
ray stop
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@ runs:
export SPARK_LOCAL_HOSTNAME=localhost
pip install tensorflow==2.8.0 ray[default]==1.9.2
./python/dllib/dev/release/release.sh linux default false false
pip install ./python/dllib/src/dist/bigdl_dllib*-py3-none-manylinux1_x86_64.whl
pip install -i https://pypi.python.org/simple ./python/dllib/src/dist/bigdl_dllib*-py3-none-manylinux1_x86_64.whl
#build orca whl without make_dist
./python/orca/dev/release/release.sh linux default true false
pip install ./python/orca/src/dist/bigdl_orca*-py3-none-manylinux1_x86_64.whl
pip install -i https://pypi.python.org/simple ./python/orca/src/dist/bigdl_orca*-py3-none-manylinux1_x86_64.whl
chmod a+x python/orca/dev/test/run-pytests-tf2-rayonspark.sh
python/orca/dev/test/run-pytests-tf2-rayonspark.sh
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@ runs:
export SPARK_LOCAL_HOSTNAME=localhost
pip install tensorflow==2.8.0 ray[default]==1.9.2
./python/dllib/dev/release/release.sh linux default false false
pip install ./python/dllib/src/dist/bigdl_dllib*-py3-none-manylinux1_x86_64.whl
pip install -i https://pypi.python.org/simple ./python/dllib/src/dist/bigdl_dllib*-py3-none-manylinux1_x86_64.whl
#build orca whl without make_dist
./python/orca/dev/release/release.sh linux default true false
pip install ./python/orca/src/dist/bigdl_orca*-py3-none-manylinux1_x86_64.whl
pip install -i https://pypi.python.org/simple ./python/orca/src/dist/bigdl_orca*-py3-none-manylinux1_x86_64.whl
chmod a+x python/orca/dev/test/run-pytests-tf2-rayonspark.sh
python/orca/dev/test/run-pytests-tf2-rayonspark.sh
Expand Down
2 changes: 1 addition & 1 deletion .github/actions/orca/orca-tutorial-xshards/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ runs:
pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} jep==3.9.0
pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} cloudpickle
wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requrirements/requirements-orca-python-ray.txt -O ${{ github.workspace }}/requirements-orca-python-ray.txt
wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requirements/requirements-orca-python-ray.txt -O ${{ github.workspace }}/requirements-orca-python-ray.txt
pip uninstall -r ${{ github.workspace }}/requirements-orca-python-ray.txt -y
pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} -r ${{ github.workspace }}/requirements-orca-python-ray.txt
export SPARK_LOCAL_HOSTNAME=localhost
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ runs:
pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3
pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} cmake mxnet==1.6.0
wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requrirements/requirements-horovod-tf2.txt -O ${{ github.workspace }}/requirements-horovod-tf2.txt
wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requirements/requirements-horovod-tf2.txt -O ${{ github.workspace }}/requirements-horovod-tf2.txt
pip uninstall -r ${{ github.workspace }}/requirements-horovod-tf2.txt -y
pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} -r ${{ github.workspace }}/requirements-horovod-tf2.txt
pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} numpy==1.19.5
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ runs:
source activate py37
pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3
wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requrirements/requirements-orca-python-jep.txt -O ${{ github.workspace }}/requirements-orca-python-jep.txt
wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requirements/requirements-orca-python-jep.txt -O ${{ github.workspace }}/requirements-orca-python-jep.txt
pip uninstall -r ${{ github.workspace }}/requirements-orca-python-jep.txt -y
pip install -i https://pypi.org/simple --pre --upgrade bigdl-orca
pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} -r ${{ github.workspace }}/requirements-orca-python-jep.txt
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ runs:
pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} cmake
HOROVOD_WITH_PYTORCH=1; HOROVOD_WITH_GLOO=1; pip install --no-cache-dir horovod==0.19.2
wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requrirements/requirements-orca-horovod-pytorch.txt -O ${{ github.workspace }}/requirements-orca-horovod-pytorch.txt
wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requirements/requirements-orca-horovod-pytorch.txt -O ${{ github.workspace }}/requirements-orca-horovod-pytorch.txt
pip uninstall -r ${{ github.workspace }}/requirements-orca-horovod-pytorch.txt -y
pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} -r ${{ github.workspace }}/requirements-orca-horovod-pytorch.txt
pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} ray[default]==1.9.1
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ runs:
source activate py37
pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3
wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requrirements/requirements-orca-python-ray.txt -O ${{ github.workspace }}/requirements-orca-python-ray.txt
wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requirements/requirements-orca-python-ray.txt -O ${{ github.workspace }}/requirements-orca-python-ray.txt
pip uninstall -r ${{ github.workspace }}/requirements-orca-python-ray.txt -y
pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} -r ${{ github.workspace }}/requirements-orca-python-ray.txt
pip uninstall -y opencv-python-headless
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ runs:
pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3
wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requrirements/requirements-orca-python-ray.txt -O ${{ github.workspace }}/requirements-orca-python-ray.txt
wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requirements/requirements-orca-python-ray.txt -O ${{ github.workspace }}/requirements-orca-python-ray.txt
pip uninstall -r ${{ github.workspace }}/requirements-orca-python-ray.txt -y
pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} -r ${{ github.workspace }}/requirements-orca-python-ray.txt
pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} cmake==3.16.3
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ runs:
pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3 pyspark
pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} pytest
wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requrirements/requirements-ppml-python-ut-spark3.txt -O ${{ github.workspace }}/requirements-ppml-python-ut-spark3.txt
wget https://raw.githubusercontent.com/analytics-zoo/gha-cicd-env/main/python-requirements/requirements-ppml-python-ut-spark3.txt -O ${{ github.workspace }}/requirements-ppml-python-ut-spark3.txt
pip uninstall -r ${{ github.workspace }}/requirements-ppml-python-ut-spark3.txt -y
pip install -i ${GONDOLIN_PIP_MIRROR} --trusted-host ${GONDOLIN_TRUSTED_HOST} -r ${{ github.workspace }}/requirements-ppml-python-ut-spark3.txt
Expand Down
Loading

0 comments on commit f2a0325

Please sign in to comment.