Don't hard-code yelpcorp PyPI registry

Internal ticket: CORESERV-12777 Same idea as Yelp/Tron#921
Yelp · Jul 31, 2023 · a0a7514 · a0a7514
1 parent 0c532e6
commit a0a7514
Show file tree

Hide file tree

Showing 24 changed files with 33 additions and 64 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -20,7 +20,6 @@ jobs:
           - py37-linux,docs,mypy,tests
           - general_itests
     env:
-      PIP_INDEX_URL: https://pypi.python.org/simple
       DOCKER_REGISTRY: ""
     steps:
       - uses: actions/checkout@v2
@@ -29,7 +28,7 @@ jobs:
           python-version: 3.7
       - run: python -m pip install --upgrade pip
       - run: pip install coveralls tox==3.2 tox-pip-extensions==1.3.0 ephemeral-port-reserve
-      - run: tox -i https://pypi.python.org/simple -e ${{ matrix.toxenv }}
+      - run: tox -e ${{ matrix.toxenv }}
   k8s_itests:
     runs-on: ubuntu-20.04
     env:

diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml
@@ -16,7 +16,6 @@ jobs:
           - py37-linux,docs,mypy,tests
           - general_itests
     env:
-      PIP_INDEX_URL: https://pypi.python.org/simple
       DOCKER_REGISTRY: ""
     steps:
       - uses: actions/checkout@v2
@@ -25,13 +24,12 @@ jobs:
           python-version: 3.7
       - run: python -m pip install --upgrade pip
       - run: pip install coveralls tox==3.2 tox-pip-extensions==1.3.0 ephemeral-port-reserve
-      - run: tox -i https://pypi.python.org/simple -e ${{ matrix.toxenv }}
+      - run: tox -e ${{ matrix.toxenv }}
   pypi:
     # lets run tests before we push anything to pypi, much like we do internally
     needs: tox
     runs-on: ubuntu-20.04
     env:
-      PIP_INDEX_URL: https://pypi.python.org/simple
       DOCKER_REGISTRY: ""
     steps:
       - uses: actions/checkout@v2

diff --git a/Makefile b/Makefile
@@ -22,42 +22,41 @@ else
 endif
 
 ifeq ($(PAASTA_ENV),YELP)
-	export PIP_INDEX_URL ?= https://pypi.yelpcorp.com/simple
 	export DOCKER_REGISTRY ?= docker-dev.yelpcorp.com/
 else
-	export PIP_INDEX_URL ?= https://pypi.python.org/simple
 	export DOCKER_REGISTRY ?= ""
+	export INDEX_URL_BUILD_ARG ?= PIP_INDEX_URL
 endif
 
 .PHONY: all docs test itest k8s_itests quick-test
 
 dev: .paasta/bin/activate
-	.paasta/bin/tox -i $(PIP_INDEX_URL)
+	.paasta/bin/tox
 
 docs: .paasta/bin/activate
-	.paasta/bin/tox -i $(PIP_INDEX_URL) -e docs
+	.paasta/bin/tox -e docs
 
 test: .paasta/bin/activate
 	if [ "$(PAASTA_ENV)" != "YELP" ]; then \
-		.paasta/bin/tox -i $(PIP_INDEX_URL) -e tests; \
+		.paasta/bin/tox -e tests; \
 	else \
-		.paasta/bin/tox -i $(PIP_INDEX_URL) -e tests-yelpy; \
+		.paasta/bin/tox -e tests-yelpy; \
 	fi
 
 test-yelpy: .paasta/bin/activate
-	.paasta/bin/tox -i $(PIP_INDEX_URL) -e tests-yelpy
+	.paasta/bin/tox -e tests-yelpy
 
 test-not-yelpy: .paasta/bin/activate
-	.paasta/bin/tox -i $(PIP_INDEX_URL) -e tests
+	.paasta/bin/tox -e tests
 
 quick-test: .tox/py37-linux
 	TZ=UTC .tox/py37-linux/bin/py.test --last-failed -x -- tests
 
 .tox/py37-linux: .paasta/bin/activate
-	.paasta/bin/tox -i $(PIP_INDEX_URL)
+	.paasta/bin/tox
 
 dev-api: .tox/py37-linux
-	.paasta/bin/tox -i $(PIP_INDEX_URL) -e dev-api
+	.paasta/bin/tox -e dev-api
 
 .paasta/bin/activate: requirements.txt requirements-dev.txt
 	test -d .paasta/bin/activate || virtualenv -p python3.7 .paasta
@@ -69,7 +68,7 @@ dev-api: .tox/py37-linux
 	touch .paasta/bin/activate
 
 itest: test .paasta/bin/activate
-	.paasta/bin/tox -i $(PIP_INDEX_URL) -e general_itests
+	.paasta/bin/tox -e general_itests
 
 itest_%:
 	# See the makefile in yelp_package/Makefile for packaging stuff
@@ -150,7 +149,7 @@ generate_deployments_for_service: | soa_config_playground .tox/py37-linux
 
 .PHONY: playground-api
 playground-api: .tox/py37-linux | soa_config_playground
-	.paasta/bin/tox -i $(PIP_INDEX_URL) -e playground-api
+	.paasta/bin/tox -e playground-api
 
 .PHONY: setup-kubernetes-job
 setup-kubernetes-job: k8s_fake_cluster generate_deployments_for_service

diff --git a/debian/rules b/debian/rules
@@ -1,8 +1,6 @@
 #!/usr/bin/make -f
 # -*- makefile -*-
 
-PIP_INDEX_URL ?= https://pypi.yelpcorp.com/simple
-
 %:
 	dh $@ --with python-virtualenv
 
@@ -21,9 +19,8 @@ PACKAGE=$(shell dh_listpackages)
 DH_VIRTUALENV_INSTALL_ROOT=/opt/venvs
 DH_VENV_DIR=debian/$(PACKAGE)$(DH_VIRTUALENV_INSTALL_ROOT)/$(PACKAGE)
 override_dh_virtualenv:
-	dh_virtualenv -i $(PIP_INDEX_URL) \
+	dh_virtualenv \
 		--python=/usr/bin/python3.7 \
 		--preinstall no-manylinux1 \
-		--preinstall=-rrequirements-bootstrap.txt \
-		--pip-tool pip-custom-platform
+		--preinstall=-rrequirements-bootstrap.txt
 	cp yelp_package/gopath/paasta_go $(DH_VENV_DIR)/bin/paasta_go
diff --git a/general_itests/fake_simple_service/Dockerfile b/general_itests/fake_simple_service/Dockerfile
@@ -14,6 +14,3 @@
 
 ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/
 FROM ${DOCKER_REGISTRY}ubuntu:xenial
-
-ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple
-ENV PIP_INDEX_URL=${PIP_INDEX_URL}
diff --git a/general_itests/fake_simple_service/Makefile b/general_itests/fake_simple_service/Makefile
@@ -16,13 +16,11 @@ DOCKER_TAG ?= fake_simple_service-$(USER)-dev
 
 ifeq ($(findstring .yelpcorp.com,$(shell hostname -f)), .yelpcorp.com)
 	DOCKER_REGISTRY ?= docker-dev.yelpcorp.com/
-	PIP_INDEX_URL ?= https://pypi.yelpcorp.com/simple
 else
 	DOCKER_REGISTRY ?= ""
-	PIP_INDEX_URL ?= https://pypi.python.org/simple
 endif
 
 .PHONY: cook-image
 
 cook-image:
-	docker build --build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) --build-arg DOCKER_REGISTRY=$(DOCKER_REGISTRY) -t $(DOCKER_TAG) .
+	docker build --build-arg DOCKER_REGISTRY=$(DOCKER_REGISTRY) -t $(DOCKER_TAG) .
diff --git a/requirements-bootstrap.txt b/requirements-bootstrap.txt
@@ -1,5 +1,4 @@
 pip==18.1
-pip-custom-platform==0.5.0
 setuptools==39.0.1
 venv-update==3.2.4
 wheel==0.32.3
diff --git a/tox.ini b/tox.ini
@@ -1,19 +1,14 @@
 [tox]
 skipsdist=True
 envlist=py37-linux
-tox_pip_extensions_ext_pip_custom_platform = true
 tox_pip_extensions_ext_venv_update = true
 docker_compose_version = 1.26.2
 
 [testenv]
-# The Makefile and override the indexserver to the public one when
-# running outside of Yelp.
-indexserver = https://pypi.yelpcorp.com/simple
 basepython = python3.7
 passenv = SSH_AUTH_SOCK
 setenv =
     TZ = UTC
-    PIP_INDEX_URL = {env:PIP_INDEX_URL:https://pypi.yelpcorp.com/simple}
 deps =
     --requirement={toxinidir}/requirements.txt
     --requirement={toxinidir}/requirements-dev.txt
@@ -24,7 +19,7 @@ commands =
     # that said, most of the time people will run make test which will use tox to install these in a
     # faster way (using venv-update) - so this is really just here for anyone that like to just invoke
     # `tox` directly and with no explicit env
-    -pip-custom-platform install -i https://pypi.yelpcorp.com/simple -r yelp_package/extra_requirements_yelp.txt
+    -pip install -r yelp_package/extra_requirements_yelp.txt
 
 [testenv:dev-api]
 envdir = .tox/py37-linux/
@@ -105,24 +100,25 @@ passenv =
     DOCKER_TLS_VERIFY
     DOCKER_HOST
     DOCKER_CERT_PATH
+    INDEX_URL_BUILD_ARG
 changedir=k8s_itests/
 commands =
     # Build /etc/paasta used by docker-compose
 	{toxinidir}/k8s_itests/scripts/setup.sh
     # Run paasta-tools k8s_itests in docker
     docker-compose down
-    docker-compose --verbose build --build-arg DOCKER_REGISTRY={env:DOCKER_REGISTRY:docker-dev.yelpcorp.com/} --build-arg PIP_INDEX_URL={env:PIP_INDEX_URL:https://pypi.yelpcorp.com/simple}
+    docker-compose --verbose build --build-arg DOCKER_REGISTRY={env:DOCKER_REGISTRY:docker-dev.yelpcorp.com/} --build-arg {env:INDEX_URL_BUILD_ARG:UNUSED}=https://pypi.org/simple
     docker-compose up \
         --abort-on-container-exit
 
 [testenv:example_cluster]
 changedir=example_cluster/
-passenv = DOCKER_TLS_VERIFY DOCKER_HOST DOCKER_CERT_PATH
+passenv = DOCKER_TLS_VERIFY DOCKER_HOST DOCKER_CERT_PATH INDEX_URL_BUILD_ARG
 deps =
     docker-compose=={[tox]docker_compose_version}
 commands =
     docker-compose down
-    docker-compose --verbose build --build-arg DOCKER_REGISTRY={env:DOCKER_REGISTRY:docker-dev.yelpcorp.com/} --build-arg PIP_INDEX_URL={env:PIP_INDEX_URL:https://pypi.yelpcorp.com/simple}
+    docker-compose --verbose build --build-arg DOCKER_REGISTRY={env:DOCKER_REGISTRY:docker-dev.yelpcorp.com/} --build-arg {env:INDEX_URL_BUILD_ARG:UNUSED}=https://pypi.org/simple
     # Fire up the marathon cluster in background
     # Run the paastatools container in foreground to catch the output
     # the `docker-compose run` vs `docker-compose up` is important here, as docker-compose run will

diff --git a/yelp_package/Makefile b/yelp_package/Makefile
@@ -19,7 +19,7 @@ SHELL=/bin/bash
 
 UID:=`id -u`
 GID:=`id -g`
-DOCKER_RUN=docker run -t -v $(CURDIR)/../:/work:rw -e PIP_INDEX_URL=$(PIP_INDEX_URL) yelp/paastatools_$*_container
+DOCKER_RUN=docker run -t -v $(CURDIR)/../:/work:rw yelp/paastatools_$*_container
 
 NOOP = true
 ifeq ($(PAASTA_ENV),YELP)
@@ -36,7 +36,8 @@ build_%_docker:
 	[ -d ../dist ] || mkdir ../dist
 	docker pull "yelp/paastatools_$*_container" || true
 	cd dockerfiles/$*/ && docker build --build-arg DOCKER_REGISTRY=$(DOCKER_REGISTRY) \
-		--build-arg PIP_INDEX_URL=$(PIP_INDEX_URL) -t "yelp/paastatools_$*_container" .
+	$(if $(filter-out $(PAASTA_ENV),YELP), --build-arg PIP_INDEX_URL=https://pypi.org/simple,) \
+		-t "yelp/paastatools_$*_container" .
 
 .SECONDEXPANSION:
 itest_%: package_$$*

diff --git a/yelp_package/dockerfiles/bionic/Dockerfile b/yelp_package/dockerfiles/bionic/Dockerfile
@@ -15,7 +15,7 @@
 ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/
 FROM ${DOCKER_REGISTRY}ubuntu:bionic
 
-ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple
+ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/bionic/simple
 ENV PIP_INDEX_URL=$PIP_INDEX_URL
 RUN rm /etc/dpkg/dpkg.cfg.d/excludes
 RUN apt-get update && apt-get install -yq gnupg2

diff --git a/yelp_package/dockerfiles/gitremote/Dockerfile b/yelp_package/dockerfiles/gitremote/Dockerfile
@@ -1,8 +1,6 @@
 ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/
 FROM ${DOCKER_REGISTRY}ubuntu:xenial
 
-ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple
-ENV PIP_INDEX_URL=$PIP_INDEX_URL
 RUN apt-get update > /dev/null && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y \
         git \

diff --git a/yelp_package/dockerfiles/hacheck-sidecar/Dockerfile b/yelp_package/dockerfiles/hacheck-sidecar/Dockerfile
@@ -1,9 +1,6 @@
 FROM docker-dev.yelpcorp.com/bionic_yelp
 ARG HACHECK_VERSION=0.18.2-yelp1
 
-ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple
-ENV PIP_INDEX_URL=$PIP_INDEX_URL
-
 RUN apt-get update && \
     apt-get install -y hacheck=${HACHECK_VERSION} paasta-tools python3-distutils && \
     mkdir -p /etc/paasta

diff --git a/yelp_package/dockerfiles/itest/api/Dockerfile b/yelp_package/dockerfiles/itest/api/Dockerfile
@@ -15,7 +15,7 @@
 ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/
 FROM ${DOCKER_REGISTRY}ubuntu:bionic
 
-ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple
+ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/bionic/simple
 ENV PIP_INDEX_URL=$PIP_INDEX_URL
 
 RUN apt-get update > /dev/null && \

diff --git a/yelp_package/dockerfiles/itest/hacheck/Dockerfile b/yelp_package/dockerfiles/itest/hacheck/Dockerfile
@@ -15,7 +15,7 @@
 ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/
 FROM ${DOCKER_REGISTRY}ubuntu:bionic
 
-ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple
+ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/bionic/simple
 ENV PIP_INDEX_URL=$PIP_INDEX_URL
 
 RUN apt-get update > /dev/null && \

diff --git a/yelp_package/dockerfiles/itest/httpdrain/Dockerfile b/yelp_package/dockerfiles/itest/httpdrain/Dockerfile
@@ -1,7 +1,7 @@
 ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/
 FROM ${DOCKER_REGISTRY}ubuntu:bionic
 
-ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple
+ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/bionic/simple
 ENV PIP_INDEX_URL=$PIP_INDEX_URL
 
 RUN apt-get update > /dev/null && \

diff --git a/yelp_package/dockerfiles/itest/k8s/Dockerfile b/yelp_package/dockerfiles/itest/k8s/Dockerfile
@@ -15,7 +15,7 @@
 ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/
 FROM ${DOCKER_REGISTRY}ubuntu:bionic
 
-ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple
+ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/bionic/simple
 ENV PIP_INDEX_URL=$PIP_INDEX_URL
 
 # Need Python 3.7

diff --git a/yelp_package/dockerfiles/itest/marathon/Dockerfile b/yelp_package/dockerfiles/itest/marathon/Dockerfile
@@ -15,9 +15,6 @@
 ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/
 FROM ${DOCKER_REGISTRY}ubuntu:xenial
 
-ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple
-ENV PIP_INDEX_URL=$PIP_INDEX_URL
-
 RUN apt-get update > /dev/null && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         apt-transport-https \

diff --git a/yelp_package/dockerfiles/itest/mesos/Dockerfile b/yelp_package/dockerfiles/itest/mesos/Dockerfile
@@ -15,9 +15,6 @@
 ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/
 FROM ${DOCKER_REGISTRY}xenial_pkgbuild
 
-ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple
-ENV PIP_INDEX_URL=$PIP_INDEX_URL
-
 # Install packages to allow apt to use a repository over HTTPS
 # https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/#xenial-1604
 RUN apt-get update > /dev/null && \

diff --git a/yelp_package/dockerfiles/itest/zookeeper/Dockerfile b/yelp_package/dockerfiles/itest/zookeeper/Dockerfile
@@ -15,9 +15,6 @@
 ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/
 FROM ${DOCKER_REGISTRY}ubuntu:xenial
 
-ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple
-ENV PIP_INDEX_URL=$PIP_INDEX_URL
-
 RUN apt-get update > /dev/null && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
          zookeeper > /dev/null && \

diff --git a/yelp_package/dockerfiles/jammy/Dockerfile b/yelp_package/dockerfiles/jammy/Dockerfile
@@ -15,7 +15,7 @@
 ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/
 FROM ${DOCKER_REGISTRY}ubuntu:jammy
 
-ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple
+ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/jammy/simple
 ENV PIP_INDEX_URL=$PIP_INDEX_URL
 RUN rm /etc/dpkg/dpkg.cfg.d/excludes
 RUN apt-get update && apt-get install -yq gnupg2

diff --git a/yelp_package/dockerfiles/mesos-paasta/Dockerfile b/yelp_package/dockerfiles/mesos-paasta/Dockerfile
@@ -1,6 +1,6 @@
 FROM example_cluster_mesosbase
 
-ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple
+ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/xenial/simple
 ENV PIP_INDEX_URL=$PIP_INDEX_URL
 
 RUN apt-get update > /dev/null && \

diff --git a/yelp_package/dockerfiles/playground/Dockerfile b/yelp_package/dockerfiles/playground/Dockerfile
@@ -1,6 +1,6 @@
 FROM example_cluster_itest_xenial
 
-ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple
+ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/xenial/simple
 ENV PIP_INDEX_URL=$PIP_INDEX_URL
 
 RUN apt-get update > /dev/null && \

diff --git a/yelp_package/dockerfiles/xenial/Dockerfile b/yelp_package/dockerfiles/xenial/Dockerfile
@@ -15,7 +15,7 @@
 ARG DOCKER_REGISTRY=docker-dev.yelpcorp.com/
 FROM ${DOCKER_REGISTRY}xenial_pkgbuild
 
-ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/simple
+ARG PIP_INDEX_URL=https://pypi.yelpcorp.com/xenial/simple
 ENV PIP_INDEX_URL=$PIP_INDEX_URL
 
 RUN echo "deb http://repos.mesosphere.com/ubuntu xenial main" > /etc/apt/sources.list.d/mesosphere.list && \

diff --git a/yelp_package/extra_requirements_yelp.txt b/yelp_package/extra_requirements_yelp.txt
@@ -1,4 +1,3 @@
---index-url=https://pypi.yelpcorp.com/simple
 atomicfile==1.0
 cached-property==1.3.1
 cffi==1.15.0