Enforce that you first deploy to PyPi, then create the release and

docker images. This will avoid any errors on PyPi causing version mismatch with our releases. Also, I am going to try using caching to save time on deploying docker images
panda-re · Dec 5, 2024 · 13ac64e · 13ac64e
1 parent f55f50e
commit 13ac64e
Showing 3 changed files with 102 additions and 82 deletions.
diff --git a/.github/workflows/publish_docker.yml b/.github/workflows/publish_docker.yml
@@ -1,6 +1,7 @@
 name: Build and Publish Docker Container and Pypanda Docs # Only for main panda-re repo, not forks
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - dev
@@ -26,8 +27,8 @@ jobs:
           use_api: true
 
   build_release_assets:
+    needs: [ create_release ]
     if: github.repository  == 'panda-re/panda' && github.ref == 'refs/heads/dev'
-    needs: create_release
     runs-on: panda-arc
     strategy:
       matrix:
@@ -48,6 +49,67 @@ jobs:
         working-directory: panda/debian
         run: ./setup.sh Ubuntu ${{ matrix.ubuntu_version }}
 
+      - name: Store the PyPanda distribution packages
+        if: ${{ matrix.ubuntu_version == env.PANDA_CONTAINER_UBUNTU_VERSION }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: pypanda
+          path: panda/debian/pandare*.whl
+          if-no-files-found: error
+          retention-days: 7
+
+      - name: Store the Panda Debian packages
+        uses: actions/upload-artifact@v4
+        with:
+          name: panda-debian
+          path: panda/debian/pandare*.deb
+          if-no-files-found: error
+          retention-days: 7
+
+  publish_to_pypi:
+    name: Publish Python 🐍 distribution 📦 to PyPI
+    if: github.repository == 'panda-re/panda' && github.ref == 'refs/heads/dev'
+    needs: [ build_release_assets ]
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/pandare  # Replace <package-name> with your PyPI project name
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+    steps:
+    - name: Download PyPanda
+      uses: actions/download-artifact@v4
+      with:
+        name: pypanda
+        path: dist/
+
+    - name: Publish distribution 📦 to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        verbose: true
+
+  upload_release_assets:
+    if: github.repository  == 'panda-re/panda' && github.ref == 'refs/heads/dev'
+    needs: [ create_release, publish_to_pypi ]
+    runs-on: panda-arc
+    steps:
+      - name: Check out
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Download PyPanda
+        uses: actions/download-artifact@v4
+        with:
+          name: pypanda
+          path: panda/debian
+
+      - name: Download Panda Debian Packages
+        uses: actions/download-artifact@v4
+        with:
+          name: panda-debian
+          path: panda/debian
+
       - name: Upload wheel and debian packages to release
         uses: softprops/action-gh-release@v2
         env:
@@ -57,33 +119,27 @@ jobs:
           files: |
             panda/debian/pandare*.whl
             panda/debian/pandare*.deb
-
-      - name: Store the PyPanda distribution packages
-        if: ${{ matrix.ubuntu_version == env.PANDA_CONTAINER_UBUNTU_VERSION }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: pypanda
-          path: panda/debian/pandare*.whl
-          if-no-files-found: error
-
-      - name: 'Login to Docker Registry'
-        if: ${{ matrix.ubuntu_version == env.PANDA_CONTAINER_UBUNTU_VERSION }}
+      
+      - name: Login to Docker Registry
         uses: docker/login-action@v3
         with:
           username: pandare
           password: ${{secrets.pandare_dockerhub}}
-
-
-      #- name: 'Login to GHCR Registry'
-      #  if: ${{ matrix.ubuntu_version == env.PANDA_CONTAINER_UBUNTU_VERSION }}
-      #  uses: docker/login-action@v3
-      #  with:
-      #    registry: ghcr.io
-      #    username: ${{ github.repository_owner }}
-      #    password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build pandadev:latest    
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          context: ${{ github.workspace }}
+          tags: |
+            pandare/pandadev:${{ github.sha }}
+            pandare/pandadev:${{ needs.create_release.outputs.v-version }}
+            pandare/pandadev:latest
+          target: developer
+          cache-from: type=registry,ref=pandare/pandadev:cache
+          cache-to: type=registry,ref=pandare/pandadev:cache,mode=max
 
       - name: Build panda:latest
-        if: ${{ matrix.ubuntu_version == env.PANDA_CONTAINER_UBUNTU_VERSION }}
         uses: docker/build-push-action@v5
         with:
           push: true
@@ -92,89 +148,50 @@ jobs:
             pandare/panda:${{ github.sha }}
             pandare/panda:${{ needs.create_release.outputs.v-version }}
             pandare/panda:latest
-        #    ghcr.io/pandare/panda:${{ github.sha }}
-        #    ghcr.io/pandare/panda:${{ needs.create_release.outputs.v-version }}
-        #    ghcr.io/pandare/panda:latest
           target: panda
-      - name: Build pandadev:latest
-        if: ${{ matrix.ubuntu_version == env.PANDA_CONTAINER_UBUNTU_VERSION }}
-        uses: docker/build-push-action@v5
-        with:
-          push: true
-          context: ${{ github.workspace }}
-          tags: |
-            pandare/pandadev:${{ github.sha }}
-            pandare/pandadev:${{ needs.create_release.outputs.v-version }}
-            pandare/pandadev:latest
-         #   ghcr.io/pandare/pandadev:${{ github.sha }}
-         #   ghcr.io/pandare/pandadev:${{ needs.create_release.outputs.v-version }}
-         #   ghcr.io/pandare/pandadev:latest
-          target: developer
+          cache-from: type=registry,ref=pandare/pandadev:cache
+          cache-to: type=registry,ref=pandare/panda:cache,mode=max
+
       - name: Checkout docs and reset
-        if: ${{ matrix.ubuntu_version == env.PANDA_CONTAINER_UBUNTU_VERSION }}
         run: rm -rf "${GITHUB_WORKSPACE}/auto_pydoc";
-             git clone https://panda-jenkins-ci:${{ secrets.PANDABOT_GITHUB_API }}@github.com/panda-re/panda-re.github.io.git --branch=master ${GITHUB_WORKSPACE}/auto_pydoc/pandare
+                 git clone https://panda-jenkins-ci:${{ secrets.PANDABOT_GITHUB_API }}@github.com/panda-re/panda-re.github.io.git --branch=master ${GITHUB_WORKSPACE}/auto_pydoc/pandare
+
       - name: Update PYPANDA docs in container
-        if: ${{ matrix.ubuntu_version == env.PANDA_CONTAINER_UBUNTU_VERSION }}
         run: docker run --rm -v ${GITHUB_WORKSPACE}/auto_pydoc:/out pandare/pandadev:latest /bin/sh -c "pip3 install pdoc3==0.10.0; cd /panda/panda/python/core; pdoc3 --html --template-dir=../docs/template --force -o /out/${GITHUB_REF##*/} pandare; chmod -R 777 /out/"
+
         # will put docs in workspace/auto_pydoc/dev/pandare and/or workspace/auto_pydoc/stable/pandare
-        # we want to copy auto_pydoc/dev/pandare to auto_pydoc/pandare/ and /auto_pydoc/stable/pandare to /auto_pydoc/pandare/stable
+        # we want to copy auto_pydoc/dev/pandare to auto_pydoc/pandare/ and /auto_pydoc/stable/pandare to /auto_pydoc/pandare/stable    
         #
         # This is a bit complicated, sorry. We want to keep pandare/{CNAME,.git/} and nothing else
         # then we copy in the new files (and merge doc-search.html and index.js with dev/pandare/
       - name: Push PYPANDA docs to GitHub Pages if docs changed
-        if: ${{ matrix.ubuntu_version == env.PANDA_CONTAINER_UBUNTU_VERSION }}
         run: cd "${GITHUB_WORKSPACE}/auto_pydoc" &&
-             mv pandare/CNAME dev &&
-             rm -rf pandare/* &&
-             mv dev/pandare/* pandare &&
-             rmdir dev/pandare &&
-             mv dev/* pandare/ &&
-             cd pandare &&
-             git config --global user.email "panda-ci@panda-re.mit.edu" &&
-             git config --global user.name "PANDA Bot" &&
-             git add . &&
-             git commit -m "Documentation update for PANDA commit ${{ github.sha  }} branch dev" &&
-             git push || true
-
-  publish-to-pypi:
-    name: Publish Python 🐍 distribution 📦 to PyPI
-    if: github.repository  == 'panda-re/panda' && github.ref == 'refs/heads/dev'
-    needs:
-    - build_release_assets
-    runs-on: ubuntu-latest
-    environment:
-      name: pypi
-      url: https://pypi.org/p/pandare  # Replace <package-name> with your PyPI project name
-    permissions:
-      id-token: write  # IMPORTANT: mandatory for trusted publishing
+          mv pandare/CNAME dev &&
+          rm -rf pandare/* &&
+          mv dev/pandare/* pandare &&
+          rmdir dev/pandare &&
+          mv dev/* pandare/ &&
+          cd pandare &&
+          git config --global user.email "panda-ci@panda-re.mit.edu" &&
+          git config --global user.name "PANDA Bot" &&
+          git add . &&
+          git commit -m "Documentation update for PANDA commit ${{ github.sha  }} branch dev" &&
+          git push || true
 
-    steps:
-    - name: Download all the dists
-      uses: actions/download-artifact@v4
-      with:
-        name: pypanda
-        path: dist/
-
-    - name: Publish distribution 📦 to PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1
-      with:
-        verbose: true
-
   build_stable:
     if: github.repository  == 'panda-re/panda' && github.ref == 'refs/heads/stable'
     runs-on: panda-arc
     steps:
     - name: Checkout PANDA at current commit
       uses: actions/checkout@v4
 
-    - name: 'Login to Docker Registry'
+    - name: Login to Docker Registry
       uses: docker/login-action@v3
       with:
         username: pandare
         password: ${{secrets.pandare_dockerhub}}
 
-    - name: Build Bionic container
+    - name: Build Panda stable container
       # Push both dev and regular container
       uses: docker/build-push-action@v5
       with:
@@ -191,6 +208,7 @@ jobs:
 
     - name: Update PYPANDA docs in container
       run: docker run --rm -v ${GITHUB_WORKSPACE}/auto_pydoc/pandare:/out pandare/pandadev:latest /bin/sh -c "pip3 install pdoc3==0.10.0; cd /panda/panda/python/core; pdoc3 --html --template-dir=../docs/template --force -o /out/${GITHUB_REF##*/} pandare; chmod -R 777 /out/"
+
       # will put docs in workspace/auto_pydoc/dev/pandare and/or workspace/auto_pydoc/stable/pandare
       # we want to copy /auto_pydoc/dev/pandare to /auto_doc and /auto_pydoc/stable/pandare to /auto_pydoc/stable
       #

diff --git a/panda/debian/.gitignore b/panda/debian/.gitignore
@@ -1 +1,2 @@
-panda.deb
+*.deb
+*.whl
diff --git a/panda/python/core/.gitignore b/panda/python/core/.gitignore
@@ -8,3 +8,4 @@ data
 a
 __pycache__
 *.egg-info
+.eggs/
-Original file line number
+Diff line change
@@ @@ -8,3 +8,4 @@ data @@
     a
     __pycache__
     *.egg-info
+    .eggs/