DB Refresh (Objects API) #27933
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: DB Refresh (Objects API) | |
| on: | |
| schedule: | |
| # - cron: '0 5 * * *' # every day 5am GMT (12/1am EST/EDT) | |
| - cron: '*/20 * * * *' # every 20 minutes | |
| # - cron: '0 * * * *' # every hour on the hour | |
| workflow_dispatch: | |
| inputs: | |
| since: | |
| # A timestamp by which new data should be fetched. If not present, will look up the last time the DB was | |
| # refreshed and fetch new data from that time. Valid formats: | |
| # - Simple date timestamp, YYYY-MM-DD, e.g. 2022-02-22. | |
| # - ISO 8601 datetime with timezone offset, YYYY-MM-DDTHH:MM:SS.SSSSSS+HH:MM, e.g. 2022-02-22T22:22:22.222222+00:00. | |
| description: 'Optional: YYYY-MM-DD or YYYY-MM-DDTHH:MM:SS.SSSSSS+HH:MM' | |
| required: false | |
| type: string | |
| cset-ids: | |
| # Space-delimited list of integers representing codeset IDs | |
| description: 'Optional: Space-delimited list of integers representing codeset IDs' | |
| required: false | |
| type: string | |
| repository_dispatch: | |
| types: [refresh-db] | |
| jobs: | |
| refresh-db: | |
| # runs-on: ubuntu-18.04 | |
| runs-on: ubuntu-latest | |
| steps: | |
| # Set up | |
| - name: Checkout repository and submodules | |
| uses: actions/checkout@v2 | |
| - name: Set up Python version | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.10' | |
| - name: Print commit hash & branch for rollbacks & troubleshooting | |
| run: | | |
| echo "Commit hash: ${{ github.sha }}" | |
| echo "Branch: ${{ github.ref }}" | |
| - name: 'Create env file' | |
| run: | | |
| mkdir env | |
| echo "${{ secrets.ENV_FILE }}" > env/.env | |
| - name: Create and start virtual environment | |
| run: | | |
| python3 -m venv venv | |
| source venv/bin/activate | |
| - name: Install dependencies | |
| run: | | |
| # Reactivate install upgrades of pip, wheel, &/or setuptools if needed | |
| # python -m pip install --upgrade pip | |
| # pip install --upgrade wheel | |
| # pip install --upgrade setuptools | |
| # VS Hub SDK w/ OAuth disabled: https://github.com/jhu-bids/TermHub/issues/863 | |
| # TOKEN=$(grep '^PALANTIR_ENCLAVE_AUTHENTICATION_BEARER_TOKEN=' env/.env | cut -d'=' -f2) | |
| # python3 -m pip install vshub_sdk --upgrade --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.artifacts.main.repository.9bc9cc56-4b8c-4560-9bfb-fba8ade55246/contents/release/pypi/simple" --extra-index-url "https://:${TOKEN}@unite.nih.gov/artifacts/api/repositories/ri.foundry-sdk-asset-bundle.main.artifacts.repository/contents/release/pypi/simple" | |
| pip install -r requirements.txt | |
| # Refresh | |
| - name: Refresh db | |
| run: | | |
| if [ -n "${{ github.event.inputs.cset-ids }}" ] && [ -n "${{ github.event.inputs.since }}" ]; then | |
| echo "Error: Both 'cset-ids' and 'since' were provided. These inputs are mutually exclusive." | |
| exit 1 | |
| elif [ -n "${{ github.event.inputs.cset-ids }}" ]; then | |
| python backend/db/refresh.py --cset-ids ${{ github.event.inputs.cset-ids }} | |
| elif [ -n "${{ github.event.inputs.since }}" ]; then | |
| python backend/db/refresh.py --since ${{ github.event.inputs.since }} | |
| else | |
| python backend/db/refresh.py | |
| fi | |
| - name: Commit changes to documentation | |
| uses: stefanzweifel/git-auto-commit-action@v4 | |
| with: | |
| commit_message: Updating documentation in docs/backend/db/analysis.md | |
| # fetch-missing-csets: Deactivated because various tweaks and default re-refresh buffer solve this problem, and | |
| # also test-missing-csets is there as a redundancy. Original context: | |
| # https://github.com/jhu-bids/TermHub/issues/573 https://github.com/jhu-bids/TermHub/issues/574 | |
| # - name: Fetch missing csets | |
| # run: make fetch-missing-csets | |
| # Test | |
| # todo: add more tests, ideally | |
| - name: Test missing csets | |
| run: make test-missing-csets |