build(taskfile): add updates from sec-parser

alphanome-ai · Oct 14, 2023 · 87fcc49 · 87fcc49
1 parent e40ab4a
commit 87fcc49
Show file tree

Hide file tree

Showing 2 changed files with 66 additions and 52 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -8,7 +8,7 @@ repos:
     hooks:
       - id: pre-commit-tasks
         name: Run pre-commit tasks
-        entry: task pre-commit
+        entry: task p
         language: system
         pass_filenames: false
         stages: [commit]
diff --git a/Taskfile.yml b/Taskfile.yml
@@ -2,80 +2,94 @@ version: '3'
 
 tasks:
 
-#####################################
-### Composite Tasks and Shortcuts ###
-#####################################
+##############################################
+### Developer Tools: Application Launchers ###
+##############################################
 
-  pre-commit: 
-    desc: Composite task to run all quick checks.
+  dash:
+    desc: Launch a local debugging dashboard in your browser.
     cmds:
-      - task unit
-      - task lint-fix
+      # Using PYTHONPATH to allow Streamlit to monitor file changes. Learn more: https://docs.streamlit.io/knowledge-base/using-streamlit/streamlit-watch-changes-other-modules-importing-app
+      - export PYTHONPATH=$PYTHONPATH:$(pwd)/sec_ai ENVIRONMENT=dev && poetry run streamlit run dev_utils/debug_dashboard/app.py --server.runOnSave=true
 
-  p:
-    desc: Shortcut for `task pre-commit`.
-    deps:
-      - pre-commit
+  unit-watch:
+    desc: "Launch unit tests and monitor for changes, rerunning tests immediately when files are modified."
+    cmds:
+      # Recommended coverage viewer in VSCode: https://marketplace.visualstudio.com/items?itemName=ryanluker.vscode-coverage-gutters
+      # Note: also update .codecov.yml when changing the target coverage.
+      - poetry run ptw -- -- -s --cov --cov-report=lcov:lcov.info --cov-report=term:skip-covered --cov-fail-under=90 {{.CLI_ARGS}} tests/unit/
 
-  ci-cd-checks:
-    desc: Composite task to run all CI/CD checks.
+  docs:
+    desc: Launch a local documentation server to preview documentation in your browser, rebuilding the documentation when files are modified.
     cmds:
-      - task unit
-      - task lint
+      - poetry run sphinx-autobuild docs/source docs/build/html
+
+###################################
+### Automated Testing Pipelines ###
+###################################
 
-######################
-### Run Unit Tests ###
-######################
+  p: 
+    desc: Run all pre-commit checks. Execute this before committing code.
+    cmds:
+      - task: unit
+      - task: lint
 
-  unit:
-    desc: Run unit tests and check code coverage.
+  pp:
+    desc: Run all pre-push checks. Execute this before pushing code (or creating a PR).
     cmds:
-      # Recommended coverage viewer in VSCode: https://marketplace.visualstudio.com/items?itemName=ryanluker.vscode-coverage-gutters
-      # Note: also update .codecov.yml when changing the target coverage.
-      - poetry run pytest -s --cov --cov-report=lcov:lcov.info --cov-report=term:skip-covered --cov-fail-under=0 tests/unit/
+      - task: lint
+      - poetry export --with doc -f requirements.txt --output "{{.ROOT_DIR}}/docs/rtd_requirements.txt"
+      - task: no-changes
+      - task: ci
+      - task: no-changes
+
+  ci:
+    desc: Run all CI/CD checks. Execute this to debug a failing CI/CD pipeline.
+    cmds:
+      - task: unit
+      - task: lint
+      - task: docs-verify
+      - task: e2e-verify
 
-  unit-watch:
-    desc: Run unit tests and check code coverage immediately when files change.
+######################################################################################
+### Hidden: Too rarely used on their own, meant to be used as part of other tasks. ###
+######################################################################################
+
+  unit: # Run unit tests and check code coverage.
     cmds:
       # Recommended coverage viewer in VSCode: https://marketplace.visualstudio.com/items?itemName=ryanluker.vscode-coverage-gutters
       # Note: also update .codecov.yml when changing the target coverage.
-      - poetry run ptw -- -s --cov --cov-report=lcov:lcov.info --cov-report=term:skip-covered --cov-fail-under=0 tests/unit/
-
-
-###########################
-### Code Quality Checks ###
-########################### 
+      - poetry run pytest -s --cov --cov-report=lcov:lcov.info --cov-report=term:skip-covered --cov-fail-under=90 {{.CLI_ARGS}} tests/unit/
 
-  lint:
-    desc: Lint the code without auto-fixing issues.
+  lint: # Lint the code and auto-fix issues.
     cmds:
-      - poetry run ruff check sec_ai/
+      - poetry run ruff check --fix sec_ai/
       - poetry run mypy
 
-  lint-fix:
-    desc: Lint the code and auto-fix issues.
+  lint-check: # Lint the code without auto-fixing issues.
     cmds:
-      - poetry run ruff check --fix sec_ai/
+      - poetry run ruff check sec_ai/
       - poetry run mypy
 
-#######################
-### Developer Tools ###
-#######################
+  docs-verify:
+    silent: true
+    cmds:
+      - poetry export --with doc -f requirements.txt --output docs/rtd_requirements.txt
+      - echo "[Taskfile] Executing notebooks and checking for errors. The output will be saved to the notebook files."
+      - for file in $(find {{.ROOT_DIR}}/docs/source/notebooks -name "*.ipynb"); do echo -n "Processing file \"$file\"... "; poetry run exec_nb --exc_stop $file --dest $file && echo "done!" || { echo "Processing failed for file \"$file\""; exit 1; }; done
 
-  dash:
-    desc: Launch the debug dashboard for debugging and demos.
+  no-changes: # Check for no changes in the working tree or index. If there are any, abort.
+    silent: true
+    internal: true
     cmds:
-      # Extend PYTHONPATH to allow Streamlit to watch for file changes.
-      # Learn more: https://docs.streamlit.io/knowledge-base/using-streamlit/streamlit-watch-changes-other-modules-importing-app
-      - export PYTHONPATH=$PYTHONPATH:$(pwd)/sec_parser ENVIRONMENT=dev && poetry run streamlit run dev_utils/debug_dashboard/app.py --server.runOnSave=true
+      - if git diff --exit-code > /dev/null 2>&1 && git diff --cached --exit-code > /dev/null 2>&1; then :; else echo "Changes detected in the working tree or index. Please commit or stash them before proceeding."; exit 1; fi
 
-  docs:
-    desc: Run the documentation server.
+  clone-sec-parser-e2e-data: # Clone the sec-parser-e2e-data repository if it does not exist.
+    silent: true
     cmds:
-      - poetry run sphinx-autobuild docs/source docs/build/html
+      - if [ -d "{{.ROOT_DIR}}/../sec-parser-e2e-data" ]; then :; else echo "Repository does not exist. Cloning from GitHub..."; git clone https://github.com/alphanome-ai/sec-parser-e2e-data "{{.ROOT_DIR}}/../sec-parser-e2e-data" || { echo "Directory ../sec-parser-e2e-data does not exist and git clone failed. Please use \"git clone\" to download it from https://github.com/alphanome-ai/sec-parser-e2e-data. Aborting."; exit 1; }; fi
 
-  poetry:
-    desc: "Run any poetry command that modifies dependencies. Example: task poetry -- add requests"
+  version-bump: # Bump the version number.
     cmds:
-      - poetry {{.CLI_ARGS}}  
+      - poetry run cz bump {{.CLI_ARGS}}
       - poetry export --with doc -f requirements.txt --output docs/rtd_requirements.txt