[code2fn] Various bug fixes and documentation updates #1307
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Unit tests and project documentation | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| env: | |
| # used by both Python and Rust services | |
| # to connect to Memgraph. | |
| # used by skema-rs services | |
| SKEMA_GRAPH_DB_HOST: "127.0.0.1" | |
| SKEMA_GRAPH_DB_PORT: "7687" | |
| SKEMA_GRAPH_DB_PROTO: "bolt://" | |
| SKEMA_RS_HOST: "127.0.0.1" | |
| SKEMA_RS_PORT: "8001" | |
| # used by Python services. | |
| # unfortunately, there doesn't seem to be a way | |
| # to reference SKEMA_RS_PORT in SKEMA_RS_ADDESS ... | |
| SKEMA_RS_ADDESS: "http://127.0.0.1:8001" | |
| jobs: | |
| docs: | |
| name: "Run tests and generate project documentation" | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] #, macos-latest, windows-latest] | |
| python-version: [ '3.8' ] #[ '3.8', '3.9' ] | |
| # TODO: add rust-version 1.62 | |
| # exclude: | |
| # - os: macos-latest | |
| # python-version: '3.8' | |
| # - os: windows-latest | |
| # python-version: '3.6' | |
| services: | |
| graphdb: | |
| image: "memgraph/memgraph-platform" | |
| ports: | |
| - "7687:7687" | |
| - "3000:3000" | |
| - "7444:7444" | |
| volumes: | |
| - mg_lib:/var/lib/memgraph | |
| - mg_log:/var/log/memgraph | |
| - mg_etc:/etc/memgraph | |
| steps: | |
| # Checkout code | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| #architecture: x64 | |
| - name: Display Python version | |
| run: | | |
| python -c "import sys; print(sys.version)" | |
| - name: Set up Rust | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: stable | |
| - name: Display Rust version | |
| run: | | |
| rustc --version | |
| - name: Setup JDK | |
| uses: actions/setup-java@v3 | |
| with: | |
| distribution: temurin | |
| java-version: 11 | |
| # Install dependencies | |
| - name: System-wide deps | |
| run: | | |
| sudo apt-get install -y graphviz \ | |
| libgraphviz-dev | |
| # Install packages (for API doc gen) | |
| - name: Install askem (Python) | |
| working-directory: . | |
| run: | | |
| # retrieve latest model for img2mml component | |
| curl -L https://artifacts.askem.lum.ai/skema/img2mml/models/cnn_xfmer_arxiv_im2mml_with_fonts_boldface_best.pt > skema/img2mml/trained_models/cnn_xfmer_arxiv_im2mml_with_fonts_boldface_best.pt | |
| pip install ".[all]" | |
| # Install tree-sitter parser (for Python component unit tests) | |
| - name: TEST | |
| working-directory: . | |
| run: python scripts/retrieve_model.py | |
| # Install tree-sitter parser (for Python component unit tests) | |
| - name: Install tree-sitter parsers | |
| working-directory: . | |
| run: python skema/program_analysis/tree_sitter_parsers/build_parsers.py --ci --all | |
| # docs (API) | |
| # generate python docs using pdoc | |
| - name: "Create documentation for Python components (API docs)" | |
| run: | | |
| # FIXME: once issues with text reading submodule resolved, top command will be sufficient | |
| #pdoc --html -c latex_math=True --force --output-dir docs/api/python skema | |
| # Skip generation due to https://github.com/pdoc3/pdoc/issues/199 | |
| pdoc --html -c latex_math=True --force --output-dir docs/api/python skema.img2mml | |
| pdoc --html -c latex_math=True --force --output-dir docs/api/python skema.isa | |
| pdoc --html -c latex_math=True --force --output-dir docs/api/python skema.skema_py | |
| pdoc --html -c latex_math=True --force --output-dir docs/api/python skema.gromet | |
| pdoc --html -c latex_math=True --force --output-dir docs/api/python skema.program_analysis | |
| pdoc --html -c latex_math=True --force --output-dir docs/api/python skema.metal.model_linker | |
| # generate Rust docs using cargo doc | |
| - name: "Create documentation for Rust components (API docs)" | |
| working-directory: ./skema/skema-rs | |
| run: | | |
| cargo doc | |
| mkdir -p $GITHUB_WORKSPACE/docs/api/rust | |
| mv target/doc/* $GITHUB_WORKSPACE/docs/api/rust/ | |
| # generate Scala docs using sbt doc | |
| # FIXME: downloading project deps is far too slow and too large to cache (3+ GB) to include this step in our CI | |
| # - name: "Create documentation for Scala components (API docs)" | |
| # working-directory: ./skema/text_reading/scala | |
| # run: | | |
| # sbt doc | |
| # mkdir -p $GITHUB_WORKSPACE/docs/api/scala | |
| # mv target/doc/* $GITHUB_WORKSPACE/docs/api/scala/ | |
| # code coverage (Scala) | |
| # FIXME: downloading project deps is far too slow to include this step | |
| # - name: "Code coverage reports for Scala components" | |
| # working-directory: ./skema/text_reading/scala | |
| # run: | | |
| # sbt clean coverage test | |
| # sbt coverageReport | |
| # sbt coverageAggregate | |
| - name: "Launch skema-rs in background for testing" | |
| working-directory: ./skema/skema-rs | |
| run: | | |
| echo "skema-rs REST service: $SKEMA_RS_HOST:$SKEMA_RS_PORT" | |
| cargo build --release --bin skema_service | |
| ./target/release/skema_service & | |
| echo "results for ${SKEMA_RS_HOST}:${SKEMA_RS_PORT}/version: `curl -s ${SKEMA_RS_HOST}:${SKEMA_RS_PORT}/version`" | |
| # NOTE: moved to runbefore the Python unit tests, so we can use the model coverage report results in testing | |
| - name: "Code2fn model coverage reports" | |
| run: python skema/program_analysis/model_coverage_report/model_coverage_report.py $GITHUB_WORKSPACE/docs/coverage/code2fn_coverage/ all | |
| # test & code coverage (Python) | |
| - name: "Unit tests and code coverage reports for Python components" | |
| # env: | |
| # SKEMA_GRAPH_DB_HOST: "bolt://127.0.0.1" | |
| run: | | |
| echo "SKEMA_RS_ADDESS: $SKEMA_RS_ADDESS" | |
| python -c "import requests; print(requests.get('$SKEMA_RS_ADDESS/version').text)" | |
| SKEMA_RS_ADDRESS=$SKEMA_RS_ADDESS pytest -vvv | |
| - name: "Unit tests for Rust components" | |
| working-directory: ./skema/skema-rs | |
| run: | | |
| cargo test --verbose --all | |
| - name: Free disk space (aggressively) | |
| # https://github.com/jlumbroso/free-disk-space | |
| uses: jlumbroso/[email protected] | |
| with: | |
| android: true | |
| dotnet: true | |
| haskell: true | |
| large-packages: true | |
| docker-images: true | |
| swap-storage: true | |
| # NOTE: this might remove things we actually need. | |
| tool-cache: true | |
| # docs (other) | |
| - name: "Create documentation (other)" | |
| run: | | |
| docker run -i -v "$GITHUB_WORKSPACE:/app" parsertongue/mkdocs:latest mkdocs build -c | |
| - name: Deploy docs | |
| if: github.ref == 'refs/heads/main' | |
| uses: peaceiris/actions-gh-pages@v3 | |
| with: | |
| # see https://docs.github.com/en/free-pro-team@latest/actions/reference/authentication-in-a-workflow#about-the-github_token-secret | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| publish_dir: ./site | |
| user_name: 'github-actions[bot]' | |
| user_email: 'github-actions[bot]@users.noreply.github.com' | |
| commit_message: ${{ github.event.head_commit.message }} |