Skip to content

Commit

Permalink
updating test workflow; adding dependabot and codeQL; adding compile …
Browse files Browse the repository at this point in the history
…report; removing unneeded parts of Makefile; adding pyproject file for Ruff linting/checks
  • Loading branch information
ialarmedalien committed Jul 31, 2024
1 parent 7d65529 commit adc177e
Show file tree
Hide file tree
Showing 9 changed files with 351 additions and 137 deletions.
5 changes: 5 additions & 0 deletions .coveragerc
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
[html]
directory = ./python-coverage

[xml]
output = ./python-coverage/coverage.xml
19 changes: 19 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
version: 2
updates:
- package-ecosystem: docker
directory: "/"
schedule:
interval: monthly
time: '11:00'
open-pull-requests-limit: 10
- package-ecosystem: pip
directory: "/"
schedule:
interval: monthly
time: '11:00'
open-pull-requests-limit: 10
- package-ecosystem: "github-actions"
directory: ".github/workflows"
schedule:
interval: "monthly"
open-pull-requests-limit: 10
47 changes: 47 additions & 0 deletions .github/workflows/codeql.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
name: "CodeQL"

on:
push:
branches:
- develop
- main
- master
pull_request:
types:
- opened
- reopened
- synchronize
- closed

jobs:
analyze:
name: Analyze
runs-on: ubuntu-latest
permissions:
actions: read
contents: read
security-events: write

strategy:
fail-fast: false
matrix:
language: [python]

steps:
- name: Checkout
uses: actions/checkout@master

- name: Initialize CodeQL
uses: github/codeql-action/init@main
with:
languages: ${{ matrix.language }}
queries: +security-and-quality

- name: Autobuild
uses: github/codeql-action/autobuild@main
if: ${{ matrix.language == 'javascript' || matrix.language == 'python' }}

- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@main
with:
category: "/language:${{ matrix.language }}"
103 changes: 46 additions & 57 deletions .github/workflows/kb_sdk_test.yaml
Original file line number Diff line number Diff line change
@@ -1,62 +1,51 @@
name: KBase SDK Tests

on:
push:
branches:
- master
- main
pull_request:
branches:
- master
- main
- develop
push:
branches:
- master
- main
pull_request:
branches:
- master
- main
- develop

jobs:

sdk_tests:
runs-on: ubuntu-latest
steps:

- name: Check out GitHub repo
if: "!contains(github.event.head_commit.message, 'skip ci')"
uses: actions/checkout@v2

- name: Check out Actions CI files
if: "!contains(github.event.head_commit.message, 'skip ci')"
uses: actions/checkout@v2
with:
repository: 'kbaseapps/kb_sdk_actions'
path: 'kb_sdk_actions'


- name: Set up test environment
if: "!contains(github.event.head_commit.message, 'skip ci')"
shell: bash
env:
KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }}
run: |
# Verify kb_sdk_actions clone worked
test -f "$HOME/kb_sdk_actions/bin/kb-sdk" && echo "CI files cloned"
# Pull kb-sdk & create startup script
docker pull kbase/kb-sdk
sh $GITHUB_WORKSPACE/kb_sdk_actions/bin/make_testdir && echo "Created test_local"
test -f "test_local/test.cfg" && echo "Confirmed config exists"
- name: Configure authentication
if: "!contains(github.event.head_commit.message, 'skip ci')"
shell: bash
env:
KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }}
run: |
# Add token to config
sed -ie "s/^test_token=.*$/&$KBASE_TEST_TOKEN/g" ./test_local/test.cfg
- name: Run tests
if: "!contains(github.event.head_commit.message, 'skip ci')"
shell: bash
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
run: |
sh $GITHUB_WORKSPACE/kb_sdk_actions/bin/kb-sdk test
bash <(curl -s https://codecov.io/bash)
sdk_tests:
runs-on: ubuntu-latest
steps:
- name: Check out GitHub repo
uses: actions/checkout@master

- name: Set up test environment
shell: bash
run: |
# Cache the group for the docker file
if [ ! -e $HOME/.kbsdk.cache ] ; then
docker run -i -v /var/run/docker.sock:/var/run/docker.sock --entrypoint ls ghcr.io/kbase/kb_sdk_patch-develop:br-0.0.4-rc-1 -l /var/run/docker.sock|awk '{print $4}' > $HOME/.kbsdk.cache
fi
# ignore the exit code
docker run -i --rm -v $HOME:$HOME -u $(id -u) -w $(pwd) -v /var/run/docker.sock:/var/run/docker.sock -e DUSER=$USER -e DSHELL=$SHELL --group-add $(cat $HOME/.kbsdk.cache) ghcr.io/kbase/kb_sdk_patch-develop:br-0.0.4-rc-1 test || true
- name: Confirm config exists and add token
shell: bash
env:
KBASE_TEST_TOKEN: ${{ secrets.KBASE_TEST_TOKEN }}
run: |
test -f "test_local/test.cfg" && echo "Confirmed config exists"
# Add token to config
sed -ie "s/^test_token=.*$/&$KBASE_TEST_TOKEN/g" ./test_local/test.cfg
- name: Run tests
shell: bash
run: |
docker run -i --rm -v $HOME:$HOME -w $(pwd) -v /var/run/docker.sock:/var/run/docker.sock -e DSHELL=$SHELL --group-add $(cat $HOME/.kbsdk.cache) ghcr.io/kbase/kb_sdk_patch-develop:br-0.0.4-rc-1 test --verbose
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
with:
token: ${{ secrets.CODECOV_TOKEN }}
fail_ci_if_error: true
34 changes: 5 additions & 29 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,18 @@ LBIN_DIR = bin
EXECUTABLE_SCRIPT_NAME = run_$(SERVICE_CAPS)_async_job.sh
STARTUP_SCRIPT_NAME = start_server.sh
TEST_SCRIPT_NAME = run_tests.sh
COMPILE_REPORT = ./compile_report.json

.PHONY: test

default: compile

all: compile build build-startup-script build-executable-script build-test-script
all: build compile

compile:
kb-sdk compile $(SPEC_FILE) \
rm $(COMPILE_REPORT) || true
KB_SDK_COMPILE_REPORT_FILE=$(COMPILE_REPORT) kb-sdk compile $(SPEC_FILE) \
--verbose \
--out $(LIB_DIR) \
--plclname $(SERVICE_CAPS)::$(SERVICE_CAPS)Client \
--jsclname javascript/Client \
Expand All @@ -31,33 +34,6 @@ compile:
build:
chmod +x $(SCRIPTS_DIR)/entrypoint.sh

build-executable-script:
mkdir -p $(LBIN_DIR)
echo '#!/bin/bash' > $(LBIN_DIR)/$(EXECUTABLE_SCRIPT_NAME)
echo 'script_dir=$$(dirname "$$(readlink -f "$$0")")' >> $(LBIN_DIR)/$(EXECUTABLE_SCRIPT_NAME)
echo 'export PYTHONPATH=$$script_dir/../$(LIB_DIR):$$PATH:$$PYTHONPATH' >> $(LBIN_DIR)/$(EXECUTABLE_SCRIPT_NAME)
echo 'python -u $$script_dir/../$(LIB_DIR)/$(SERVICE_CAPS)/$(SERVICE_CAPS)Server.py $$1 $$2 $$3' >> $(LBIN_DIR)/$(EXECUTABLE_SCRIPT_NAME)
chmod +x $(LBIN_DIR)/$(EXECUTABLE_SCRIPT_NAME)

build-startup-script:
mkdir -p $(LBIN_DIR)
echo '#!/bin/bash' > $(SCRIPTS_DIR)/$(STARTUP_SCRIPT_NAME)
echo 'script_dir=$$(dirname "$$(readlink -f "$$0")")' >> $(SCRIPTS_DIR)/$(STARTUP_SCRIPT_NAME)
echo 'export KB_DEPLOYMENT_CONFIG=$$script_dir/../deploy.cfg' >> $(SCRIPTS_DIR)/$(STARTUP_SCRIPT_NAME)
echo 'export PYTHONPATH=$$script_dir/../$(LIB_DIR):$$PATH:$$PYTHONPATH' >> $(SCRIPTS_DIR)/$(STARTUP_SCRIPT_NAME)
echo 'uwsgi --master --processes 5 --threads 5 --http :5000 --wsgi-file $$script_dir/../$(LIB_DIR)/$(SERVICE_CAPS)/$(SERVICE_CAPS)Server.py' >> $(SCRIPTS_DIR)/$(STARTUP_SCRIPT_NAME)
chmod +x $(SCRIPTS_DIR)/$(STARTUP_SCRIPT_NAME)

build-test-script:
echo '#!/bin/bash' > $(TEST_DIR)/$(TEST_SCRIPT_NAME)
echo 'script_dir=$$(dirname "$$(readlink -f "$$0")")' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME)
echo 'export KB_DEPLOYMENT_CONFIG=$$script_dir/../deploy.cfg' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME)
echo 'export KB_AUTH_TOKEN=`cat /kb/module/work/token`' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME)
echo 'export PYTHONPATH=$$script_dir/../$(LIB_DIR):$$PATH:$$PYTHONPATH' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME)
echo 'cd $$script_dir/../$(TEST_DIR)' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME)
echo 'python -m nose --with-coverage --cover-package=$(SERVICE_CAPS) --cover-html --cover-html-dir=/kb/module/work/test_coverage --nocapture --nologcapture .' >> $(TEST_DIR)/$(TEST_SCRIPT_NAME)
chmod +x $(TEST_DIR)/$(TEST_SCRIPT_NAME)

test:
if [ ! -f /kb/module/work/token ]; then echo -e '\nOutside a docker container please run "kb-sdk test" rather than "make test"\n' && exit 1; fi
xvfb-run bash $(TEST_DIR)/$(TEST_SCRIPT_NAME)
Expand Down
1 change: 1 addition & 0 deletions compile_report.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"functions":{"run_FastTree":{"name":"run_FastTree","comment":"Method for Tree building of either DNA or PROTEIN sequences\n**\n** input_type: MSA\n** output_type: Tree","place":null,"input":[{"type":"kb_fasttree.FastTree_Params","comment":"FastTree Input Params"}],"output":[{"type":"kb_fasttree.FastTree_Output","comment":"FastTree Output"}]}},"module_name":"kb_fasttree","sdk_version":"1.2.6","sdk_git_commit":"cedcfc2326db32bbdb09eb6d031838a93853c530","impl_file_path":"lib/kb_fasttree/kb_fasttreeImpl.py","spec_files":[{"content":"/*\n** A KBase module: kb_fasttree\n**\n** This module runs FastTree to make Trees for either DNA or PROTEIN MSAs\n** \n*/\n\nmodule kb_fasttree {\n\n /* \n ** The workspace object refs are of form:\n **\n ** objects = ws.get_objects([{'ref': params['workspace_id']+'/'+params['obj_name']}])\n **\n ** \"ref\" means the entire name combining the workspace id and the object name\n ** \"id\" is a numerical identifier of the workspace or object, and should just be used for workspace\n ** \"name\" is a string identifier of a workspace or object. This is received from Narrative.\n */\n typedef string workspace_name;\n typedef string data_obj_name;\n typedef string data_obj_ref;\n\n\n /* FastTree Input Params\n */\n typedef structure {\n workspace_name workspace_name;\n\tstring desc;\n\tdata_obj_ref input_ref;\n data_obj_name output_name;\n\tint species_tree_flag; /* boolean */\n\tdata_obj_ref intree_ref;\n\tint fastest; /* boolean */\n\tint pseudo; /* boolean */\n\tint gtr; /* boolean */\n\tint wag; /* boolean */\n\tint noml; /* boolean */\n\tint nome; /* boolean */\n int cat; /* actually is an int */\n\tint nocat; /* boolean */\n int gamma; /* boolean */\n } FastTree_Params;\n\n\n /* FastTree Output\n */\n typedef structure {\n\tdata_obj_name report_name;\n\tdata_obj_ref report_ref;\n data_obj_ref output_ref;\n } FastTree_Output;\n\t\n\n /* Method for Tree building of either DNA or PROTEIN sequences\n **\n ** input_type: MSA\n ** output_type: Tree\n */\n funcdef run_FastTree (FastTree_Params params) returns (FastTree_Output) authentication required;\n};\n","file_name":"kb_fasttree.spec","is_main":1}],"function_places":{}}
123 changes: 123 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
[tool.ruff]
line-length = 100

extend-exclude = [
"lib/installed_clients/*"
]

# Assume Python 3.12.
target-version = "py312"

[tool.ruff.lint]
select = [
# core
"F", # Pyflakes
"E", # pycodestyle errors
"W", # pycodestyle warnings
"C90", # mccabe +
"I", # isort
"N", # pep8-naming
"D", # pydocstyle
"UP", # pyupgrade
# extensions
"YTT", # flake8-2020
"ANN", # flake8-annotations
"ASYNC", # flake8-async
"S", # flake8-bandit
"BLE", # flake8-blind-except
"FBT", # flake8-boolean-trap
"B", # flake8-bugbear
"A", # flake8-builtins
# "COM", # flake8-commas
"CPY", # flake8-copyright
"C4", # flake8-comprehensions
"DTZ", # flake8-datetimez
"T10", # flake8-debugger
# "DJ", # flake8-django
"EM", # flake8-errmsg
"EXE", # flake8-executable
"FA", # flake8-future-annotations
"ISC", # flake8-implicit-str-concat
"ICN", # flake8-import-conventions
"G", # flake8-logging-format
"INP", # flake8-no-pep420
"PIE", # flake8-pie
"T20", # flake8-print
"PYI", # flake8-pyi
"PT", # flake8-pytest-style
"Q", # flake8-quotes
"RSE", # flake8-raise
"RET", # flake8-return
"SLF", # flake8-self
# "SLOT", # flake8-slots
"SIM", # flake8-simplify
"TID", # flake8-tidy-imports
"TCH", # flake8-type-checking
"INT", # flake8-gettext
"ARG", # flake8-unused-arguments
"PTH", # flake8-use-pathlib
"TD", # flake8-todos
"FIX", # flake8-fixme
"ERA", # eradicate
# "PD", # pandas-vet
"PGH", # pygrep-hooks
"PL", # Pylint
"TRY", # tryceratops
"FLY", # flynt
# "NPY", # NumPy-specific rules
"AIR", # Airflow
"PERF", # Perflint
"FURB", # refurb
"LOG", # flake8-logging
"RUF", # Ruff-specific rules
]

# E203: whitespace before ‘,’, ‘;’, or ‘:’
# E501: line length
# ISC001: conflicts with Ruff's formatter
# W503: line break after binary operator
ignore = [
"E203",
"E501",
"ISC001",
"S101",
]

# Allow autofix for all enabled rules (when `--fix`) is provided.
fixable = ["ALL"]
unfixable = []

# Exclude a variety of commonly ignored directories.
exclude = [
"__pypackages__",
"_build",
".bzr",
".direnv",
".eggs",
".git-rewrite",
".git",
".github",
".hg",
".mypy_cache",
".nox",
".pants.d",
".pytype",
".ruff_cache",
".svn",
".tox",
".venv",
"*.pyc",
"buck-out",
"build",
"deps",
"dist",
"node_modules",
"other_schema",
"python-coverage",
"sample_data",
"venv",
]
per-file-ignores = {}

[tool.ruff.lint.pydocstyle]
convention = "google"
13 changes: 11 additions & 2 deletions test/run_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,14 @@ script_dir=$(dirname "$(readlink -f "$0")")
export KB_DEPLOYMENT_CONFIG=$script_dir/../deploy.cfg
export KB_AUTH_TOKEN=`cat /kb/module/work/token`
export PYTHONPATH=$script_dir/../lib:$PATH:$PYTHONPATH
cd $script_dir/../test
python -u -m unittest discover -p "*_test.py"

# run without collecting coverage data
# pytest -vv test

# collect coverage data
pytest \
--cov=lib/ \
--cov-config=.coveragerc \
--cov-report=html \
--cov-report=xml \
test
Loading

0 comments on commit adc177e

Please sign in to comment.