diff --git a/.github/workflows/worker-task-Publish.yaml b/.github/workflows/worker-task-Publish.yaml index ce02e3a..cd3d8c9 100644 --- a/.github/workflows/worker-task-Publish.yaml +++ b/.github/workflows/worker-task-Publish.yaml @@ -18,7 +18,7 @@ jobs: - name: Build and publish Docker Image uses: openzim/docker-publish-action@v10 with: - image-name: kiwix/mirrors-qa-worker-task + image-name: kiwix/mirrors-qa-task-worker latest-on-tag: true tag-pattern: /^v([0-9.]+)$/ restrict-to: kiwix/mirrors-qa diff --git a/.gitignore b/.gitignore index 8722dd2..4909849 100644 --- a/.gitignore +++ b/.gitignore @@ -165,5 +165,8 @@ cython_debug/ # project files *.pem -data +*.conf +dev/data/** +!dev/data/README.md +!dev/.env id_rsa diff --git a/dev/.env b/dev/.env new file mode 100644 index 0000000..1a0553a --- /dev/null +++ b/dev/.env @@ -0,0 +1 @@ +WORKER_ID="elf" diff --git a/dev/README.md b/dev/README.md index 49afcff..74220be 100644 --- a/dev/README.md +++ b/dev/README.md @@ -1,6 +1,6 @@ This docker-compose configuration to be used **only** for development purpose. -## List of containers +## List of services ### backend @@ -15,39 +15,45 @@ This container is a PostgreSQL DB. DB data is kept in a volume, persistent acros ### scheduler This container creates test entries for idle workers (i.e workers who have not been seen in the last `IDLE_WORKER_SECONDS` environment variable) -### worker_manager +### task-worker +This container records the speed results for a particular test. -This container is the main worker container, responsible to start tasks. It is commented by default. Before uncommenting it: +### worker-manager -- [Get wireguard configuration files](#get-wireguard-configuration-files) -- [Create a test worker](#create-a-test-worker) -- [Build the task worker image](#build-the-task-image) - -## Instructions +This container is responsible for starting the task containers, setting up a wireguard interface, collecting the results from the task container and updates the results of the test on the backend API via REST. -**NOTE:** Unless otherwise stated, all the commands are run from the `dev` directory. - -First start the Docker-Compose stack: +## Starting the services ```sh -cd dev docker compose up --build ``` +The above command only starts the `backend`, `postgresqldb` and `scheduler` +services. -### get wireguard configuration files -- Create a `data` directory: - ```sh - mkdir data - ``` - This is the name of the directory used as a bind mount in the compose file. +To start the `worker-manager`, [you need to register a worker](#registering-a-worker). Afterwards, run: +```sh +docker compose --profile worker up --build +``` + +## Registering a Worker + +- [Get wireguard configuration files](#get-wireguard-configuration-files) +- [Create a test worker](#create-a-test-worker) -- Move the Mullvad configuration files into the `data` directory. Configuration files should be named in the format `.conf` where + +**NOTE:** Unless otherwise stated, all files and commands are with respective to the `dev` directory. + + +### Get wireguard configuration files + +- Move the [Mullvad](https://mullvad.net/) configuration files into the `data` directory. + Configuration files should be named in the format `.conf` where `` is the ISO 3166-1 alpha-2 code of the country. - On first time start up, the `worker_manager` will select a random file to start up the `wireguard` container. + On start, the `worker-manager` will select a random file to start up the `wireguard` container. On receiving a task from the backend scheduler, it will search for a configuration file belonging to that task and reconfigure the wireguard container. If the configuration file does not exist, it will skip the test. -### create a test worker +## create a test worker - Generate a private key: ```sh @@ -55,16 +61,52 @@ docker compose up --build ``` The key name `id_rsa` is used as a bind mount in the compose file. -- Assuming the backend service is up, create a worker in the backend: +- Assuming the backend service is up (`docker compose up backend`), create a worker and assign them a list of countries to test for. + If an existing worker is responsible for those countries, they are assigned to the new worker. In this example, we create a worker + named `test`. ```sh - docker exec -i mirrors-qa-backend mirrors-qa-backend create-worker --countries=us,fr,ca test_worker < ./id_rsa - export WORKER_ID=test_worker + docker exec -i mirrors-qa-backend mirrors-qa-backend create-worker --countries=us,fr,ca test < ./id_rsa ``` +- Set the name of the worker to the `WORKER_ID` variable in the `.env` file. -### build the task image -- Build the task worker image. - ```sh - cd ../worker/task - docker build -t mirrors-qa-task-worker . - export TASK_WORKER_IMAGE=mirrors-qa-task-worker - ``` +- Start the services with the worker enabled using `docker compose up --profile worker up --build` + +## Environment variables + +**NOTE:** All environment variables ending with a `_DURATION` suffix accept values that would be valid for [humanfriendly](https://humanfriendly.readthedocs.io/en/latest/api.html#humanfriendly.parse_timespan) + +### backend +The `backend` code houses the `scheduler` and the `RESTful API`. The following environment variables are shared by both services: +- `POSTGRES_URI`: PostgreSQL DSN string +- `REQUESTS_TIMEOUT_DURATION`: how long before a request to an external API times out +- `PAGE_SIZE` - number of rows to return from a request which returns a list of items +- `MIRRORS_LIST_URL`: the URL to fetch list of mirrors from. +- `EXCLUDED_MIRRORS`: hostname of mirror URLs to exclude seperated by commas. + +### REST API +- `JWT_SECRET` +- `MESSAGE_VALIDITY_DURATION`: how long should the authentication message be considered as valid from when it was signed +- `TOKEN_EXPIRY_DURATION`: how long access tokens should live + +### scheduler +- `SCHEDULER_SLEEP_DURATION`: how long the scheduler should sleep after creating tests for idle workers +- `IDLE_WORKER_DURATION`: mark a worker as idle if it hasn't been seen within duration +- `EXPIRE_TEST_DURATION`: expire tests whose results are still pending after duration + +### worker-manager +- `SLEEP_DURATION`: how long the manager should sleep before polling the REST API +- `BACKEND_API_URI` +- `DOCKER_SOCKET` +- `PRIVATE_KEY_FILE`: name of private key file +- `DOCKER_CLIENT_TIMEOUT_DURATION`: how long before a connection to the Docker daemon times out +- `DOCKER_API_RETRIES`: how many times to retry requests to the Docker daemon +- `DOCKER_API_RETRY_DURATION`: how long to wait before retrying a failed request +- `WIREGUARD_IMAGE` +- `WIREGUARD_KERNEL_MODULES`: where to load wireguard kernel modules from (default `/lib/modules`) +- `WIREGUARD_HEALTHCHECK_INTERVAL_SECONDS` +- `WIREGUARD_HEALTHCHECK_TIMEOUT_SECONDS` +- `WIREGUARD_HEALTHCHECK_RETRIES` +- `TASK_WORKER_IMAGE` + +## task-worker +- `REQUESTS_TIMEOUT_SECONDS`: how many seconds beore a request times out diff --git a/dev/data/README.md b/dev/data/README.md new file mode 100644 index 0000000..e69de29 diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index fe3a121..8cf6e34 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -25,6 +25,7 @@ services: POSTGRES_URI: postgresql+psycopg://mirrors_qa:mirrors_qa@postgresdb:5432/mirrors_qa JWT_SECRET: DH8kSxcflUVfNRdkEiJJCn2dOOKI3qfw DEBUG: true + REQUESTS_TIMEOUT_DURATION: 30s ports: - 8000:80 healthcheck: @@ -41,26 +42,39 @@ services: container_name: mirrors-qa-scheduler environment: POSTGRES_URI: postgresql+psycopg://mirrors_qa:mirrors_qa@postgresdb:5432/mirrors_qa - IDLE_WORKER_SECONDS: 180 + IDLE_WORKER_DURATION: 1d + SCHEDULER_SLEEP_DURATION: 1m + EXPIRE_TEST_DURATION: 2m DEBUG: true command: mirrors-qa-backend scheduler -# worker_manager: -# build: -# context: ../worker/manager -# depends_on: -# backend: -# condition: service_healthy -# container_name: mirrors-qa-manager -# volumes: -# - /var/run/docker.sock:/var/run/docker.sock -# - /lib/modules:/lib/modules -# - ./data:/data -# - ./id_rsa:/etc/ssh/keys/id_rsa -# environment: -# - BACKEND_API_URI=http://backend -# - TASK_WORKER_IMAGE=${TASK_WORKER_IMAGE:?TASK_WORKER_IMAGE not set} -# - WORKER_ID=${WORKER_ID:?WORKER_ID not set} -# command: mirrors-qa-manager --verbose ${WORKER_ID} + worker-manager: + build: + context: ../worker/manager + depends_on: + backend: + condition: service_healthy + task-worker: + condition: service_started + profiles: + - worker + container_name: mirrors-qa-manager + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /lib/modules:/lib/modules + - ./data:/data + - ./id_rsa:/etc/ssh/keys/id_rsa + environment: + - BACKEND_API_URI=http://backend + - SLEEP_DURATION=5m + - TASK_WORKER_IMAGE=mirrors-qa-task-worker + command: mirrors-qa-manager --verbose ${WORKER_ID} + task-worker: + build: + context: ../worker/task + image: mirrors-qa-task-worker + profiles: + - worker + command: ["mirrors-qa-task", "--help"] volumes: pg-data-mirrors-qa: