From 849294d6890fe1b812c51da007e4f0b066e5b3bc Mon Sep 17 00:00:00 2001
From: Maaike <maaike.limper@gmail.com>
Date: Fri, 10 Jan 2025 16:47:02 +0100
Subject: [PATCH 01/17] commit to create draft PR
---
.gitignore | 4 +
docker-compose.yml => docker-compose.base.yml | 32 +++--
docker-compose.monitoring.yml | 18 +--
wis2box-ctl.py | 110 +++++++++++++++++-
4 files changed, 133 insertions(+), 31 deletions(-)
rename docker-compose.yml => docker-compose.base.yml (86%)
diff --git a/.gitignore b/.gitignore
index 20bf19c9..b9f3b024 100644
--- a/.gitignore
+++ b/.gitignore
@@ -45,3 +45,7 @@ wis2box.env
docker/.env
.ipynb_checkpoints
+tests/data/.ssh/id_rsa
+tests/data/.ssh/id_rsa.pub
+
+docker-compose.yml
diff --git a/docker-compose.yml b/docker-compose.base.yml
similarity index 86%
rename from docker-compose.yml
rename to docker-compose.base.yml
index f45a6684..1758dc78 100644
--- a/docker-compose.yml
+++ b/docker-compose.base.yml
@@ -13,7 +13,7 @@ services:
wis2box-ui:
container_name: wis2box-ui
- image: ghcr.io/wmo-im/wis2box-ui:latest
+ image: wmoim/wis2box-ui:latest
restart: always
env_file:
- wis2box.env
@@ -22,14 +22,14 @@ services:
wis2box-webapp:
container_name: wis2box-webapp
- image: ghcr.io/wmo-im/wis2box-webapp:latest
+ image: wmoim/wis2box-webapp:latest
env_file:
- wis2box.env
restart: always
wis2box-api:
container_name: wis2box-api
- image: ghcr.io/wmo-im/wis2box-api:latest
+ image: wmoim/wis2box-api:latest
restart: always
env_file:
- wis2box.env
@@ -104,10 +104,8 @@ services:
mosquitto:
container_name: mosquitto
- #image: ghcr.io/wmo-im/wis2box-broker:latest
+ image: wmoim/wis2box-broker:local
restart: always
- build:
- context: ./wis2box-broker
env_file:
- wis2box.env
volumes:
@@ -115,13 +113,10 @@ services:
wis2box-management:
container_name: wis2box-management
+ image: wmoim/wis2box-management:local
mem_limit: 1g
memswap_limit: 1g
restart: always
- #image: ghcr.io/wmo-im/wis2box-management:latest
- build:
- context: ./wis2box-management
- #user: wis2box:wis2box
env_file:
- wis2box.env
volumes:
@@ -134,9 +129,22 @@ services:
condition: service_healthy
command: ["wis2box", "pubsub" , "subscribe"]
+ # mqtt_metrics_collector, listens to mqtt-broker
+ mqtt_metrics_collector:
+ container_name: mqtt_metrics_collector
+ restart: unless-stopped
+ env_file:
+ - wis2box.env
+ image: wmoim/wis2box-mqtt-metrics-collector:local
+ depends_on:
+ - mosquitto
+ - wis2box-management
+ ports:
+ - 8001:8001
+
wis2box-auth:
container_name: wis2box-auth
- image: ghcr.io/wmo-im/wis2box-auth:latest
+ image: wmoim/wis2box-auth:latest
restart: always
env_file:
- wis2box.env
@@ -147,7 +155,7 @@ services:
wis2downloader:
container_name: wis2downloader
- image: ghcr.io/wmo-im/wis2downloader:v0.3.2
+ image: wmoim/wis2downloader:latest
restart: always
env_file:
- wis2box.env
diff --git a/docker-compose.monitoring.yml b/docker-compose.monitoring.yml
index d4291023..eade2ea8 100644
--- a/docker-compose.monitoring.yml
+++ b/docker-compose.monitoring.yml
@@ -34,22 +34,6 @@ services:
vpcbr: # this is the place where we assign the static ipv4 address
ipv4_address: 10.5.0.2
default:
-
- # mqtt_metrics_collector, listens to mqtt-broker
- mqtt_metrics_collector:
- <<: *logging
- container_name: mqtt_metrics_collector
- restart: unless-stopped
- env_file:
- - wis2box.env
- #image: ghcr.io/wmo-im/wis2box-mqtt-metrics-collector:latest
- build:
- context: ./wis2box-mqtt-metrics-collector
- depends_on:
- - mosquitto
- - wis2box-management
- ports:
- - 8001:8001
# prometheus to collect metrics
prometheus:
@@ -129,6 +113,8 @@ services:
<<: *logging
wis2box-auth:
<<: *logging
+ mqtt_metrics_collector:
+ <<: *logging
minio:
<<: *logging
web-proxy:
diff --git a/wis2box-ctl.py b/wis2box-ctl.py
index 070a8d19..5d0e2788 100755
--- a/wis2box-ctl.py
+++ b/wis2box-ctl.py
@@ -22,8 +22,11 @@
import argparse
import os
+import re
+import requests
import subprocess
+
if subprocess.call(['docker', 'compose'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) > 0:
DOCKER_COMPOSE_COMMAND = 'docker-compose'
else:
@@ -92,6 +95,99 @@
args = parser.parse_args()
+# define ENUM for docker image usage, LATEST or LOCAL
+LOCAL = 0
+LATEST = 1
+
+LOCAL_IMAGES = [
+ 'wis2box-management',
+ 'wis2box-broker',
+ 'wis2box-mqtt-metrics-collector'
+]
+
+def build_local_images() -> None:
+ """
+ Build local images
+
+ :returns: None.
+ """
+ for image in LOCAL_IMAGES:
+ print(f'Building {image}')
+ run(split(f'docker build -t wmoim/{image}:local {image}'))
+
+ return None
+
+def get_latest_image_tag(image: str) -> str:
+ """
+ list image tags by querying docker hub api
+ skip the 'latest' tag
+ return the most recent tag
+
+ :param image: required, string. Name of the image.
+
+ :returns: string. The most recent tag.
+ """
+
+ url = f"https://hub.docker.com/v2/repositories/wmoim/{image}/tags/"
+ tags = []
+ try:
+ # Paginate through results to collect all tags
+ while url:
+ response = requests.get(url)
+ response.raise_for_status() # Raise an error for HTTP failures
+ data = response.json()
+ tags.extend([
+ tag['name'] for tag in data.get('results', [])
+ if tag['name'] != 'latest' # Skip 'latest' tag
+ ])
+ url = data.get('next') # Get the next page URL
+ except requests.RequestException as e:
+ raise RuntimeError(f"Failed to fetch tags for image '{image}': {e}")
+
+ if not tags:
+ raise ValueError(f"No valid tags found for image '{image}'")
+ else:
+ print(f"Found {len(tags)} tags for image '{image}: {tags}'")
+
+ # define a function to sort tags by version number
+ def tag_sort_key(tag):
+ # Extract numeric and non-numeric parts
+ parts = re.split(r'(\d+)', tag) # Split into numeric and non-numeric segments
+ return [
+ int(part) if part.isdigit() else 'ZZZZZZZZZZ' # Use a large number for non-numeric parts
+ for part in parts
+ ]
+
+ # Sort tags by version number in descending order
+ tags.sort(key=tag_sort_key, reverse=True)
+ return tags[0]
+
+
+def update_docker_images(image_type: int) -> None:
+ """
+ Write docker-compose.yml using docker-compose.base.yml as base
+ and set image-versions based on the latest available images in wmoim
+
+ image_type: int. LATEST or LOCAL
+
+ :returns: None.
+ """
+
+ with open('docker-compose.base.yml', 'r') as f:
+ lines = f.readlines()
+
+ with open('docker-compose.yml', 'w') as f:
+ for line in lines:
+ if 'image: wmoim/' in line and image_type == LATEST:
+ image = line.split('wmoim/')[1].split(':')[0]
+ print(f'Get latest image tag for {image}')
+ tag = get_latest_image_tag(image)
+ print(f'Set {image} to {tag}')
+ f.write(f' image: wmoim/{image}:{tag}\n')
+ else:
+ f.write(line)
+
+ return None
def split(value: str) -> list:
"""
@@ -169,8 +265,8 @@ def make(args) -> None:
if args.command == "config":
run(split(f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} config'))
elif args.command == "build":
- run(split(
- f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} build {containers}'))
+ build_local_images()
+ update_docker_images(LOCAL)
elif args.command in ["up", "start", "start-dev"]:
run(split(
'docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions'),
@@ -199,7 +295,15 @@ def make(args) -> None:
run(split(
f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} down --remove-orphans {containers}'))
elif args.command == "update":
- run(split(f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} pull'))
+ update_docker_images(LATEST)
+ # restart all containers
+ run(split(
+ f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} down --remove-orphans'))
+ run(split(
+ f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} up -d'))
+ # prune dangling images
+ _ = run(split('docker images --filter dangling=true -q --no-trunc'))
+ run(split(f'docker rmi {_}'))
elif args.command == "prune":
run(split('docker builder prune -f'))
run(split('docker container prune -f'))
From 8680e78a57b5f1951460ebc8a70bba3766730ab5 Mon Sep 17 00:00:00 2001
From: Maaike <maaike.limper@gmail.com>
Date: Fri, 10 Jan 2025 16:58:56 +0100
Subject: [PATCH 02/17] use LATEST by default
---
wis2box-ctl.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/wis2box-ctl.py b/wis2box-ctl.py
index 5d0e2788..14d88687 100755
--- a/wis2box-ctl.py
+++ b/wis2box-ctl.py
@@ -268,6 +268,9 @@ def make(args) -> None:
build_local_images()
update_docker_images(LOCAL)
elif args.command in ["up", "start", "start-dev"]:
+ # if docker-compose.yml does not exist, run update to create it
+ if not os.path.exists('docker-compose.yml'):
+ update_docker_images(LATEST)
run(split(
'docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions'),
silence_stderr=True)
From fc9fb41fc4b9bf2868f48adbd2524abeb11be2cd Mon Sep 17 00:00:00 2001
From: Maaike <maaike.limper@gmail.com>
Date: Mon, 13 Jan 2025 14:06:48 +0100
Subject: [PATCH 03/17] try out new test command
---
.github/workflows/tests-docker.yml | 8 ++------
1 file changed, 2 insertions(+), 6 deletions(-)
diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml
index 36f926c9..07ee6581 100644
--- a/.github/workflows/tests-docker.yml
+++ b/.github/workflows/tests-docker.yml
@@ -43,13 +43,9 @@ jobs:
sed -i "s/localhost/$IP/g" wis2box.env
cat wis2box.env
python3 wis2box-ctl.py config
- - name: build wis2box
+ - name: build wis2box and check status ⚙️
run: |
- python3 wis2box-ctl.py build
- python3 wis2box-ctl.py update
- - name: start containers ⚙️
- run: |
- python3 wis2box-ctl.py start
+ python3 wis2box-ctl.py update-local-build
python3 wis2box-ctl.py status -a
- name: show environment and check collections exist ⚙️
run: |
From 21ef25ec22852746daca0280b567773c3dc699d9 Mon Sep 17 00:00:00 2001
From: Maaike <maaike.limper@gmail.com>
Date: Mon, 13 Jan 2025 16:34:58 +0100
Subject: [PATCH 04/17] make different update commands, function for cleaning
images
---
wis2box-ctl.py | 153 +++++++++++++++++++++++++++++--------------------
1 file changed, 92 insertions(+), 61 deletions(-)
diff --git a/wis2box-ctl.py b/wis2box-ctl.py
index 14d88687..bef73c0b 100755
--- a/wis2box-ctl.py
+++ b/wis2box-ctl.py
@@ -25,6 +25,7 @@
import re
import requests
import subprocess
+import shutil
if subprocess.call(['docker', 'compose'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) > 0:
@@ -41,25 +42,17 @@
"""
parser = argparse.ArgumentParser(
- description='manage a compposition of docker containers to implement a wis 2 box',
- formatter_class=argparse.RawTextHelpFormatter)
+ description='Manage a composition of Docker containers to implement wis2box',
+ formatter_class=argparse.RawTextHelpFormatter
+)
-parser.add_argument(
- '--ssl',
- dest='ssl',
- action='store_true',
- help='run wis2box with SSL enabled')
-
-parser.add_argument(
- '--simulate',
- dest='simulate',
- action='store_true',
- help='simulate execution by printing action rather than executing')
+parser.add_argument('--simulate',
+ action='store_true',
+ help='Simulate execution by printing action rather than executing')
commands = [
'build',
'config',
- 'down',
'execute',
'lint',
'logs',
@@ -70,41 +63,54 @@
'start-dev',
'status',
'stop',
- 'up',
'update',
+ 'update-local-build',
+ 'update-latest-tags'
]
parser.add_argument('command',
choices=commands,
- help="""
+ help="""The command to execute:
- config: validate and view Docker configuration
- - build [containers]: build all services
- - start [containers]: start system
- - start-dev [containers]: start system in local development mode
- - login [container]: login to the container (default: wis2box-management)
- - login-root [container]: login to the container as root
- - stop: stop [container] system
+ - build: build all services
+ - start: start system
+ - start-dev: start system in local development mode
+ - login: login to the container (default: wis2box-management)
+ - stop: stop system
- update: update Docker images
- prune: cleanup dangling containers and images
- - restart [containers]: restart one or all containers
- - status [containers|-a]: view status of wis2box containers
+ - restart: restart containers
+ - status: view status of wis2box containers
- lint: run PEP8 checks against local Python code
""")
-parser.add_argument('args', nargs=argparse.REMAINDER)
+parser.add_argument('args', nargs=argparse.REMAINDER, help='Additional arguments for the command')
args = parser.parse_args()
-# define ENUM for docker image usage, LATEST or LOCAL
-LOCAL = 0
-LATEST = 1
-
LOCAL_IMAGES = [
'wis2box-management',
'wis2box-broker',
'wis2box-mqtt-metrics-collector'
]
+def remove_docker_images(filter: str) -> None:
+ # Get the IDs of images matching the filter
+ result = subprocess.run(
+ ['docker', 'images', '--filter', f'reference={filter}', '-q', '--no-trunc'],
+ capture_output=True,
+ text=True
+ )
+
+ image_ids = result.stdout.strip()
+ if image_ids: # If there are images to remove
+ for image_id in image_ids.splitlines():
+ try:
+ subprocess.run(['docker', 'rmi', image_id], check=True, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL)
+ except subprocess.CalledProcessError as e:
+ # do nothing
+ pass
+
def build_local_images() -> None:
"""
Build local images
@@ -163,30 +169,56 @@ def tag_sort_key(tag):
return tags[0]
-def update_docker_images(image_type: int) -> None:
+def update_docker_images(use_local_build: bool = False, use_latest: bool = False) -> None:
"""
Write docker-compose.yml using docker-compose.base.yml as base
- and set image-versions based on the latest available images in wmoim
-
- image_type: int. LATEST or LOCAL
+
+ use_local_build: required, boolean. If True, build local images
+ use_tags: required, boolean. If True, pull last tagged release for image. If False, use local images or tag='latest' # noqa
+
:returns: None.
"""
+ if os.path.exists('docker-compose.yml'):
+ print('Backing up current docker-compose.yml to docker-compose.yml.bak')
+ shutil.copy('docker-compose.yml', 'docker-compose.yml.bak')
+
+ if use_local_build:
+ print('Building local images')
+ build_local_images()
+
+ print('Updating docker-compose.yml')
+
with open('docker-compose.base.yml', 'r') as f:
lines = f.readlines()
-
with open('docker-compose.yml', 'w') as f:
for line in lines:
- if 'image: wmoim/' in line and image_type == LATEST:
+ if 'image: wmoim/' in line:
image = line.split('wmoim/')[1].split(':')[0]
- print(f'Get latest image tag for {image}')
- tag = get_latest_image_tag(image)
+
+ # determine the tag to use
+ tag = 'latest'
+ if image in LOCAL_IMAGES and use_local_build:
+ tag = 'local'
+ elif image not in LOCAL_IMAGES and use_local_build:
+ tag = 'latest'
+ elif not use_latest:
+ print(f'Get latest image tag for {image}')
+ tag = get_latest_image_tag(image)
+
+ # pull the image if it is not local, o
+ if tag != 'local':
+ print(f'Pulling wmoim/{image}:{tag}')
+ # pull the latest tag for the image
+ run(split(f'docker pull wmoim/{image}:{tag}'))
+
+ # update the image tag in the docker-compose.yml
print(f'Set {image} to {tag}')
f.write(f' image: wmoim/{image}:{tag}\n')
else:
f.write(line)
-
+ print('docker-compose.yml updated')
return None
def split(value: str) -> list:
@@ -251,11 +283,8 @@ def make(args) -> None:
if 'WIS2BOX_SSL_CERT' in line:
ssl_cert = line.split('=')[1].strip()
docker_compose_args = DOCKER_COMPOSE_ARGS
- if args.ssl or (ssl_key and ssl_cert):
+ if (ssl_key and ssl_cert):
docker_compose_args +=" --file docker-compose.ssl.yml"
- if args.ssl and not (ssl_key and ssl_cert):
- print("ERROR: SSL is enabled but WIS2BOX_SSL_KEY and WIS2BOX_SSL_CERT are not set in wis2box.env")
- exit(1)
# if you selected a bunch of them, default to all
containers = "" if not args.args else ' '.join(args.args)
@@ -264,13 +293,9 @@ def make(args) -> None:
if args.command == "config":
run(split(f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} config'))
- elif args.command == "build":
- build_local_images()
- update_docker_images(LOCAL)
elif args.command in ["up", "start", "start-dev"]:
- # if docker-compose.yml does not exist, run update to create it
if not os.path.exists('docker-compose.yml'):
- update_docker_images(LATEST)
+ update_docker_images()
run(split(
'docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions'),
silence_stderr=True)
@@ -282,6 +307,22 @@ def make(args) -> None:
run(split(f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} --file docker-compose.dev.yml up -d'))
else:
run(split(f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} up -d'))
+ elif args.command in ["update", "update-local-build", "update-latest-tags"]:
+ if args.command == "update-local-build":
+ update_docker_images(use_local_build=True)
+ elif args.command == "update-use-latest":
+ update_docker_images(use_local_build=False, use_latest=True)
+ else:
+ update_docker_images()
+ # restart all containers
+ run(split(
+ f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} down --remove-orphans'))
+ run(split(
+ f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} up -d'))
+ # perform cleanup of images after update, unless updating local build
+ if not args.command == "update-local-build":
+ remove_docker_images('wmoim/wis2*')
+ remove_docker_images('ghcr.io/wmo-im/wis2*')
elif args.command == "execute":
run(['docker', 'exec', '-i', 'wis2box-management', 'sh', '-c', containers])
elif args.command == "login":
@@ -297,24 +338,14 @@ def make(args) -> None:
else:
run(split(
f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} down --remove-orphans {containers}'))
- elif args.command == "update":
- update_docker_images(LATEST)
- # restart all containers
- run(split(
- f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} down --remove-orphans'))
- run(split(
- f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} up -d'))
- # prune dangling images
- _ = run(split('docker images --filter dangling=true -q --no-trunc'))
- run(split(f'docker rmi {_}'))
elif args.command == "prune":
run(split('docker builder prune -f'))
run(split('docker container prune -f'))
run( split('docker volume prune -f'))
- _ = run(split('docker images --filter dangling=true -q --no-trunc'))
- run(split(f'docker rmi {_}'))
- _ = run(split('docker ps -a -q'))
- run(split(f'docker rm {_}'))
+ # prune any unused images starting with wmoim/wis2
+ remove_docker_images('wmoim/wis2*')
+ # prune any unused images starting with ghcr.io/wmo-im/wis2
+ remove_docker_images('ghcr.io/wmo-im/wis2*')
elif args.command == "restart":
if containers:
run(split(
From 1db610039632bdb7d67954eafc2fe83a237f6466 Mon Sep 17 00:00:00 2001
From: Maaike <maaike.limper@gmail.com>
Date: Mon, 13 Jan 2025 16:40:10 +0100
Subject: [PATCH 05/17] update gitignore
---
.gitignore | 1 +
1 file changed, 1 insertion(+)
diff --git a/.gitignore b/.gitignore
index b9f3b024..e24b1508 100644
--- a/.gitignore
+++ b/.gitignore
@@ -49,3 +49,4 @@ tests/data/.ssh/id_rsa
tests/data/.ssh/id_rsa.pub
docker-compose.yml
+docker-compose.yml.bak
From 0166f93072e4c43271380bf912d846568447b22b Mon Sep 17 00:00:00 2001
From: Maaike <maaike.limper@gmail.com>
Date: Mon, 13 Jan 2025 16:53:49 +0100
Subject: [PATCH 06/17] update tests
---
.github/workflows/tests-docker.yml | 5 ++++-
.github/workflows/zaproxy.yml | 2 +-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml
index 07ee6581..41afca7b 100644
--- a/.github/workflows/tests-docker.yml
+++ b/.github/workflows/tests-docker.yml
@@ -43,9 +43,12 @@ jobs:
sed -i "s/localhost/$IP/g" wis2box.env
cat wis2box.env
python3 wis2box-ctl.py config
- - name: build wis2box and check status ⚙️
+ - name: build wis2box locally ⚙️
run: |
python3 wis2box-ctl.py update-local-build
+ - name: start containers ⚙️
+ run: |
+ python3 wis2box-ctl.py start
python3 wis2box-ctl.py status -a
- name: show environment and check collections exist ⚙️
run: |
diff --git a/.github/workflows/zaproxy.yml b/.github/workflows/zaproxy.yml
index a8796082..f9c43cbf 100644
--- a/.github/workflows/zaproxy.yml
+++ b/.github/workflows/zaproxy.yml
@@ -11,7 +11,7 @@ jobs:
- name: build and start containers using tests/test.env ⚙️
run: |
cp tests/test.env wis2box.env
- python3 wis2box-ctl.py build
+ python3 wis2box-ctl.py update-local-build
python3 wis2box-ctl.py start
python3 wis2box-ctl.py status -a
sleep 30
From f8593027cf1d64463e907fa70239f7294bcc7b2c Mon Sep 17 00:00:00 2001
From: Maaike <maaike.limper@gmail.com>
Date: Mon, 13 Jan 2025 17:11:48 +0100
Subject: [PATCH 07/17] updated command for quickstart
---
docs/source/reference/quickstart.rst | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/docs/source/reference/quickstart.rst b/docs/source/reference/quickstart.rst
index da6c6c05..d9236571 100644
--- a/docs/source/reference/quickstart.rst
+++ b/docs/source/reference/quickstart.rst
@@ -24,12 +24,11 @@ To run with the 'quickstart' configuration, copy this file to ``wis2box.env`` in
cp tests/test.env wis2box.env
-Build and update wis2box:
+Build and update wis2box from the source code:
.. code-block:: bash
- python3 wis2box-ctl.py build
- python3 wis2box-ctl.py update
+ python3 wis2box-ctl.py update-local-build
Start wis2box and login to the wis2box-management container:
From 46c4326a3d8effd9bb2262fe9f244668e94a98aa Mon Sep 17 00:00:00 2001
From: Maaike <maaike.limper@gmail.com>
Date: Tue, 14 Jan 2025 10:47:21 +0100
Subject: [PATCH 08/17] don't manage wis2downloader in wis2box-release as
requested by David Berry
---
docker-compose.base.yml | 2 +-
wis2box-ctl.py | 14 +++++++-------
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/docker-compose.base.yml b/docker-compose.base.yml
index 1758dc78..c755ec92 100644
--- a/docker-compose.base.yml
+++ b/docker-compose.base.yml
@@ -155,7 +155,7 @@ services:
wis2downloader:
container_name: wis2downloader
- image: wmoim/wis2downloader:latest
+ image: wmoim/wis2downloader:v0.3.2
restart: always
env_file:
- wis2box.env
diff --git a/wis2box-ctl.py b/wis2box-ctl.py
index bef73c0b..8616322f 100755
--- a/wis2box-ctl.py
+++ b/wis2box-ctl.py
@@ -194,7 +194,7 @@ def update_docker_images(use_local_build: bool = False, use_latest: bool = False
lines = f.readlines()
with open('docker-compose.yml', 'w') as f:
for line in lines:
- if 'image: wmoim/' in line:
+ if 'image: wmoim/wis2box' in line:
image = line.split('wmoim/')[1].split(':')[0]
# determine the tag to use
@@ -321,8 +321,8 @@ def make(args) -> None:
f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} up -d'))
# perform cleanup of images after update, unless updating local build
if not args.command == "update-local-build":
- remove_docker_images('wmoim/wis2*')
- remove_docker_images('ghcr.io/wmo-im/wis2*')
+ remove_docker_images('wmoim/wis2box*')
+ remove_docker_images('ghcr.io/wmo-im/wis2box*')
elif args.command == "execute":
run(['docker', 'exec', '-i', 'wis2box-management', 'sh', '-c', containers])
elif args.command == "login":
@@ -342,10 +342,10 @@ def make(args) -> None:
run(split('docker builder prune -f'))
run(split('docker container prune -f'))
run( split('docker volume prune -f'))
- # prune any unused images starting with wmoim/wis2
- remove_docker_images('wmoim/wis2*')
- # prune any unused images starting with ghcr.io/wmo-im/wis2
- remove_docker_images('ghcr.io/wmo-im/wis2*')
+ # prune any unused images starting with wmoim/wis2box
+ remove_docker_images('wmoim/wis2box*')
+ # prune any unused images starting with ghcr.io/wmo-im/wis2box
+ remove_docker_images('ghcr.io/wmo-im/wis2box*')
elif args.command == "restart":
if containers:
run(split(
From 3018a20cb5165e687383c70a01ef3ac30f14af8e Mon Sep 17 00:00:00 2001
From: Maaike <maaike.limper@gmail.com>
Date: Tue, 14 Jan 2025 10:54:00 +0100
Subject: [PATCH 09/17] set ref of wis2downloader back to original
---
docker-compose.base.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docker-compose.base.yml b/docker-compose.base.yml
index c755ec92..67b75d03 100644
--- a/docker-compose.base.yml
+++ b/docker-compose.base.yml
@@ -155,7 +155,7 @@ services:
wis2downloader:
container_name: wis2downloader
- image: wmoim/wis2downloader:v0.3.2
+ image: ghcr.io/wmo-im/wis2downloader:v0.3.2
restart: always
env_file:
- wis2box.env
From bfefbf691f26540c33b6c3724d7afcbf729972d5 Mon Sep 17 00:00:00 2001
From: Maaike <maaike.limper@gmail.com>
Date: Tue, 14 Jan 2025 17:34:18 +0100
Subject: [PATCH 10/17] use github release to determine tags ...
---
docker-compose.base.yml | 14 ++---
wis2box-ctl.py | 130 +++++++++++++++++-----------------------
wis2box.version | 1 +
3 files changed, 63 insertions(+), 82 deletions(-)
create mode 100644 wis2box.version
diff --git a/docker-compose.base.yml b/docker-compose.base.yml
index 67b75d03..8951accb 100644
--- a/docker-compose.base.yml
+++ b/docker-compose.base.yml
@@ -13,7 +13,7 @@ services:
wis2box-ui:
container_name: wis2box-ui
- image: wmoim/wis2box-ui:latest
+ image: ghcr.io/wmo-im/wis2box-ui:latest
restart: always
env_file:
- wis2box.env
@@ -22,14 +22,14 @@ services:
wis2box-webapp:
container_name: wis2box-webapp
- image: wmoim/wis2box-webapp:latest
+ image: ghcr.io/wmo-im/wis2box-webapp:latest
env_file:
- wis2box.env
restart: always
wis2box-api:
container_name: wis2box-api
- image: wmoim/wis2box-api:latest
+ image: ghcr.io/wmo-im/wis2box-api:latest
restart: always
env_file:
- wis2box.env
@@ -104,7 +104,7 @@ services:
mosquitto:
container_name: mosquitto
- image: wmoim/wis2box-broker:local
+ image: ghcr.io/wmo-im/wis2box-broker:latest
restart: always
env_file:
- wis2box.env
@@ -113,7 +113,7 @@ services:
wis2box-management:
container_name: wis2box-management
- image: wmoim/wis2box-management:local
+ image: ghcr.io/wmo-im/wis2box-management:latest
mem_limit: 1g
memswap_limit: 1g
restart: always
@@ -135,7 +135,7 @@ services:
restart: unless-stopped
env_file:
- wis2box.env
- image: wmoim/wis2box-mqtt-metrics-collector:local
+ image: ghcr.io/wmo-im/wis2box-mqtt-metrics-collector:latest
depends_on:
- mosquitto
- wis2box-management
@@ -144,7 +144,7 @@ services:
wis2box-auth:
container_name: wis2box-auth
- image: wmoim/wis2box-auth:latest
+ image: ghcr.io/wmo-im/wis2box-auth:latest
restart: always
env_file:
- wis2box.env
diff --git a/wis2box-ctl.py b/wis2box-ctl.py
index 8616322f..3728e3b0 100755
--- a/wis2box-ctl.py
+++ b/wis2box-ctl.py
@@ -27,6 +27,11 @@
import subprocess
import shutil
+# read wis2box.version file if it exists
+WIS2BOX_VERSION = 'LOCAL_BUILD'
+if os.path.exists('wis2box.version'):
+ with open('wis2box.version', 'r') as f:
+ WIS2BOX_VERSION = f.read().strip()
if subprocess.call(['docker', 'compose'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) > 0:
DOCKER_COMPOSE_COMMAND = 'docker-compose'
@@ -63,9 +68,7 @@
'start-dev',
'status',
'stop',
- 'update',
- 'update-local-build',
- 'update-latest-tags'
+ 'update'
]
parser.add_argument('command',
@@ -119,63 +122,52 @@ def build_local_images() -> None:
"""
for image in LOCAL_IMAGES:
print(f'Building {image}')
- run(split(f'docker build -t wmoim/{image}:local {image}'))
-
+ run(split(f'docker build -t ghcr.io/wmo-im/{image}:local {image}'))
return None
-
-def get_latest_image_tag(image: str) -> str:
+
+def get_latest_release_tag(image: str, major_release: str = '') -> str:
"""
- list image tags by querying docker hub api
- skip the 'latest' tag
- return the most recent tag
- :param image: required, string. Name of the image.
+ Fetches the latest release tag for a GitHub repository.
+
+ :param image: required, string. The name of the image repository.
+ :param major_release: required, string. The major release version.
- :returns: string. The most recent tag.
+ :return: The latest release tag or an error message if not found.
"""
-
- url = f"https://hub.docker.com/v2/repositories/wmoim/{image}/tags/"
- tags = []
- try:
- # Paginate through results to collect all tags
- while url:
- response = requests.get(url)
- response.raise_for_status() # Raise an error for HTTP failures
- data = response.json()
- tags.extend([
- tag['name'] for tag in data.get('results', [])
- if tag['name'] != 'latest' # Skip 'latest' tag
- ])
- url = data.get('next') # Get the next page URL
- except requests.RequestException as e:
- raise RuntimeError(f"Failed to fetch tags for image '{image}': {e}")
-
- if not tags:
- raise ValueError(f"No valid tags found for image '{image}'")
- else:
- print(f"Found {len(tags)} tags for image '{image}: {tags}'")
-
- # define a function to sort tags by version number
- def tag_sort_key(tag):
- # Extract numeric and non-numeric parts
- parts = re.split(r'(\d+)', tag) # Split into numeric and non-numeric segments
- return [
- int(part) if part.isdigit() else 'ZZZZZZZZZZ' # Use a large number for non-numeric parts
- for part in parts
- ]
-
- # Sort tags by version number in descending order
- tags.sort(key=tag_sort_key, reverse=True)
- return tags[0]
+ url = f'https://api.github.com/repos/wmo-im/{image}/releases'
+ headers = {'Accept': 'application/vnd.github.v3+json'}
+ print(f'Fetching latest release tag from GitHub for {image} using wis2box-version={major_release}')
-def update_docker_images(use_local_build: bool = False, use_latest: bool = False) -> None:
+ options = []
+ try:
+ response = requests.get(url, headers=headers)
+ if response.status_code == 200:
+ releases = response.json()
+ for release in releases:
+ print(release['tag_name'])
+ if major_release in release['tag_name']:
+ options.append(release['tag_name'])
+ else:
+ print(f'Error fetching latest release tag for {image}: {response.status_code}')
+ except requests.exceptions.RequestException as e:
+ print(f'Error fetching latest release tag for {image}: {e}')
+
+ # throw error if options is empty
+ if not options:
+ raise ValueError(f'No release tags found for {image} with major release {major_release}')
+
+ # sort descending and return the first element
+ options.sort(reverse=True)
+ return options[0]
+
+def update_docker_images(wis2box_version: str) -> None:
"""
Write docker-compose.yml using docker-compose.base.yml as base
- use_local_build: required, boolean. If True, build local images
- use_tags: required, boolean. If True, pull last tagged release for image. If False, use local images or tag='latest' # noqa
+ :param wis2box_version: required, string. The version of wis2box to use.
:returns: None.
"""
@@ -184,38 +176,31 @@ def update_docker_images(use_local_build: bool = False, use_latest: bool = False
print('Backing up current docker-compose.yml to docker-compose.yml.bak')
shutil.copy('docker-compose.yml', 'docker-compose.yml.bak')
- if use_local_build:
+ if wis2box_version == 'LOCAL_BUILD':
print('Building local images')
build_local_images()
- print('Updating docker-compose.yml')
+ print(f'Updating docker-compose.yml for wis2box_version={wis2box_version}')
with open('docker-compose.base.yml', 'r') as f:
lines = f.readlines()
with open('docker-compose.yml', 'w') as f:
for line in lines:
- if 'image: wmoim/wis2box' in line:
- image = line.split('wmoim/')[1].split(':')[0]
-
- # determine the tag to use
+ if 'image: ghcr.io/wmo-im/wis2box' in line:
+ image = line.split('ghcr.io/wmo-im/')[1].split(':')[0]
tag = 'latest'
- if image in LOCAL_IMAGES and use_local_build:
+ if image in LOCAL_IMAGES and wis2box_version == 'LOCAL_BUILD':
tag = 'local'
- elif image not in LOCAL_IMAGES and use_local_build:
- tag = 'latest'
- elif not use_latest:
- print(f'Get latest image tag for {image}')
- tag = get_latest_image_tag(image)
-
- # pull the image if it is not local, o
+ elif wis2box_version != 'LOCAL_BUILD':
+ tag = get_latest_release_tag(image, wis2box_version)
+ # pull the image if it is not local
if tag != 'local':
- print(f'Pulling wmoim/{image}:{tag}')
+ print(f'Pulling ghcr.io/wmo-im/{image}:{tag}')
# pull the latest tag for the image
- run(split(f'docker pull wmoim/{image}:{tag}'))
-
+ run(split(f'docker pull ghcr.io/wmo-im/{image}:{tag}'))
# update the image tag in the docker-compose.yml
print(f'Set {image} to {tag}')
- f.write(f' image: wmoim/{image}:{tag}\n')
+ f.write(f' image: ghcr.io/wmo-im/{image}:{tag}\n')
else:
f.write(line)
print('docker-compose.yml updated')
@@ -307,20 +292,15 @@ def make(args) -> None:
run(split(f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} --file docker-compose.dev.yml up -d'))
else:
run(split(f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} up -d'))
- elif args.command in ["update", "update-local-build", "update-latest-tags"]:
- if args.command == "update-local-build":
- update_docker_images(use_local_build=True)
- elif args.command == "update-use-latest":
- update_docker_images(use_local_build=False, use_latest=True)
- else:
- update_docker_images()
+ elif args.command in ["update"]:
+ update_docker_images(WIS2BOX_VERSION)
# restart all containers
run(split(
f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} down --remove-orphans'))
run(split(
f'{DOCKER_COMPOSE_COMMAND} {docker_compose_args} up -d'))
# perform cleanup of images after update, unless updating local build
- if not args.command == "update-local-build":
+ if not WIS2BOX_VERSION == 'LOCAL_BUILD':
remove_docker_images('wmoim/wis2box*')
remove_docker_images('ghcr.io/wmo-im/wis2box*')
elif args.command == "execute":
diff --git a/wis2box.version b/wis2box.version
new file mode 100644
index 00000000..452a44d4
--- /dev/null
+++ b/wis2box.version
@@ -0,0 +1 @@
+LOCAL_BUILD
\ No newline at end of file
From dbb016970f11cfc845b754b062c5d90fd9d4ef99 Mon Sep 17 00:00:00 2001
From: Maaike <maaike.limper@gmail.com>
Date: Tue, 14 Jan 2025 17:35:51 +0100
Subject: [PATCH 11/17] zaproxy fix for updated PR
---
.github/workflows/zaproxy.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/zaproxy.yml b/.github/workflows/zaproxy.yml
index f9c43cbf..a016435f 100644
--- a/.github/workflows/zaproxy.yml
+++ b/.github/workflows/zaproxy.yml
@@ -11,7 +11,7 @@ jobs:
- name: build and start containers using tests/test.env ⚙️
run: |
cp tests/test.env wis2box.env
- python3 wis2box-ctl.py update-local-build
+ python3 wis2box-ctl.py update
python3 wis2box-ctl.py start
python3 wis2box-ctl.py status -a
sleep 30
From 5823fc36e3dec6315e1dcd885fb771a145d560c1 Mon Sep 17 00:00:00 2001
From: Maaike <maaike.limper@gmail.com>
Date: Tue, 14 Jan 2025 17:37:43 +0100
Subject: [PATCH 12/17] commit update tests-docker
---
.github/workflows/tests-docker.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml
index 41afca7b..3a7a3bb4 100644
--- a/.github/workflows/tests-docker.yml
+++ b/.github/workflows/tests-docker.yml
@@ -45,7 +45,7 @@ jobs:
python3 wis2box-ctl.py config
- name: build wis2box locally ⚙️
run: |
- python3 wis2box-ctl.py update-local-build
+ python3 wis2box-ctl.py update
- name: start containers ⚙️
run: |
python3 wis2box-ctl.py start
From cd9aa8f15093cea15c0cd4dca23013b9eccb0185 Mon Sep 17 00:00:00 2001
From: Maaike <maaike.limper@gmail.com>
Date: Tue, 14 Jan 2025 17:42:11 +0100
Subject: [PATCH 13/17] rename major_release to wis2box_version for consistency
---
wis2box-ctl.py | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/wis2box-ctl.py b/wis2box-ctl.py
index 3728e3b0..31bc067f 100755
--- a/wis2box-ctl.py
+++ b/wis2box-ctl.py
@@ -125,20 +125,20 @@ def build_local_images() -> None:
run(split(f'docker build -t ghcr.io/wmo-im/{image}:local {image}'))
return None
-def get_latest_release_tag(image: str, major_release: str = '') -> str:
+def get_latest_release_tag(image: str, wis2box_version: str = '') -> str:
"""
Fetches the latest release tag for a GitHub repository.
:param image: required, string. The name of the image repository.
- :param major_release: required, string. The major release version.
+ :param wis2box_version: required, string. The major release version.
:return: The latest release tag or an error message if not found.
"""
url = f'https://api.github.com/repos/wmo-im/{image}/releases'
headers = {'Accept': 'application/vnd.github.v3+json'}
- print(f'Fetching latest release tag from GitHub for {image} using wis2box-version={major_release}')
+ print(f'Fetching latest release tag from GitHub for {image} using wis2box-version={wis2box_version}')
options = []
try:
@@ -147,7 +147,7 @@ def get_latest_release_tag(image: str, major_release: str = '') -> str:
releases = response.json()
for release in releases:
print(release['tag_name'])
- if major_release in release['tag_name']:
+ if wis2box_version in release['tag_name']:
options.append(release['tag_name'])
else:
print(f'Error fetching latest release tag for {image}: {response.status_code}')
@@ -156,7 +156,7 @@ def get_latest_release_tag(image: str, major_release: str = '') -> str:
# throw error if options is empty
if not options:
- raise ValueError(f'No release tags found for {image} with major release {major_release}')
+ raise ValueError(f'No release tags found for {image} with major release {wis2box_version}')
# sort descending and return the first element
options.sort(reverse=True)
From 359b5d4744e94de6045c95718e1fcfeac8142db7 Mon Sep 17 00:00:00 2001
From: Maaike <maaike.limper@gmail.com>
Date: Thu, 16 Jan 2025 14:47:20 +0100
Subject: [PATCH 14/17] use separate release referencing images
---
.github/workflows/tests-docker.yml | 3 +-
wis2box-ctl.py | 96 ++++++++++++++++++++----------
2 files changed, 66 insertions(+), 33 deletions(-)
diff --git a/.github/workflows/tests-docker.yml b/.github/workflows/tests-docker.yml
index 3a7a3bb4..dcdaf6eb 100644
--- a/.github/workflows/tests-docker.yml
+++ b/.github/workflows/tests-docker.yml
@@ -36,10 +36,11 @@ jobs:
mkdir -p tests/data/downloads
chown -R $(whoami):docker tests/data/downloads
chmod -R 775 tests/data/downloads
- - name: setup wis2box configuration, replace localhost with IP on host 📦
+ - name: setup wis2box configuration, replace localhost with IP on host, use LOCAL_BUILD as wis2box.version 📦
run: |
export IP=$(hostname -I | awk '{print $1}')
cp tests/test.env wis2box.env
+ echo "LOCAL_BUILD" > wis2box.version
sed -i "s/localhost/$IP/g" wis2box.env
cat wis2box.env
python3 wis2box-ctl.py config
diff --git a/wis2box-ctl.py b/wis2box-ctl.py
index 31bc067f..d25e67bf 100755
--- a/wis2box-ctl.py
+++ b/wis2box-ctl.py
@@ -22,11 +22,12 @@
import argparse
import os
-import re
import requests
import subprocess
import shutil
+from packaging.version import Version
+
# read wis2box.version file if it exists
WIS2BOX_VERSION = 'LOCAL_BUILD'
if os.path.exists('wis2box.version'):
@@ -92,9 +93,9 @@
args = parser.parse_args()
LOCAL_IMAGES = [
- 'wis2box-management',
- 'wis2box-broker',
- 'wis2box-mqtt-metrics-collector'
+ 'ghcr.io/wmo-im/wis2box-management',
+ 'ghcr.io/wmo-im/wis2box-broker',
+ 'ghcr.io/wmo-im/wis2box-mqtt-metrics-collector'
]
def remove_docker_images(filter: str) -> None:
@@ -125,49 +126,76 @@ def build_local_images() -> None:
run(split(f'docker build -t ghcr.io/wmo-im/{image}:local {image}'))
return None
-def get_latest_release_tag(image: str, wis2box_version: str = '') -> str:
+def get_resolved_version(base_version: str) -> str:
"""
+ Fetches the latest release tag for the wis2box-images repository.
- Fetches the latest release tag for a GitHub repository.
-
- :param image: required, string. The name of the image repository.
- :param wis2box_version: required, string. The major release version.
+ :rbase_version: required, string. The major release version.
:return: The latest release tag or an error message if not found.
"""
- url = f'https://api.github.com/repos/wmo-im/{image}/releases'
+
+ # NOTE using maaikelimper/wis2box-images for demo purposes, should be wmo-im/wis2box-images
+ url = f'https://api.github.com/repos/maaikelimper/wis2box-images/releases'
headers = {'Accept': 'application/vnd.github.v3+json'}
- print(f'Fetching latest release tag from GitHub for {image} using wis2box-version={wis2box_version}')
-
options = []
try:
response = requests.get(url, headers=headers)
if response.status_code == 200:
releases = response.json()
for release in releases:
- print(release['tag_name'])
- if wis2box_version in release['tag_name']:
+ if base_version in release['tag_name']:
options.append(release['tag_name'])
else:
- print(f'Error fetching latest release tag for {image}: {response.status_code}')
+ print(f'Error fetching latest release tag for {base_version}: {response.status_code}')
except requests.exceptions.RequestException as e:
- print(f'Error fetching latest release tag for {image}: {e}')
+ print(f'Error fetching latest release tag for {base_version}: {e}')
# throw error if options is empty
if not options:
- raise ValueError(f'No release tags found for {image} with major release {wis2box_version}')
+ raise ValueError(f'No wis2box-release found matching wis2box-version={base_version}')
- # sort descending and return the first element
- options.sort(reverse=True)
- return options[0]
+ # Use semantic versioning for sorting
+ sorted_versions = sorted(options, key=lambda v: Version(v), reverse=True)
+
+ resolved_version = sorted_versions[0]
+ return resolved_version
+
+def get_latest_release_tag(image: str, resolved_version: str = '') -> str:
+ """
+
+ Fetches the latest release tag for a GitHub repository.
+
+ :param image: required, string. The name of the image repository.
+ :param base_version: required, string. The major release version.
+
+ :return: The latest release tag or an error message if not found.
+ """
+
+ # look up the image tag for the release by downloading the wis2box-images.json file
+ # NOTE using maaikelimper/wis2box-images for demo purposes, should be wmo-im/wis2box-images
+ url = f'https://github.com/maaikelimper/wis2box-images/releases/download/{resolved_version}/wis2box-images.json'
+ try:
+ response = requests.get(url)
+ if response.status_code == 200:
+ images = response.json()
+ if image in images:
+ return images[image]
+ else:
+ print(f'Error fetching image tag for {image} from {url}: {response.status_code}')
+ except requests.exceptions.RequestException as e:
+ print(f'Error fetching image tag for {image} from {url}: {e}')
+
+ # throw error if image tag is not found
+ raise ValueError(f'No image tag found for {image} in {url}')
-def update_docker_images(wis2box_version: str) -> None:
+def update_docker_images(base_version: str) -> None:
"""
Write docker-compose.yml using docker-compose.base.yml as base
- :param wis2box_version: required, string. The version of wis2box to use.
+ :param base_version: required, string. The version of wis2box to use.
:returns: None.
"""
@@ -176,31 +204,35 @@ def update_docker_images(wis2box_version: str) -> None:
print('Backing up current docker-compose.yml to docker-compose.yml.bak')
shutil.copy('docker-compose.yml', 'docker-compose.yml.bak')
- if wis2box_version == 'LOCAL_BUILD':
+ if base_version == 'LOCAL_BUILD':
print('Building local images')
build_local_images()
- print(f'Updating docker-compose.yml for wis2box_version={wis2box_version}')
+ resolved_version = base_version
+ if base_version != 'LOCAL_BUILD':
+ resolved_version = get_resolved_version(base_version)
+
+ print(f'Updating docker-compose.yml, using base_version={resolved_version}')
with open('docker-compose.base.yml', 'r') as f:
lines = f.readlines()
with open('docker-compose.yml', 'w') as f:
for line in lines:
- if 'image: ghcr.io/wmo-im/wis2box' in line:
- image = line.split('ghcr.io/wmo-im/')[1].split(':')[0]
+ if 'image: ' in line:
+ image = line.split('image: ')[1].split(':')[0]
tag = 'latest'
- if image in LOCAL_IMAGES and wis2box_version == 'LOCAL_BUILD':
+ if image in LOCAL_IMAGES and base_version == 'LOCAL_BUILD':
tag = 'local'
- elif wis2box_version != 'LOCAL_BUILD':
- tag = get_latest_release_tag(image, wis2box_version)
+ elif base_version != 'LOCAL_BUILD':
+ tag = get_latest_release_tag(image, resolved_version)
# pull the image if it is not local
if tag != 'local':
- print(f'Pulling ghcr.io/wmo-im/{image}:{tag}')
+ print(f'Pulling {image}:{tag}')
# pull the latest tag for the image
- run(split(f'docker pull ghcr.io/wmo-im/{image}:{tag}'))
+ run(split(f'docker pull {image}:{tag}'))
# update the image tag in the docker-compose.yml
print(f'Set {image} to {tag}')
- f.write(f' image: ghcr.io/wmo-im/{image}:{tag}\n')
+ f.write(f' image: {image}:{tag}\n')
else:
f.write(line)
print('docker-compose.yml updated')
From 5e7e2c026f6f08bfcb7622019a7b7a4e88ee531d Mon Sep 17 00:00:00 2001
From: Maaike <maaike.limper@gmail.com>
Date: Thu, 16 Jan 2025 15:31:36 +0100
Subject: [PATCH 15/17] fix build images
---
wis2box-ctl.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/wis2box-ctl.py b/wis2box-ctl.py
index d25e67bf..3c23e2c6 100755
--- a/wis2box-ctl.py
+++ b/wis2box-ctl.py
@@ -121,9 +121,11 @@ def build_local_images() -> None:
:returns: None.
"""
+
for image in LOCAL_IMAGES:
print(f'Building {image}')
- run(split(f'docker build -t ghcr.io/wmo-im/{image}:local {image}'))
+ context = image.split('/')[-1]
+ run(split(f'docker build -t {image}:local {context}'))
return None
def get_resolved_version(base_version: str) -> str:
From 0b0748abb743084a74de8abdc5dadaa34dc86f47 Mon Sep 17 00:00:00 2001
From: Maaike <maaike.limper@gmail.com>
Date: Thu, 16 Jan 2025 15:56:37 +0100
Subject: [PATCH 16/17] fix update-flow for LOCAL_BUILD
---
wis2box-ctl.py | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/wis2box-ctl.py b/wis2box-ctl.py
index 3c23e2c6..1405bfd2 100755
--- a/wis2box-ctl.py
+++ b/wis2box-ctl.py
@@ -177,7 +177,12 @@ def get_latest_release_tag(image: str, resolved_version: str = '') -> str:
# look up the image tag for the release by downloading the wis2box-images.json file
# NOTE using maaikelimper/wis2box-images for demo purposes, should be wmo-im/wis2box-images
- url = f'https://github.com/maaikelimper/wis2box-images/releases/download/{resolved_version}/wis2box-images.json'
+ github_repo = 'maaikelimper/wis2box-images'
+
+ url = f'https://github.com/{github_repo}/releases/download/{resolved_version}/wis2box-images.json'
+ if resolved_version == 'LOCAL_BUILD':
+ # use the version of images in main branch
+ url = f'https://raw.githubusercontent.com/{github_repo}/refs/heads/main/wis2box-images.json'
try:
response = requests.get(url)
if response.status_code == 200:
@@ -222,10 +227,10 @@ def update_docker_images(base_version: str) -> None:
for line in lines:
if 'image: ' in line:
image = line.split('image: ')[1].split(':')[0]
- tag = 'latest'
+ tag = ''
if image in LOCAL_IMAGES and base_version == 'LOCAL_BUILD':
tag = 'local'
- elif base_version != 'LOCAL_BUILD':
+ else:
tag = get_latest_release_tag(image, resolved_version)
# pull the image if it is not local
if tag != 'local':
From 781996d5f83c511d3d11553bef4528a18aff54f7 Mon Sep 17 00:00:00 2001
From: Maaike <maaike.limper@gmail.com>
Date: Thu, 16 Jan 2025 16:40:41 +0100
Subject: [PATCH 17/17] update zap rules, ignore new warning with low risk
---
.zap/rules.tsv | 1 +
1 file changed, 1 insertion(+)
diff --git a/.zap/rules.tsv b/.zap/rules.tsv
index fcdad969..dc50cafa 100644
--- a/.zap/rules.tsv
+++ b/.zap/rules.tsv
@@ -22,3 +22,4 @@
10110 IGNORE Dangerous JS Functions Low
10105 IGNORE Authentication Credentials Captured Medium
10003 IGNORE Vulnerable JS Library Medium
+90004 IGNORE Insufficient Site Isolation Against Spectre Vulnerability Low