Merge pull request #1563 from ScilifelabDataCentre/dev

New release: v2.8.1

.github/workflows/publish_and_trivyscan.yml

@@ -96,12 +96,12 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           name: technical-overview-pdf
-          path: dds_web/static/dds-technical-overview.pdf
+          path: dds_web/static
       - name: Download troubleshooting PDF
         uses: actions/download-artifact@v4
         with:
           name: troubleshooting-pdf
-          path: dds_web/static/dds-troubleshooting.pdf
+          path: dds_web/static
       - name: Docker metadata
         id: meta
         uses: docker/metadata-action@v5
@@ -111,14 +111,16 @@ jobs:
       - name: Ensure lowercase name
         run: echo IMAGE_REPOSITORY=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV
       - name: Build for scan
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           file: Dockerfiles/backend.Dockerfile
           context: .
           push: false
           tags: ghcr.io/${{ env.IMAGE_REPOSITORY }}:sha-${{ github.sha }}
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/[email protected]
+        uses: aquasecurity/[email protected]
+        env:
+          TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db,public.ecr.aws/aquasecurity/trivy-db
         with:
           image-ref: "ghcr.io/${{ env.IMAGE_REPOSITORY }}:sha-${{ github.sha }}"
           format: "sarif"
@@ -130,7 +132,7 @@ jobs:
           sarif_file: "trivy-results.sarif"
           category: trivy-build
       - name: Publish image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           file: Dockerfiles/backend.Dockerfile
           context: .

.github/workflows/trivy-scheduled-dev.yml

@@ -23,7 +23,9 @@ jobs:
         run: echo REPOSITORY_OWNER=$(echo ${{ github.repository_owner }} | tr "[:upper:]" "[:lower:]") >> $GITHUB_ENV
 
       - name: Run Trivy on latest dev image
-        uses: aquasecurity/[email protected]
+        uses: aquasecurity/[email protected]
+        env:
+          TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db,public.ecr.aws/aquasecurity/trivy-db
         with:
           image-ref: "ghcr.io/${{ env.REPOSITORY_OWNER }}/dds-backend:dev"
           format: "sarif"

.github/workflows/trivy-scheduled-master.yml

@@ -25,7 +25,9 @@ jobs:
         run: echo REPOSITORY_OWNER=$(echo ${{ github.repository_owner }} | tr "[:upper:]" "[:lower:]") >> $GITHUB_ENV
 
       - name: Run Trivy on latest release image
-        uses: aquasecurity/[email protected]
+        uses: aquasecurity/[email protected]
+        env:
+          TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db,public.ecr.aws/aquasecurity/trivy-db
         with:
           image-ref: "ghcr.io/${{ env.REPOSITORY_OWNER }}/dds-backend:latest"
           format: "sarif"

CHANGELOG.rst

@@ -1,6 +1,17 @@
 Changelog
 ==========
 
+.. _2.8.1:
+
+2.8.1 - 2024-10-23
+~~~~~~~~~~~~~~~~~~~~~~~
+
+- New features:
+    - warning_level option when a unit is created defaults to 0.8.
+    - Add option to MOTD endpoint to send an email to unit users only.
+    - Modify the invoicing commands to send the instance name in the emails.
+- Documentation: Update readme to indicate that the backend image is published to GGCR, not DockerHub.
+
 .. _2.8.0:
 
 2.8.0 - 2024-09-24

README.md

@@ -257,7 +257,7 @@ Equally, if you want to tear down you need to run pytest _twice_ without it, as
 
 ## Production Instance
 
-The production version of the backend image is published at [Dockerhub](https://hub.docker.com/repository/docker/scilifelabdatacentre/dds-backend). It can also be built by running:
+The production version of the backend image is published at the [GitHub Container Registry (GHCR)](ghcr.io/scilifelabdatacentre/dds-backend). It can also be built by running:
 
 ```bash
 docker build --target production -f Dockerfiles/backend.Dockerfile .

SPRINTLOG.md

@@ -430,3 +430,17 @@ _Nothing merged during this sprint_
 
 - Flask command to update unit quotas ([#1551](https://github.com/ScilifelabDataCentre/dds_web/pull/1551))
 - Bump python base image to 3.12 and related libraries in both web and client([#1548](https://github.com/ScilifelabDataCentre/dds_web/pull/1548))
+- Warning_level option defaults to 0.8([#1557](https://github.com/ScilifelabDataCentre/dds_web/pull/1557))
+
+# 2024-09-24 - 2024-10-04
+
+- Add option to motd command for sending to unit users only([#1552](https://github.com/ScilifelabDataCentre/dds_web/pull/1552))
+
+# 2024-10-07 - 2024-10-18
+
+- Update readme: backend image is published to GHCR, not DockerHub ([#1558](https://github.com/ScilifelabDataCentre/dds_web/pull/1558))
+- Workflow bug fixed: PDFs (Technical Overview and Troubleshooting) were downloaded to incorrect directory([#1559](https://github.com/ScilifelabDataCentre/dds_web/pull/1559))
+- Update trivy action and add a second mirror repository to reduce TOO MANY REQUEST issue([#1560](https://github.com/ScilifelabDataCentre/dds_web/pull/1560))
+- Modify the invoicing commands to send the instance name in the emails([#1561](https://github.com/ScilifelabDataCentre/dds_web/pull/1561))
+- Fix the MOTD endpoint according to post merge review([#1564](https://github.com/ScilifelabDataCentre/dds_web/pull/1564))
+- New version & changelog([#1565](https://github.com/ScilifelabDataCentre/dds_web/pull/1565))

dds_web/__init__.py

@@ -171,6 +171,10 @@ def create_app(testing=False, database_uri=None):
         # Initiate app object
         app = flask.Flask(__name__, instance_relative_config=False)
 
+        # All variables in the env that start with FLASK_* will be loaded into the app config
+        # 'FLASK_' will be dropped, e.g. FLASK_TESTVAR will be loaded as TESTVAR
+        app.config.from_prefixed_env()
+
         # Default development config
         app.config.from_object("dds_web.config.Config")
 

dds_web/api/superadmin_only.py

@@ -163,6 +163,15 @@ def post(self):
         if not motd_obj or not motd_obj.active:
             raise ddserr.DDSArgumentError(message=f"There is no active MOTD with ID '{motd_id}'.")
 
+        # check if sent to unit users only or all users
+        unit_only: bool = request_json.get("unit_only", False)
+        if not isinstance(unit_only, bool):
+            raise ddserr.DDSArgumentError(message="The 'unit_only' argument must be a boolean.")
+        if unit_only:
+            users_to_send = db.session.query(models.UnitUser)
+        else:
+            users_to_send = db.session.query(models.User)
+
         # Create email content
         # put motd_obj.message etc in there etc
         subject: str = "Important Information: Data Delivery System"
@@ -172,7 +181,7 @@ def post(self):
         # Setup email connection
         with mail.connect() as conn:
             # Email users
-            for user in utils.page_query(db.session.query(models.User)):
+            for user in utils.page_query(users_to_send):
                 primary_email = user.primary_email
                 if not primary_email:
                     flask.current_app.logger.warning(
@@ -197,7 +206,13 @@ def post(self):
                 # Send email
                 utils.send_email_with_retry(msg=msg, obj=conn)
 
-        return {"message": f"MOTD '{motd_id}' has been sent to the users."}
+        return_msg = f"MOTD '{motd_id}' has been "
+        if unit_only:
+            return_msg += "sent to unit personnel only."
+        else:
+            return_msg += "sent to all users."
+
+        return {"message": return_msg}
 
 
 class FindUser(flask_restful.Resource):

dds_web/commands.py

@@ -85,7 +85,7 @@ def fill_db_wrapper(db_type):
 @click.option("--days_in_available", "-da", type=int, required=False, default=90)
 @click.option("--days_in_expired", "-de", type=int, required=False, default=30)
 @click.option("--quota", "-q", type=int, required=True)
-@click.option("--warn-at", "-w", type=int, required=False, default=80)
+@click.option("--warn-at", "-w", type=click.FloatRange(0.0, 1.0), required=False, default=0.8)
 @flask.cli.with_appcontext
 def create_new_unit(
     name,
@@ -841,16 +841,23 @@ def monthly_usage():
         send_email_with_retry,
     )
 
+    # Get the instance name (DEVELOPMENT, PRODUCTION, etc.)
+    instance_name = flask.current_app.config.get("INSTANCE_NAME")
+
     # Email settings
     email_recipient: str = flask.current_app.config.get("MAIL_DDS")
     # -- Success
     email_subject: str = "[INVOICING CRONJOB]"
+    if instance_name:  # instance name can be none, so check if it is set and add it to the subject
+        email_subject += f" ({instance_name})"
+
     email_body: str = (
         "The calculation of the monthly usage succeeded; The byte hours "
         "for all active projects have been saved to the database."
     )
     # -- Failure
     error_subject: str = f"{email_subject} <ERROR> Error in monthly-usage cronjob"
+
     error_body: str = (
         "There was an error in the cronjob 'monthly-usage', used for calculating the"
         " byte hours for every active project in the last month.\n\n"
@@ -972,13 +979,20 @@ def send_usage(months):
     from dds_web.database import models
     from dds_web.utils import current_time, page_query, send_email_with_retry
 
+    # Get the instance name (DEVELOPMENT, PRODUCTION, etc.)
+    instance_name = flask.current_app.config.get("INSTANCE_NAME")
+
     # Email settings
     email_recipient: str = flask.current_app.config.get("MAIL_DDS")
     # -- Success
     email_subject: str = "[SEND-USAGE CRONJOB]"
+    if instance_name:  # instance name can be none, so check if it is set and add it to the subject
+        email_subject += f" ({instance_name})"
+
     email_body: str = f"Here is the usage for the last {months} months.\n"
     # -- Failure
     error_subject: str = f"{email_subject} <ERROR> Error in send-usage cronjob"
+
     error_body: str = (
         "There was an error in the cronjob 'send-usage', used for sending"
         " information about the storage usage for each SciLifeLab unit. \n\n"
@@ -1133,7 +1147,15 @@ def collect_stats():
 
     # Get email address
     recipient: str = flask.current_app.config.get("MAIL_DDS")
-    error_subject: str = "[CRONJOB] Error during collection of DDS unit- and user statistics."
+
+    # Get the instance name (DEVELOPMENT, PRODUCTION, etc.)
+    instance_name = flask.current_app.config.get("INSTANCE_NAME")
+
+    error_subject: str = "[CRONJOB]"
+    if instance_name:  # instance name can be none, so check if it is set and add it to the subject
+        error_subject += f" ({instance_name})"
+    error_subject += " Error during collection of DDS unit and user statistics."
+
     error_body: str = (
         f"The cronjob 'reporting' experienced issues. Please see logs. Time: {current_time}."
     )
@@ -1256,7 +1278,7 @@ def monitor_usage():
 
         # Get info from database
         quota: int = unit.quota
-        warn_after: int = unit.warning_level
+        warn_after: float = unit.warning_level
         current_usage: int = unit.size
 
         # Check if 0 and then skip the next steps
@@ -1273,7 +1295,7 @@ def monitor_usage():
         # Information to log and potentially send
         info_string: str = (
             f"- Quota:{quota} bytes\n"
-            f"- Warning level: {warn_after*quota} bytes ({warn_after*100}%)\n"
+            f"- Warning level: {int(warn_after*quota)} bytes ({int(warn_after*100)}%)\n"
             f"- Current usage: {current_usage} bytes ({perc_used}%)\n"
         )
         flask.current_app.logger.debug(

dds_web/database/models.py

@@ -11,6 +11,7 @@
 
 # Installed
 import sqlalchemy
+from sqlalchemy.orm import validates
 import flask
 import argon2
 import flask_login
@@ -225,6 +226,12 @@ def size(self):
 
         return sum([p.size for p in self.projects])
 
+    @validates("warning_level")
+    def validate_level(self, key, value):
+        if not (0.0 <= value <= 1.0):
+            raise ValueError("Warning level must be a float between 0 and 1")
+        return value
+
 
 class Project(db.Model):
     """

dds_web/static/swagger.yaml

@@ -1447,6 +1447,9 @@ paths:
                 motd_id:
                   type: integer
                   example: 1
+                unit_only:
+                  type: boolean
+                  example: false
   /user/find:
     get:
       tags:

dds_web/static/swaggerv3.yaml

@@ -1409,6 +1409,9 @@ paths:
                 motd_id:
                   type: integer
                   example: 1
+                unit_only:
+                  type: boolean
+                  example: false
   /user/find:
     get:
       tags:

dds_web/version.py

@@ -1,3 +1,3 @@
 # Do not do major version upgrade during 2024.
 # If mid or minor version reaches 9, continue with 10, 11 etc etc.
-__version__ = "2.8.0"
+__version__ = "2.8.1"

docker-compose.yml

@@ -62,6 +62,8 @@ services:
       - DDS_APP_CONFIG=/code/dds_web/sensitive/dds_app.cfg
       - FLASK_DEBUG=true
       - FLASK_APP=dds_web
+      - FLASK_INSTANCE_NAME=LOCAL_DEVELOPMENT
+
       - DB_TYPE=${DDS_DB_TYPE}
       # - RATELIMIT_STORAGE_URI=redis://dds_redis
     depends_on:

tests/api/test_superadmin_only.py

@@ -588,7 +588,34 @@ def test_send_motd_no_primary_email(client: flask.testing.FlaskClient) -> None:
         assert "incorrect subject" not in outbox[-1].subject
 
 
-def test_send_motd_ok(client: flask.testing.FlaskClient) -> None:
+def test_send_motd_incorrect_type_unit_user_only(client: flask.testing.FlaskClient) -> None:
+    """The parameter unit_only should be a boolean"""
+    # Authenticate
+    token: typing.Dict = get_token(username=users["Super Admin"], client=client)
+
+    # Create a motd
+    message: str = "This is a message that should become a MOTD and then be sent to all the users."
+    new_motd: models.MOTD = models.MOTD(message=message)
+    db.session.add(new_motd)
+    db.session.commit()
+
+    # Make sure the motd is created
+    created_motd: models.MOTD = models.MOTD.query.filter_by(message=message).one_or_none()
+    assert created_motd
+
+    # Attempt request
+    with unittest.mock.patch.object(flask_mail.Connection, "send") as mock_mail_send:
+        response: werkzeug.test.WrapperTestResponse = client.post(
+            tests.DDSEndpoint.MOTD_SEND,
+            headers=token,
+            json={"motd_id": created_motd.id, "unit_only": "some_string"},
+        )
+        assert response.status_code == http.HTTPStatus.BAD_REQUEST
+        assert "The 'unit_only' argument must be a boolean." in response.json.get("message")
+        assert mock_mail_send.call_count == 0
+
+
+def test_send_motd_ok_all(client: flask.testing.FlaskClient) -> None:
     """Send a motd to all users."""
     # Authenticate
     token: typing.Dict = get_token(username=users["Super Admin"], client=client)
@@ -609,7 +636,39 @@ def test_send_motd_ok(client: flask.testing.FlaskClient) -> None:
     # Attempt request and catch email
     with mail.record_messages() as outbox:
         response: werkzeug.test.WrapperTestResponse = client.post(
-            tests.DDSEndpoint.MOTD_SEND, headers=token, json={"motd_id": created_motd.id}
+            tests.DDSEndpoint.MOTD_SEND,
+            headers=token,
+            json={"motd_id": created_motd.id, "unit_only": False},
+        )
+        assert response.status_code == http.HTTPStatus.OK
+        assert len(outbox) == num_users
+        assert "Important Information: Data Delivery System" in outbox[-1].subject
+
+
+def test_send_motd_ok_unitusers(client: flask.testing.FlaskClient) -> None:
+    """Send a motd to all unitusers users."""
+    # Authenticate
+    token: typing.Dict = get_token(username=users["Super Admin"], client=client)
+
+    # Create a motd
+    message: str = "This is a message that should become a MOTD and then be sent to all the users."
+    new_motd: models.MOTD = models.MOTD(message=message)
+    db.session.add(new_motd)
+    db.session.commit()
+
+    # Make sure the motd is created
+    created_motd: models.MOTD = models.MOTD.query.filter_by(message=message).one_or_none()
+    assert created_motd
+
+    # Get number of users
+    num_users = models.UnitUser.query.count()
+
+    # Attempt request and catch email
+    with mail.record_messages() as outbox:
+        response: werkzeug.test.WrapperTestResponse = client.post(
+            tests.DDSEndpoint.MOTD_SEND,
+            headers=token,
+            json={"motd_id": created_motd.id, "unit_only": True},
         )
         assert response.status_code == http.HTTPStatus.OK
         assert len(outbox) == num_users