Merge pull request #358 from deNBI/dev

Dev

.github/workflows/codeql-analysis.yml

@@ -35,7 +35,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.python-version

@@ -1 +1 @@
-3.11.4
+3.12.0

FastapiOpenRestyConfigurator/app/main/model/serializers.py

@@ -2,9 +2,10 @@
 Serializers for incoming and outgoing models.
 """
 import re
-
+import logging
 from pydantic import BaseModel, Field, validator
 
+logger = logging.getLogger("validation")
 # Metadata for used tags.
 tags_metadata = [
     {
@@ -25,7 +26,7 @@
     },
 ]
 
-owner_regex = r"([a-z0-9\-]{30,})"
+owner_regex = r'^[a-zA-Z0-9@.-]{30,}$'
 user_key_url_regex = r"^[a-zA-Z0-9]{3,25}$"
 upstream_url_regex = r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+"
 
@@ -37,7 +38,7 @@ class BackendBase(BaseModel):
     owner: str = Field(
         ...,
         title="Owner",
-        description="Owner of the backend without the @elixir.org suffix.",
+        description="Owner of the backend",
         example="21894723853fhdzug92"
     )
     template: str = Field(
@@ -54,16 +55,17 @@ class BackendBase(BaseModel):
     )
 
     @validator("owner")
-    def owner_validation(cls, v):
+    def owner_validation(cls, owner):
         """
         Validate owner string.
-        :param v: Value to assign to owner.
+        :param owner: Value to assign to owner.
         :return: Value or AssertionError.
         """
-        assert re.fullmatch(owner_regex, v), \
-            "The owner name can only contain alphabetics and numerics with at least 30 chars. " \
-            "Also no @elixir.org prefix at the end please!"
-        return v
+        logger.info(f"Validate owner name -> {owner}")
+        if re.fullmatch(owner_regex, owner):
+            return owner
+        else:
+            raise AssertionError("The owner name can only contain alphabets, numerics, and '@' with at least 30 characters.")
 
 
 class BackendIn(BackendBase):
@@ -150,7 +152,7 @@ class User(BaseModel):
     """
     User model.
     """
-    
+
     user: str
 
 

FastapiOpenRestyConfigurator/app/main/service/backend.py

@@ -17,8 +17,7 @@
 logger = logging.getLogger("service")
 settings = get_settings()
 
-file_regex = r"(\d*)%([a-z0-9\-]*)%([^%]*)%([^%]*)%([^%]*)\.conf"
-
+file_regex = r"(\d*)%([a-z0-9\-\@]*)%([^%]*)%([^%]*)%([^%]*)\.conf"
 
 async def random_with_n_digits(n):
     range_start = 10**(n-1)

FastapiOpenRestyConfigurator/app/main/service/user.py

@@ -29,9 +29,15 @@ async def get_users(backend_id):
 
 async def add_user(backend_id, user_id):
     backend_id = secure_filename(str(backend_id))
-    user_id = secure_filename(str(user_id))
+    if "@" in user_id:
+        user_id_parts = user_id.split("@")
+        user_id_part1 = secure_filename(user_id_parts[0])
+        user_id_part2 = secure_filename(user_id_parts[1])
+        user_id = f"{user_id_part1}@{user_id_part2}"
+    else:
+        user_id = secure_filename(str(user_id))
     user_id_path = f"{settings.FORC_USER_PATH}/{backend_id}"
-    user_file_name = f"{user_id}@elixir-europe.org"
+    user_file_name = f"{user_id}"
     if not os.path.exists(user_id_path):
         try:
             os.mkdir(user_id_path)
@@ -56,7 +62,7 @@ async def delete_user(backend_id, user_id):
     backend_id = secure_filename(str(backend_id))
     user_id = secure_filename(str(user_id))
     user_id_path = f"{settings.FORC_USER_PATH}/{backend_id}"
-    user_file_name = f"{user_id}@elixir-europe.org"
+    user_file_name = f"{user_id}"
     user_file_path = f"{user_id_path}/{user_file_name}"
     if not os.path.exists(user_id_path):
         logger.exception(f"No user folder found for backend: {backend_id}.")

FastapiOpenRestyConfigurator/app/main/util/logging.py

@@ -21,11 +21,17 @@
             "class": "logging.StreamHandler",
             "stream": "ext://sys.stderr",
         },
+        "file": {
+            "formatter": "default",
+            "class": "logging.FileHandler",
+            "filename": "/var/log/all_forc_logs.log"
+        },
     },
     "loggers": {
-        "internal": {"handlers": ["default"], "level": settings.LOG_LEVEL},
-        "view": {"handlers": ["default"], "level": settings.LOG_LEVEL},
-        "service": {"handlers": ["default"], "level": settings.LOG_LEVEL},
-        "util": {"handlers": ["default"], "level": settings.LOG_LEVEL}
+        "internal": {"handlers": ["default", "file"], "level": settings.LOG_LEVEL},
+        "view": {"handlers": ["default", "file"], "level": settings.LOG_LEVEL},
+        "service": {"handlers": ["default", "file"], "level": settings.LOG_LEVEL},
+        "validation": {"handlers": ["default", "file"], "level": settings.LOG_LEVEL},
+        "util": {"handlers": ["default", "file"], "level": settings.LOG_LEVEL}
     },
 }

FastapiOpenRestyConfigurator/requirements.txt

@@ -1,7 +1,7 @@
-fastapi==0.101.0
+fastapi==0.103.2
 uvicorn==0.23.2
-werkzeug==2.3.6
+werkzeug==2.3.7
 Jinja2==3.1.2
 python-dotenv==1.0.0
-gunicorn==20.1.0
-pydantic-settings
+gunicorn==21.2.0
+pydantic-settings

FlaskOpenRestyConfigurator/app/main/service/user.py

@@ -0,0 +1,83 @@
+import logging
+import os
+import shutil
+
+from app.main.service.openresty import reloadOpenresty
+from ..config import user_path
+
+logging.basicConfig()
+logger = logging.getLogger()
+logger.setLevel(logging.INFO)
+
+
+def get_users(backend_id):
+    user_id_path = "{0}/{1}".format(user_path, backend_id)
+    if not os.path.exists(user_id_path) and not os.access(user_id_path, os.R_OK):
+        logger.error("Not able to access configured user id path. Backend id: {0}".format(backend_id))
+        return []
+    return os.listdir(user_id_path)
+
+
+def add_user(backend_id, user_id):
+    user_id_path = "{0}/{1}".format(user_path, backend_id)
+    user_file_name = "{0}".format(user_id)
+    if not os.path.exists(user_id_path):
+        try:
+            os.mkdir(user_id_path)
+        except OSError:
+            logger.error("Not able to create backend directory with id {0} for user.".format(backend_id))
+            return 1
+    if not os.access(user_id_path, os.W_OK):
+        logger.error("Not able to access configured user id path.")
+        return 2
+    existing_users = os.listdir(user_id_path)
+    for file in existing_users:
+        if file == user_file_name:
+            logger.info("User already added to backend.")
+            return 3
+    with open(user_id_path + "/" + user_file_name, 'w') as userFile:
+        userFile.write("")
+
+    # attempt to reload openrest
+    reloadOpenresty()
+    return 0
+
+
+def delete_user(backend_id, user_id):
+    user_id_path = "{0}/{1}".format(user_path, backend_id)
+    user_file_name = "{0}".format(user_id)
+    user_file_path = "{0}/{1}".format(user_id_path, user_file_name)
+    if not os.path.exists(user_id_path):
+        logger.error("No user folder found for backend: {0}.".format(backend_id))
+        return 1
+    if not os.path.exists(user_file_path):
+        logger.error("No user file {1} found for backend: {0}.".format(backend_id, user_file_name))
+        return 1
+    if not os.access(user_file_path, os.W_OK):
+        logger.error("Not able to access configured user id path.")
+        return 2
+    logger.info("Deleting user {0} from backend {1}.".format(user_file_name, backend_id))
+    try:
+        existing_users = os.listdir(user_id_path)
+        if len(existing_users) != 1:
+            os.remove(user_file_path)
+        else:
+            delete_all(backend_id)
+        reloadOpenresty()
+        return 0
+    except OSError:
+        logger.error("Not able to delete user {1} from backend {0}.".format(backend_id, user_file_name))
+        return 3
+
+
+def delete_all(backend_id):
+    user_id_path = "{0}/{1}".format(user_path, backend_id)
+    if not os.path.exists(user_id_path):
+        logger.info("No user folder found for backend: {0}.".format(backend_id))
+        return 1
+    try:
+        shutil.rmtree(user_id_path, ignore_errors=True)
+        return 0
+    except OSError:
+        logger.error("Not able to delete users for backend {0}.".format(backend_id))
+        return 3

FlaskOpenRestyConfigurator/app/main/util/dto.py

@@ -0,0 +1,58 @@
+from flask_restplus import Namespace, fields
+
+
+authorizations = {
+    'apikey': {
+        'type': 'apiKey',
+        'in': 'header',
+        'name': 'X-API-KEY'
+    }
+}
+
+
+# General DTO for a backend json object
+class BackendDto:
+    api = Namespace('backends', description="All backend related resources. Backends are generated nginx location snippets, generated from templates.", authorizations=authorizations)
+
+    backend = api.model('Backend', {
+        'id': fields.Integer(required=True, description="Unique ID of backend", example="78345"),
+        'owner': fields.String(required=True, description="User who owns this backend."),
+        'location_url': fields.String(required=True, description="Protected reverse-proxy path which leads to specific backend"),
+        'template': fields.String(required=True, description="Used backend template", example="rstudio"),
+        'template_version': fields.String(required=True, description="Template Version", example="v1")
+    })
+
+    createBackend = api.model('CreateBackend', {
+        'owner': fields.String(required=True, description="User who owns this backend.", example="21894723853fhdzug92"),
+        'user_key_url': fields.String(required=True, description="User set location url prefix", example="myFavoriteRstudio"),
+        'upstream_url': fields.String(required=True, description="Inject the full url (with protocol) for the real location of the backend service in the template.", example="http://localhost:7001/"),
+        'template': fields.String(required=True, description="Used backend template", example="rstudio"),
+        'template_version': fields.String(required=True, description="Template Version", example="v1")
+    })
+
+
+class UserDto:
+    api = Namespace('users', description="All user related endpoints. Users are people allowed to access a backend.", authorizations=authorizations)
+
+    createUser = api.model('createUser', {
+        'owner': fields.String(required=True, description="User who owns this backend. "),
+        'user': fields.String(required=True, description="User who will be added to this backend.")
+    })
+
+
+# General DTO for a template json object
+class TemplateDto:
+    api = Namespace('templates', description="All template related endpoints. Templates are used to generate OpenResty location configurations.", authorizations=authorizations)
+
+    template = api.model('Template', {
+        'name': fields.String(required=True, description="Name of the template.", example="rstudio"),
+        'version': fields.String(required=True, description="Version of this template.", example="v13")
+    })
+
+
+class UtilsDto:
+    api = Namespace('utils', description="Misc endpoints.", authorizations=authorizations)
+
+    version = api.model('Version', {
+        'version': fields.String(required=True, description="Current running version of this service framework", example="v1.0.0")
+    })

FlaskOpenRestyConfigurator/app/main/util/validate.py

@@ -0,0 +1,25 @@
+import re
+
+ownerRegex = r'^[[email protected]]{30,}$'
+userKeyUrlRegex = r"^[a-zA-Z0-9]{3,25}$"
+
+upstreamURLRegex = r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+"
+
+
+
+def validatePostBackendContent(payload):
+    #check owner
+    owner = payload['owner']
+    if not re.fullmatch(ownerRegex, owner):
+        return {"error" : "The owner name can only contain alphabetics, numerics and @ - with at least 30 chars."}
+
+    user_key_url = payload['user_key_url']
+    if not re.fullmatch(userKeyUrlRegex, user_key_url):
+        return {"error" : "The user key url prefix can only contain alphabetics and numerics with at least 3 and a maximum of 25 chars."}
+
+    upstreamURL = payload['upstream_url']
+    if not re.fullmatch(upstreamURLRegex, upstreamURL):
+        return {"error" : "This is not a valid upstream url. Example: http://129.70.168.5:3000"}
+
+    return {"status" : "okay"}
+

examples/templates/cwlab%v01.conf

@@ -14,7 +14,7 @@
                         end
 
                         -- Protect this location and allow only one specific ELIXIR User
-                        if (res.id_token.sub ~= "{{ owner }}@elixir-europe.org" and not user_service.file_exists(ngx.var.user_path .. res.id_token.sub)) then
+                        if (res.id_token.sub ~= "{{ owner }}" and not user_service.file_exists(ngx.var.user_path .. res.id_token.sub)) then
                                 ngx.exit(ngx.HTTP_FORBIDDEN)
                         end
                 }

examples/templates/cwlab%v02.conf

@@ -31,7 +31,7 @@
             end
 
             -- Protect this location and allow only one specific ELIXIR User
-            if (res.id_token.sub ~= "{{ owner }}@elixir-europe.org" and not user_service.file_exists(ngx.var.user_path .. res.id_token.sub)) then
+            if (res.id_token.sub ~= "{{ owner }}" and not user_service.file_exists(ngx.var.user_path .. res.id_token.sub)) then
                 ngx.exit(ngx.HTTP_FORBIDDEN)
             end
 

examples/templates/emgb%v01.conf

@@ -30,7 +30,8 @@
             end
 
             -- Protect this location and allow only one specific ELIXIR User
-            if (res.id_token.sub ~= "{{ owner }}@elixir-europe.org" and not user_service.file_exists(ngx.var.user_path .. res.id_token.sub)) then
+            if (res.id_token.sub ~= "{{ owner }}" and not user_service.file_exists(ngx.var.user_path .. res.id_token.sub)) then
+
                 ngx.exit(ngx.HTTP_FORBIDDEN)
             end
 

examples/templates/guacamole%v01.conf

@@ -14,7 +14,7 @@ location /{{ key_url }}/ {
             end
 
             -- Protect this location and allow only one specific ELIXIR User
-            if res.id_token.sub ~= "{{ owner }}@elixir-europe.org" then
+            if res.id_token.sub ~= "{{ owner }}" then
                     ngx.exit(ngx.HTTP_FORBIDDEN)
             end
     }

examples/templates/guacamole%v02.conf

@@ -15,7 +15,7 @@ location /{{ key_url }}/ {
             end
 
             -- Protect this location and allow only one specific ELIXIR User
-            if (res.id_token.sub ~= "{{ owner }}@elixir-europe.org" and not user_service.file_exists(ngx.var.user_path .. res.id_token.sub)) then
+            if (res.id_token.sub ~= "{{ owner }}" and not user_service.file_exists(ngx.var.user_path .. res.id_token.sub)) then
                     ngx.exit(ngx.HTTP_FORBIDDEN)
             end
     }

examples/templates/guacamole%v03.conf

@@ -29,7 +29,7 @@
             end
 
             -- Protect this location and allow only one specific ELIXIR User
-            if (res.id_token.sub ~= "{{ owner }}@elixir-europe.org" and not user_service.file_exists(ngx.var.user_path .. res.id_token.sub)) then
+            if (res.id_token.sub ~= "{{ owner }} and not user_service.file_exists(ngx.var.user_path .. res.id_token.sub)) then
                 ngx.exit(ngx.HTTP_FORBIDDEN)
             end
 

examples/templates/jupyterlab%v01.conf

@@ -13,7 +13,7 @@ location /{{ key_url }} {
                         end
 
                         -- Protect this location and allow only one specific ELIXIR User
-                        if res.id_token.sub ~= "{{ owner }}@elixir-europe.org" then
+                        if res.id_token.sub ~= "{{ owner }}" then
                                 ngx.exit(ngx.HTTP_FORBIDDEN)
                         end
                 }

examples/templates/jupyterlab%v02.conf

@@ -14,7 +14,7 @@ location /{{ key_url }} {
                         end
 
                         -- Protect this location and allow only one specific ELIXIR User
-                        if (res.id_token.sub ~= "{{ owner }}@elixir-europe.org" and not user_service.file_exists(ngx.var.user_path .. res.id_token.sub)) then
+                        if (res.id_token.sub ~= "{{ owner }}" and not user_service.file_exists(ngx.var.user_path .. res.id_token.sub)) then
                                 ngx.exit(ngx.HTTP_FORBIDDEN)
                         end
                 }

examples/templates/jupyterlab%v03.conf

@@ -29,7 +29,7 @@
             end
 
             -- Protect this location and allow only one specific ELIXIR User
-            if (res.id_token.sub ~= "{{ owner }}@elixir-europe.org" and not user_service.file_exists(ngx.var.user_path .. res.id_token.sub)) then
+            if (res.id_token.sub ~= "{{ owner }}" and not user_service.file_exists(ngx.var.user_path .. res.id_token.sub)) then
                 ngx.exit(ngx.HTTP_FORBIDDEN)
             end