From 60b0db1f566c576df3d604943063aa5498570707 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 24 Aug 2023 08:01:59 +0200 Subject: [PATCH 01/20] first tests --- src/actinia_core/core/common/app.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/actinia_core/core/common/app.py b/src/actinia_core/core/common/app.py index 60f48e123..c6ae7c7ef 100644 --- a/src/actinia_core/core/common/app.py +++ b/src/actinia_core/core/common/app.py @@ -165,7 +165,7 @@ } } ] -else: +elif global_config.AUTHENTICATION: # Set the security definition in an unconventional way flask_api._swagger_object["securityDefinitions"] = { "basicAuth": {"type": "basic"} @@ -173,3 +173,20 @@ flask_api._swagger_object["security"] = [{"basicAuth": []}] auth = HTTPBasicAuth() +else: + # No authentication + auth = HTTPBasicAuth() + # auth = None + + # class NoAuthentication(HTTPBasicAuth): + # def login_required(user=None, pw=None): + # return False + # auth = NoAuthentication() + + # class NoAuthentication(): + # login_required = False + # auth = NoAuthentication() + + # @auth.login_required + # def login_required(): + # return False From f5a0482fadcbb67828f673333697a2ad14c02aad Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 24 Aug 2023 08:08:04 +0200 Subject: [PATCH 02/20] add a no-auth user --- src/actinia_core/core/common/app.py | 20 +----- src/actinia_core/core/common/config.py | 11 ++- src/actinia_core/core/common/user_noauth.py | 80 +++++++++++++++++++++ src/actinia_core/rest/base/user_auth.py | 21 ++++-- 4 files changed, 109 insertions(+), 23 deletions(-) create mode 100644 src/actinia_core/core/common/user_noauth.py diff --git a/src/actinia_core/core/common/app.py b/src/actinia_core/core/common/app.py index c6ae7c7ef..7a6f37d8e 100644 --- a/src/actinia_core/core/common/app.py +++ b/src/actinia_core/core/common/app.py @@ -4,7 +4,7 @@ # performance processing of geographical data that uses GRASS GIS for # computational tasks. For details, see https://actinia.mundialis.de/ # -# Copyright (c) 2016-2018 Sören Gebbert and mundialis GmbH & Co. KG +# Copyright (c) 2016-2023 Sören Gebbert and mundialis GmbH & Co. KG # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -116,9 +116,9 @@ from actinia_api import API_VERSION, URL_PREFIX __license__ = "GPLv3" -__author__ = "Sören Gebbert, Julia Haas" +__author__ = "Sören Gebbert, Julia Haas, Anika Weinmann" __copyright__ = ( - "Copyright 2016-2021, Sören Gebbert and mundialis GmbH & Co. KG" + "Copyright 2016-2023, Sören Gebbert and mundialis GmbH & Co. KG" ) __maintainer__ = "mundialis" @@ -176,17 +176,3 @@ else: # No authentication auth = HTTPBasicAuth() - # auth = None - - # class NoAuthentication(HTTPBasicAuth): - # def login_required(user=None, pw=None): - # return False - # auth = NoAuthentication() - - # class NoAuthentication(): - # login_required = False - # auth = NoAuthentication() - - # @auth.login_required - # def login_required(): - # return False diff --git a/src/actinia_core/core/common/config.py b/src/actinia_core/core/common/config.py index 300e19bf6..cf0dae6f9 100644 --- a/src/actinia_core/core/common/config.py +++ b/src/actinia_core/core/common/config.py @@ -4,7 +4,7 @@ # performance processing of geographical data that uses GRASS GIS for # computational tasks. For details, see https://actinia.mundialis.de/ # -# Copyright (c) 2016-2018 Sören Gebbert and mundialis GmbH & Co. KG +# Copyright (c) 2016-2023 Sören Gebbert and mundialis GmbH & Co. KG # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -34,7 +34,7 @@ __license__ = "GPLv3" __author__ = "Sören Gebbert, Anika Weinmann" __copyright__ = ( - "Copyright 2016-2022, Sören Gebbert and mundialis GmbH & Co. KG" + "Copyright 2016-2023, Sören Gebbert and mundialis GmbH & Co. KG" ) __maintainer__ = "mundialis GmbH & Co. KG" @@ -320,6 +320,8 @@ def __init__(self): self.PLUGINS = [] # ENDPOINTS_CONFIG: configuration csv file for endpoints self.ENDPOINTS_CONFIG = None + # AUTHENTICATION: If set False no authentication is needed + self.AUTHENTICATION = True """ KEYCLOAK: has only to be set if keycloak server is configured with @@ -560,6 +562,7 @@ def write(self, path=DEFAULT_CONFIG_PATH): config.set("API", "FORCE_HTTPS_URLS", str(self.FORCE_HTTPS_URLS)) config.set("API", "PLUGINS", str(self.PLUGINS)) config.set("API", "ENDPOINTS_CONFIG", str(self.ENDPOINTS_CONFIG)) + config.set("API", "AUTHENTICATION", str(self.AUTHENTICATION)) config.add_section("KEYCLOAK") config.set( @@ -810,6 +813,10 @@ def read(self, path=DEFAULT_CONFIG_PATH): self.ENDPOINTS_CONFIG = config.get( "API", "ENDPOINTS_CONFIG" ) + if config.has_option("API", "AUTHENTICATION"): + self.AUTHENTICATION = config.getboolean( + "API", "AUTHENTICATION" + ) if config.has_section("KEYCLOAK"): if config.has_option("KEYCLOAK", "CONFIG_PATH"): diff --git a/src/actinia_core/core/common/user_noauth.py b/src/actinia_core/core/common/user_noauth.py new file mode 100644 index 000000000..7631a0fb6 --- /dev/null +++ b/src/actinia_core/core/common/user_noauth.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- +####### +# actinia-core - an open source REST API for scalable, distributed, high +# performance processing of geographical data that uses GRASS GIS for +# computational tasks. For details, see https://actinia.mundialis.de/ +# +# Copyright (c) 2023 mundialis GmbH & Co. KG +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +####### + +""" +User management for no authentication +""" + +# from passlib.apps import custom_app_context as pwd_context +# import jwt +# from datetime import datetime, timezone, timedelta +from actinia_core.core.common.config import global_config +from actinia_core.core.redis_user import redis_user_interface +from actinia_core.core.common.user import ActiniaUser + + +__author__ = "Anika Weinmann" +__copyright__ = "Copyright 2023, mundialis GmbH & Co. KG" +__maintainer__ = "mundialis GmbH & Co. KG" + + +class ActiniaUserNoAuth(ActiniaUser): + """ + The Actinia Core user management class for no authentication + """ + + db = redis_user_interface + + def __init__(self): + """Constructor + Initialize and create a user object for no authentication. + """ + self.user_id = global_config.DEFAULT_USER + self.user_group = global_config.DEFAULT_USER_GROUP + self.password_hash = None + self.user_role = "superadmin" + self.permissions = None + self.cell_limit = global_config.MAX_CELL_LIMIT + self.accessible_datasets = { + "nc_spm_08": ["PERMANENT", "user1", "landsat"], + "ECAD": ["PERMANENT"], + "latlong_wgs84": ["PERMANENT"], + } + self.accessible_modules = global_config.MODULE_ALLOW_LIST + self.process_num_limit = global_config.PROCESS_NUM_LIMIT + self.process_time_limit = global_config.PROCESS_TIME_LIMT + + @staticmethod + def create_user(): + """Create a new user object for no authentication and initialize it + + Returns: + actinia_core_api.common.user_noauth.ActiniaUserNoAuth: + A new user object in case of success, or None in case of failure + """ + user = ActiniaUserNoAuth() + user.hash_password("") + + if user.commit() is True: + return user + return None diff --git a/src/actinia_core/rest/base/user_auth.py b/src/actinia_core/rest/base/user_auth.py index a12e58f75..0dada39a8 100644 --- a/src/actinia_core/rest/base/user_auth.py +++ b/src/actinia_core/rest/base/user_auth.py @@ -5,7 +5,7 @@ # performance processing of geographical data that uses GRASS GIS for # computational tasks. For details, see https://actinia.mundialis.de/ # -# Copyright (c) 2016-2018 Sören Gebbert and mundialis GmbH & Co. KG +# Copyright (c) 2016-2023 Sören Gebbert and mundialis GmbH & Co. KG # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -33,16 +33,29 @@ from actinia_core.core.common.app import auth from actinia_core.core.common.keycloak_user import ActiniaKeycloakUser from actinia_core.core.common.user import ActiniaUser +from actinia_core.core.common.user_noauth import ActiniaUserNoAuth from actinia_core.core.messages_logger import MessageLogger __license__ = "GPLv3" -__author__ = "Sören Gebbert, Julia Haas" +__author__ = "Sören Gebbert, Julia Haas, Anika Weinmann" __copyright__ = ( - "Copyright 2016-2022, Sören Gebbert and mundialis GmbH & Co. KG" + "Copyright 2016-2023, Sören Gebbert and mundialis GmbH & Co. KG" ) __maintainer__ = "mundialis" +if not global_config.AUTHENTICATION: + + # No authentication + @auth.login_required + def login_required(a, b): + return False + + @auth.verify_password + def verify_password(username_or_token, password): + g.user = ActiniaUserNoAuth.create_user() + return True + if global_config.KEYCLOAK_CONFIG_PATH: @auth.verify_token @@ -63,7 +76,7 @@ def verify_token(token): return True -if global_config.KEYCLOAK_CONFIG_PATH is None: +if global_config.KEYCLOAK_CONFIG_PATH is None and global_config.AUTHENTICATION: @auth.verify_password def verify_password(username_or_token, password): From 189617d921a151ae79a05e6e0b76b79010bfd3c6 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 24 Aug 2023 09:09:56 +0200 Subject: [PATCH 03/20] change pre-commits --- .pre-commit-config.yaml | 80 +++++++++++++---------------------------- 1 file changed, 24 insertions(+), 56 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1700efd6b..922ebdb00 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,59 +1,27 @@ --- repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + # - repo: https://github.com/pre-commit/pre-commit-hooks + # rev: v4.4.0 + # hooks: + # - id: trailing-whitespace + # exclude: | + # (?x)^( + # .*\.ref$| + # .*\.svg$| + # build/| + # dist/| + # src/actinia_core.egg-info/ + # ) + # - id: end-of-file-fixer + # exclude: | + # (?x)^( + # .*\.ref$| + # .*\.svg$| + # build/| + # dist/| + # src/actinia_core.egg-info/ + # ) +- repo: https://github.com/mundialis/github-workflows + rev: 8a5bab5f1b3e18d8edb8cb5d78a31e3b6b4646c5 hooks: - - id: trailing-whitespace - exclude: | - (?x)^( - .*\.ref$| - .*\.svg$| - build/| - dist/| - src/actinia_core.egg-info/ - ) - - id: end-of-file-fixer - exclude: | - (?x)^( - .*\.ref$| - .*\.svg$| - build/| - dist/| - src/actinia_core.egg-info/ - ) - - repo: https://github.com/psf/black - rev: 22.3.0 - hooks: - - id: black - args: [--line-length, '79'] - exclude: | - (?x)^( - build/| - dist/| - src/actinia_core.egg-info/ - ) - - repo: https://github.com/pycqa/flake8 - rev: 3.8.0 - hooks: - - id: flake8 - args: [--config=.flake8] - exclude: | - (?x)^( - build/| - dist/| - src/actinia_core.egg-info/ - ) - -# still deactivated, needs major code cleanup: -# - repo: https://github.com/igorshubovych/markdownlint-cli -# rev: v0.33.0 -# hooks: -# - id: markdownlint -# - repo: https://github.com/adrienverge/yamllint.git -# rev: v1.29.0 -# hooks: -# - id: yamllint -# args: [--format, parsable, --strict, -d, -# '{extends: default, rules: {truthy: disable, -# line-length: {max: 120, -# allow-non-breakable-words: true}}}'] + - id: linting From b69542f048373f04c15646a0469f78e29e119382 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 24 Aug 2023 11:00:48 +0200 Subject: [PATCH 04/20] pre-commit --- .pre-commit-config.yaml | 44 ++++++++++++++++++++--------------------- CHANGELOG.md | 4 ++-- job_resumption.md | 2 +- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 922ebdb00..ef3fe1770 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,27 +1,27 @@ --- repos: - # - repo: https://github.com/pre-commit/pre-commit-hooks - # rev: v4.4.0 - # hooks: - # - id: trailing-whitespace - # exclude: | - # (?x)^( - # .*\.ref$| - # .*\.svg$| - # build/| - # dist/| - # src/actinia_core.egg-info/ - # ) - # - id: end-of-file-fixer - # exclude: | - # (?x)^( - # .*\.ref$| - # .*\.svg$| - # build/| - # dist/| - # src/actinia_core.egg-info/ - # ) -- repo: https://github.com/mundialis/github-workflows + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: trailing-whitespace + exclude: | + (?x)^( + .*\.ref$| + .*\.svg$| + build/| + dist/| + src/actinia_core.egg-info/ + ) + - id: end-of-file-fixer + exclude: | + (?x)^( + .*\.ref$| + .*\.svg$| + build/| + dist/| + src/actinia_core.egg-info/ + ) + - repo: https://github.com/mundialis/github-workflows rev: 8a5bab5f1b3e18d8edb8cb5d78a31e3b6b4646c5 hooks: - id: linting diff --git a/CHANGELOG.md b/CHANGELOG.md index b167e2a50..90cf669a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,7 @@ released from main **Full Changelog**: https://github.com/actinia-org/actinia-core/compare/4.9.2...4.9.3 -"generated with gh api repos/actinia-org/actinia-core/releases/generate-notes -f tag_name="4.9.3" -f target_commitish=main -q .body" +"generated with gh api repos/actinia-org/actinia-core/releases/generate-notes -f tag_name="4.9.3" -f target_commitish=main -q .body" ## [4.9.2] - 2023-06-29 released from main @@ -45,7 +45,7 @@ released from main **Full Changelog**: https://github.com/actinia-org/actinia-core/compare/4.9.1...4.9.2 "generated with `gh api repos/actinia-org/actinia-core/releases/generate-notes -f tag_name="4.9.2" -f target_commitish=main -q .body`" - + ## [4.9.1] - 2023-06-14 released from main diff --git a/job_resumption.md b/job_resumption.md index 3107daa1b..090a19574 100644 --- a/job_resumption.md +++ b/job_resumption.md @@ -437,7 +437,7 @@ actiniaget $AUTH $URL/resource_storage # delete resource storage files older than 5 days actiniadelete $AUTH $URL/resource_storage?olderthan=5 -actiniaget $AUTH +actiniaget $AUTH # delete all files in the user resource storage actiniadelete $AUTH $URL/resource_storage From 23f900d2792ddae0659ba827ad31247a9403017c Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Thu, 24 Aug 2023 14:30:54 +0200 Subject: [PATCH 05/20] linting --- .flake8 | 2 ++ src/actinia_core/rest/base/user_auth.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index d4ef350e3..b3da9504c 100644 --- a/.flake8 +++ b/.flake8 @@ -3,6 +3,7 @@ # E402 module level import not at top of file # E501 line too long (80 > 79 characters) # F401 '.health_check.health_check' imported but unused +# F811 redefinition of unused 'verify_password' from line 55 # F821 undefined name 'QFile' # W605 invalid escape sequence '\<' @@ -30,6 +31,7 @@ per-file-ignores = ./src/actinia_core/core/interim_results.py: W605 ./src/actinia_core/core/list_grass_modules.py: F821 ./src/actinia_core/testsuite.py: F401 + ./src/actinia_core/rest/base/user_auth.py: F811 ./src/actinia_core/rest/ephemeral_processing.py: W605 ./src/actinia_core/processing/actinia_processing/ephemeral_renderer_base/*: E501 ./src/actinia_core/processing/actinia_processing/ephemeral_with_export/raster_export.py: E501 diff --git a/src/actinia_core/rest/base/user_auth.py b/src/actinia_core/rest/base/user_auth.py index 0dada39a8..668069265 100644 --- a/src/actinia_core/rest/base/user_auth.py +++ b/src/actinia_core/rest/base/user_auth.py @@ -45,7 +45,6 @@ if not global_config.AUTHENTICATION: - # No authentication @auth.login_required def login_required(a, b): @@ -56,6 +55,7 @@ def verify_password(username_or_token, password): g.user = ActiniaUserNoAuth.create_user() return True + if global_config.KEYCLOAK_CONFIG_PATH: @auth.verify_token From fa6c29abdf3b25cfb2d68b8ac7ccd6ec54837649 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Fri, 25 Aug 2023 10:55:10 +0200 Subject: [PATCH 06/20] first running tests --- tests/test_noauth.py | 437 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 437 insertions(+) create mode 100644 tests/test_noauth.py diff --git a/tests/test_noauth.py b/tests/test_noauth.py new file mode 100644 index 000000000..08c1ac8d5 --- /dev/null +++ b/tests/test_noauth.py @@ -0,0 +1,437 @@ +# -*- coding: utf-8 -*- +####### +# actinia-core - an open source REST API for scalable, distributed, high +# performance processing of geographical data that uses GRASS GIS for +# computational tasks. For details, see https://actinia.mundialis.de/ +# +# Copyright (c) 2016-2018 Sören Gebbert and mundialis GmbH & Co. KG +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +####### + +""" +Tests: Location test case +""" +import configparser +from flask.json import loads as json_loads, dumps as json_dumps +import os +import pytest +import requests +import unittest +from actinia_core.core.common.app import flask_app +from actinia_core.core.common.config import global_config +from actinia_core.core.common import redis_interface +from actinia_core.core.common.process_queue import create_process_queue +from actinia_core.testsuite import ActiniaRequests # ActiniaTestCaseBase, URL_PREFIX, + + +try: + from .test_resource_base import ActiniaResourceTestCaseBase, URL_PREFIX +except ModuleNotFoundError: + from test_resource_base import ActiniaResourceTestCaseBase, URL_PREFIX + +__license__ = "GPLv3" +__author__ = "Anika Weinmann" +__copyright__ = ( + "Copyright 2023, mundialis GmbH & Co. KG" +) +__maintainer__ = "mundialis" + + +class ActiniaWithoutAuthentication(ActiniaResourceTestCaseBase): + """Test bsae class to test actinia without autentication""" + server_test = False + cfg_file = os.environ.get( + "ACTINIA_CUSTOM_TEST_CFG", "/etc/default/actinia" + ) + auth_value = None + tmp_cfg_file = "%s_tmp" % cfg_file + auth = "False" + + if "ACTINIA_SERVER_TEST" in os.environ: + server_test = bool(os.environ["ACTINIA_SERVER_TEST"]) + + @classmethod + def save_config(cls, src, dest, value): + config = configparser.ConfigParser() + config.read(src) + if value: + config["API"]["authentication"] = value + else: + del config["API"]["authentication"] + with open(dest, "w") as configfile: + config.write(configfile) + + @classmethod + def setUpClass(cls): + cls.auth_value = global_config.AUTHENTICATION + if cls.auth_value is True: + os.replace(cls.cfg_file, cls.tmp_cfg_file) + cls.save_config( + cls.tmp_cfg_file, + cls.cfg_file, + cls.auth, + ) + + if cls.server_test is False and cls.custom_actinia_cfg is False: + global_config.REDIS_SERVER_SERVER = "localhost" + global_config.REDIS_SERVER_PORT = 7000 + global_config.GRASS_RESOURCE_DIR = "/tmp" + global_config.DOWNLOAD_CACHE = "/tmp/download_cache" + global_config.REDIS_QUEUE_SERVER_URL = "localhost" + global_config.REDIS_QUEUE_SERVER_PORT = 6379 + global_config.NUMBER_OF_WORKERS = 3 + + # If the custom_actinia_cfg variable is set, then the actinia config + # file will be read to configure Redis queue + if cls.server_test is False and cls.custom_actinia_cfg is not False: + global_config.read(cls.custom_actinia_cfg) + # Start the redis interface + redis_args = ( + global_config.REDIS_SERVER_URL, + global_config.REDIS_SERVER_PORT, + ) + if ( + global_config.REDIS_SERVER_PW + and global_config.REDIS_SERVER_PW is not None + ): + redis_args = (*redis_args, global_config.REDIS_SERVER_PW) + + redis_interface.connect(*redis_args) + + # Process queue + create_process_queue(config=global_config) + + @classmethod + def tearDownClass(cls): + # reset config + if cls.auth is True: + cls.save_config(cls.tmp_cfg_file, cls.cfg_file, "True") + os.remove(cls.tmp_cfg_file) + if cls.server_test is False: + redis_interface.disconnect() + + def setUp(self): + # We need to set the application context + self.app_context = flask_app.app_context() + self.app_context.push() + + # Check if the local or server site tests should be performed + if self.server_test is False: + flask_app.config["TESTING"] = True + + self.server = flask_app.test_client() + else: + self.server = ActiniaRequests() + + def tearDown(self): + self.app_context.pop() + + def waitAsyncStatusAssertHTTP( + self, + response, + http_status=200, + status="finished", + message_check=None, + ): + """Poll the status of a resource and assert its finished HTTP status + + The response will be checked if the resource was accepted. Hence it + must always be HTTP 200 status. + + The status URL from the response is then polled until status: finished, + error or terminated. + The result of the poll can be checked against its HTTP status and its + actinia status message. + + Args: + response: The accept response + http_status (int): The HTTP status that should be checked + status (str): The return status of the response + message_check (str): A string that must be in the message field + + Returns: response + + """ + resp_data = super().waitAsyncStatusAssertHTTP( + response, None, http_status, status, message_check + ) + return resp_data + + +@pytest.mark.noauth +class TestActiniaWithoutAuthentication(ActiniaWithoutAuthentication): + + def test_version(self): + rv = self.server.get( + URL_PREFIX + "/version" + ) + self.assertEqual( + rv.status_code, + 200, + "HTML status code is wrong %i" % rv.status_code, + ) + data = json_loads(rv.data) + self.assertIn("api_version", data) + self.assertIn("version", data) + self.assertIn("grass_version", data) + + def test_list_locations(self): + rv = self.server.get( + URL_PREFIX + "/locations" + ) + print(rv.data) + self.assertEqual( + rv.status_code, + 200, + "HTML status code is wrong %i" % rv.status_code, + ) + self.assertEqual( + rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype + ) + self.assertIn( + "locations", json_loads(rv.data), "No locations in response" + ) + locations = json_loads(rv.data)["locations"] + self.assertIn("nc_spm_08", locations, "Wrong location listed") + +# def test_location_info(self): +# rv = self.server.get( +# URL_PREFIX + "/locations/nc_spm_08/info", +# headers=self.admin_auth_header, +# ) +# print(rv.data) +# self.assertEqual( +# rv.status_code, +# 200, +# "HTML status code is wrong %i" % rv.status_code, +# ) +# self.assertEqual( +# rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype +# ) +# +# region_settings = json_loads(rv.data)["process_results"]["region"] +# json_loads(rv.data)["process_results"]["projection"] +# +# self.assertTrue("depths" in region_settings) +# self.assertTrue("ewres" in region_settings) +# self.assertTrue("cols" in region_settings) +# self.assertTrue("rows" in region_settings) +# +# def test_location_global_db_error(self): +# # ERROR: Try to create a location as admin that exists in the global +# # database +# rv = self.server.post( +# URL_PREFIX + "/locations/nc_spm_08", +# data=json_dumps({"epsg": "4326"}), +# content_type="application/json", +# headers=self.admin_auth_header, +# ) +# print(rv.data) +# self.assertEqual( +# rv.status_code, +# 400, +# "HTML status code is wrong %i" % rv.status_code, +# ) +# self.assertEqual( +# rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype +# ) +# +# def test_location_creation_and_deletion(self): +# # Delete a potentially existing location +# rv = self.server.delete( +# URL_PREFIX + "/locations/test_location", +# headers=self.admin_auth_header, +# ) +# +# # Create new location as admin +# rv = self.server.post( +# URL_PREFIX + "/locations/test_location", +# data=json_dumps({"epsg": "4326"}), +# content_type="application/json", +# headers=self.admin_auth_header, +# ) +# print(rv.data) +# self.assertEqual( +# rv.status_code, +# 200, +# "HTML status code is wrong %i" % rv.status_code, +# ) +# self.assertEqual( +# rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype +# ) +# +# # ERROR: Try to create a location as admin that already exists +# rv = self.server.post( +# URL_PREFIX + "/locations/test_location", +# data=json_dumps({"epsg": "4326"}), +# content_type="application/json", +# headers=self.admin_auth_header, +# ) +# print(rv.data) +# self.assertEqual( +# rv.status_code, +# 400, +# "HTML status code is wrong %i" % rv.status_code, +# ) +# self.assertEqual( +# rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype +# ) +# +# # Delete location +# rv = self.server.delete( +# URL_PREFIX + "/locations/test_location", +# headers=self.admin_auth_header, +# ) +# print(rv.data) +# self.assertEqual( +# rv.status_code, +# 200, +# "HTML status code is wrong %i" % rv.status_code, +# ) +# self.assertEqual( +# rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype +# ) +# +# # ERROR: Delete should fail, since location does not exists +# rv = self.server.delete( +# URL_PREFIX + "/locations/test_location", +# headers=self.admin_auth_header, +# ) +# print(rv.data) +# self.assertEqual( +# rv.status_code, +# 400, +# "HTML status code is wrong %i" % rv.status_code, +# ) +# self.assertEqual( +# rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype +# ) +# +# def test_location_creation_and_deletion_as_user(self): +# # Delete a potentially existing location +# rv = self.server.delete( +# URL_PREFIX + "/locations/test_location", +# headers=self.user_auth_header, +# ) +# +# # Create new location as user +# rv = self.server.post( +# URL_PREFIX + "/locations/test_location", +# data=json_dumps({"epsg": "4326"}), +# content_type="application/json", +# headers=self.user_auth_header, +# ) +# self.assertEqual( +# rv.status_code, +# 200, +# "Location creation by user: HTML status code is wrong %i" +# % rv.status_code, +# ) +# self.assertEqual( +# rv.mimetype, +# "application/json", +# "Location creation by user: Wrong mimetype %s" % rv.mimetype, +# ) +# +# # ERROR: Try to create a location as user that already exists +# rv = self.server.post( +# URL_PREFIX + "/locations/test_location", +# data=json_dumps({"epsg": "4326"}), +# content_type="application/json", +# headers=self.user_auth_header, +# ) +# self.assertEqual( +# rv.status_code, +# 400, +# "Location recreation by user: HTML status code is wrong %i" +# % rv.status_code, +# ) +# self.assertEqual( +# rv.mimetype, +# "application/json", +# "Location recreation by user: Wrong mimetype %s" % rv.mimetype, +# ) +# +# # Delete location +# rv = self.server.delete( +# URL_PREFIX + "/locations/test_location", +# headers=self.user_auth_header, +# ) +# self.assertEqual( +# rv.status_code, +# 200, +# "Location deletion by user: HTML status code is wrong %i" +# % rv.status_code, +# ) +# self.assertEqual( +# rv.mimetype, +# "application/json", +# "Location deletion by user: Wrong mimetype %s" % rv.mimetype, +# ) +# +# # ERROR: Delete should fail, since location does not exists +# rv = self.server.delete( +# URL_PREFIX + "/locations/test_location", +# headers=self.user_auth_header, +# ) +# self.assertEqual( +# rv.status_code, +# 400, +# "Location redeletion by user: HTML status code is wrong %i" +# % rv.status_code, +# ) +# self.assertEqual( +# rv.mimetype, +# "application/json", +# "Location redeletion by user: Wrong mimetype %s" % rv.mimetype, +# ) +# +# def test_location_creation_and_deletion_as_guest(self): +# # ERROR: Try to create a location as guest +# rv = self.server.post( +# URL_PREFIX + "/locations/test_location_user", +# data=json_dumps({"epsg": "4326"}), +# content_type="application/json", +# headers=self.guest_auth_header, +# ) +# print(rv.data) +# self.assertEqual( +# rv.status_code, +# 401, +# "HTML status code is wrong %i" % rv.status_code, +# ) +# self.assertEqual( +# rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype +# ) +# +# # ERROR: Delete should fail since the guest user is not authorized +# rv = self.server.delete( +# URL_PREFIX + "/locations/test_location_user", +# headers=self.guest_auth_header, +# ) +# print(rv.data) +# self.assertEqual( +# rv.status_code, +# 401, +# "HTML status code is wrong %i" % rv.status_code, +# ) +# self.assertEqual( +# rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype +# ) + + +if __name__ == "__main__": + unittest.main() From 891ced915dd6b53b82a87b94e44a3146c82bb999 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Fri, 25 Aug 2023 14:16:46 +0200 Subject: [PATCH 07/20] tests running with right config --- .dockerignore | 1 + .gitignore | 1 + Makefile | 3 + docker/actinia-core-tests/Dockerfile | 1 + .../actinia-test-noauth.cfg | 44 ++ pyproject.toml | 3 +- tests/test_noauth.py | 480 ++++++++---------- tests_with_redis.sh | 14 +- 8 files changed, 267 insertions(+), 280 deletions(-) create mode 100644 docker/actinia-core-tests/actinia-test-noauth.cfg diff --git a/.dockerignore b/.dockerignore index 1a384c6bd..79b41de55 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,6 +5,7 @@ docker !docker/actinia-core-dev/actinia.cfg !docker/actinia-core-dev/endpoints.csv !docker/actinia-core-tests/actinia-test.cfg +!docker/actinia-core-tests/actinia-test-noauth.cfg .github .travis .travis.yml diff --git a/.gitignore b/.gitignore index 6de62a88e..a5668ad23 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,4 @@ docker/redis_queue_data/dump.rdb !docker/actinia-core-dev/actinia.cfg !docker/actinia-core-dev/endpoints.csv !docker/actinia-core-tests/actinia-test.cfg +!docker/actinia-core-tests/actinia-test-noauth.cfg diff --git a/Makefile b/Makefile index 961cc499d..24448f1f5 100644 --- a/Makefile +++ b/Makefile @@ -18,5 +18,8 @@ unittest: devtest: sh ./tests_with_redis.sh dev +noauthtest: + sh ./tests_with_redis.sh noauth + integrationtest: sh ./tests_with_redis.sh integrationtest diff --git a/docker/actinia-core-tests/Dockerfile b/docker/actinia-core-tests/Dockerfile index c0e8cf9ba..ebf731eac 100644 --- a/docker/actinia-core-tests/Dockerfile +++ b/docker/actinia-core-tests/Dockerfile @@ -40,6 +40,7 @@ RUN grass --tmp-location EPSG:4326 --exec g.extension -s \ # copy needed files and configs for test COPY docker/actinia-core-alpine/actinia.cfg /etc/default/actinia COPY docker/actinia-core-tests/actinia-test.cfg /etc/default/actinia_test +COPY docker/actinia-core-tests/actinia-test-noauth.cfg /etc/default/actinia_test_noauth RUN pip3 install pytest pytest-cov diff --git a/docker/actinia-core-tests/actinia-test-noauth.cfg b/docker/actinia-core-tests/actinia-test-noauth.cfg new file mode 100644 index 000000000..b0a09f2fc --- /dev/null +++ b/docker/actinia-core-tests/actinia-test-noauth.cfg @@ -0,0 +1,44 @@ +[GRASS] +grass_gis_start_script = /usr/local/bin/grass +grass_database = /actinia_core/grassdb +grass_user_database = /actinia_core/userdata +grass_tmp_database = /actinia_core/workspace/temp_db +grass_resource_dir = /actinia_core/resources +grass_addon_path = /root/.grass8/addons/ +grass_gis_base = /usr/local/grass +grass_modules_xml_path = /usr/local/grass/gui/wxpython/xml/module_items.xml +grass_default_location = nc_spm_08 + +[API] +plugins = [] +force_https_urls = True +authentication = False + +[LIMITS] +max_cell_limit = 22500000 +process_time_limt = 60 +process_num_limit = 20 +number_of_workers = 3 + +[REDIS] +redis_server_url = localhost +redis_server_port = 6379 +redis_queue_server_url = localhost +redis_queue_server_port = 6379 +worker_queue_name = actinia_job +worker_logfile = /actinia_core/workspace/tmp/actinia_worker_test.log + +[LOGGING] +log_interface = fluentd +log_fluent_host = fluentd +log_fluent_port = 24224 +log_level = 1 + +[MISC] +tmp_workdir = /actinia_core/workspace/tmp +download_cache = /actinia_core/workspace/download_cache +secret_key = token_signing_key_changeme + +[MANAGEMENT] +default_user = user +default_user_group = group diff --git a/pyproject.toml b/pyproject.toml index 91b85376b..35ab53499 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,11 +98,12 @@ API_Docs = "https://redocly.github.io/redoc/?url=https://actinia.mundialis.de/la [tool.pytest.ini_options] minversion = "6.0" -addopts = "--cov actinia_core --cov-report term-missing --verbose --tb=line -x" +addopts = "--cov actinia_core --cov-report term-missing --verbose --tb=line -x -s" testpaths = [ "tests", ] markers = [ "dev: test current in development", "unittest: completely independent test", + "noauth: tests for actinia without authentication", ] diff --git a/tests/test_noauth.py b/tests/test_noauth.py index 08c1ac8d5..452711975 100644 --- a/tests/test_noauth.py +++ b/tests/test_noauth.py @@ -24,67 +24,87 @@ """ Tests: Location test case """ -import configparser -from flask.json import loads as json_loads, dumps as json_dumps import os +from flask.json import dumps as json_dumps +from flask.json import loads as json_loads import pytest -import requests import unittest from actinia_core.core.common.app import flask_app from actinia_core.core.common.config import global_config from actinia_core.core.common import redis_interface from actinia_core.core.common.process_queue import create_process_queue -from actinia_core.testsuite import ActiniaRequests # ActiniaTestCaseBase, URL_PREFIX, - - -try: - from .test_resource_base import ActiniaResourceTestCaseBase, URL_PREFIX -except ModuleNotFoundError: - from test_resource_base import ActiniaResourceTestCaseBase, URL_PREFIX +from actinia_core.testsuite import ( + ActiniaRequests, + ActiniaTestCaseBase, + URL_PREFIX, +) __license__ = "GPLv3" __author__ = "Anika Weinmann" -__copyright__ = ( - "Copyright 2023, mundialis GmbH & Co. KG" -) -__maintainer__ = "mundialis" +__copyright__ = "Copyright 2023, mundialis GmbH & Co. KG" +__maintainer__ = "mundialis GmbH & Co. KG" +__email__ = "info@mundialis.de" -class ActiniaWithoutAuthentication(ActiniaResourceTestCaseBase): +# Set this variable to use a actinia config file in a docker container +if "ACTINIA_CUSTOM_TEST_CFG" in os.environ: + custom_actinia_cfg = str(os.environ["ACTINIA_CUSTOM_TEST_CFG"]) + +PC = { + "version": 1, + "list": [ + { + "id": "1", + "module": "g.region", + "inputs": [{"param": "raster", "value": "elevation@PERMANENT"}], + "flags": "p", + }, + { + "id": "2", + "module": "r.colors.out_sld", + "inputs": [ + {"param": "map", "value": "elevation@PERMANENT"}, + {"param": "style_name", "value": "elevation"}, + ], + "stdout": {"id": "sld", "format": "list", "delimiter": "\n"}, + }, + { + "id": "3", + "module": "r.mapcalc", + "inputs": [{"param": "expression", "value": "baum = 5"}], + }, + { + "id": "4", + "module": "r.info", + "inputs": [{"param": "map", "value": "elevation@PERMANENT"}], + "flags": "g", + "stdout": {"id": "r_info_map1", "format": "kv", "delimiter": "="}, + }, + { + "id": "5", + "module": "g.list", + "inputs": [{"param": "type", "value": "raster"}], + }, + { + "id": "6", + "module": "r.info", + "inputs": [{"param": "map", "value": "baum"}], + "flags": "g", + "stdout": {"id": "r_info_map2", "format": "kv", "delimiter": "="}, + }, + ], +} + + +class ActiniaWithoutAuthentication(ActiniaTestCaseBase): """Test bsae class to test actinia without autentication""" server_test = False - cfg_file = os.environ.get( - "ACTINIA_CUSTOM_TEST_CFG", "/etc/default/actinia" - ) - auth_value = None - tmp_cfg_file = "%s_tmp" % cfg_file - auth = "False" if "ACTINIA_SERVER_TEST" in os.environ: server_test = bool(os.environ["ACTINIA_SERVER_TEST"]) - @classmethod - def save_config(cls, src, dest, value): - config = configparser.ConfigParser() - config.read(src) - if value: - config["API"]["authentication"] = value - else: - del config["API"]["authentication"] - with open(dest, "w") as configfile: - config.write(configfile) - @classmethod def setUpClass(cls): - cls.auth_value = global_config.AUTHENTICATION - if cls.auth_value is True: - os.replace(cls.cfg_file, cls.tmp_cfg_file) - cls.save_config( - cls.tmp_cfg_file, - cls.cfg_file, - cls.auth, - ) - if cls.server_test is False and cls.custom_actinia_cfg is False: global_config.REDIS_SERVER_SERVER = "localhost" global_config.REDIS_SERVER_PORT = 7000 @@ -98,6 +118,7 @@ def setUpClass(cls): # file will be read to configure Redis queue if cls.server_test is False and cls.custom_actinia_cfg is not False: global_config.read(cls.custom_actinia_cfg) + # global_config.read(cls.custom_actinia_cfg) # Start the redis interface redis_args = ( global_config.REDIS_SERVER_URL, @@ -116,10 +137,6 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): - # reset config - if cls.auth is True: - cls.save_config(cls.tmp_cfg_file, cls.cfg_file, "True") - os.remove(cls.tmp_cfg_file) if cls.server_test is False: redis_interface.disconnect() @@ -174,10 +191,9 @@ def waitAsyncStatusAssertHTTP( @pytest.mark.noauth class TestActiniaWithoutAuthentication(ActiniaWithoutAuthentication): - def test_version(self): - rv = self.server.get( - URL_PREFIX + "/version" - ) + def test_01_version(self): + """Test version endpoint""" + rv = self.server.get(f"{URL_PREFIX}/version") self.assertEqual( rv.status_code, 200, @@ -188,11 +204,9 @@ def test_version(self): self.assertIn("version", data) self.assertIn("grass_version", data) - def test_list_locations(self): - rv = self.server.get( - URL_PREFIX + "/locations" - ) - print(rv.data) + def test_02_list_locations(self): + """Test list location endpoint""" + rv = self.server.get(f"{URL_PREFIX}/locations") self.assertEqual( rv.status_code, 200, @@ -207,230 +221,140 @@ def test_list_locations(self): locations = json_loads(rv.data)["locations"] self.assertIn("nc_spm_08", locations, "Wrong location listed") -# def test_location_info(self): -# rv = self.server.get( -# URL_PREFIX + "/locations/nc_spm_08/info", -# headers=self.admin_auth_header, -# ) -# print(rv.data) -# self.assertEqual( -# rv.status_code, -# 200, -# "HTML status code is wrong %i" % rv.status_code, -# ) -# self.assertEqual( -# rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype -# ) -# -# region_settings = json_loads(rv.data)["process_results"]["region"] -# json_loads(rv.data)["process_results"]["projection"] -# -# self.assertTrue("depths" in region_settings) -# self.assertTrue("ewres" in region_settings) -# self.assertTrue("cols" in region_settings) -# self.assertTrue("rows" in region_settings) -# -# def test_location_global_db_error(self): -# # ERROR: Try to create a location as admin that exists in the global -# # database -# rv = self.server.post( -# URL_PREFIX + "/locations/nc_spm_08", -# data=json_dumps({"epsg": "4326"}), -# content_type="application/json", -# headers=self.admin_auth_header, -# ) -# print(rv.data) -# self.assertEqual( -# rv.status_code, -# 400, -# "HTML status code is wrong %i" % rv.status_code, -# ) -# self.assertEqual( -# rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype -# ) -# -# def test_location_creation_and_deletion(self): -# # Delete a potentially existing location -# rv = self.server.delete( -# URL_PREFIX + "/locations/test_location", -# headers=self.admin_auth_header, -# ) -# -# # Create new location as admin -# rv = self.server.post( -# URL_PREFIX + "/locations/test_location", -# data=json_dumps({"epsg": "4326"}), -# content_type="application/json", -# headers=self.admin_auth_header, -# ) -# print(rv.data) -# self.assertEqual( -# rv.status_code, -# 200, -# "HTML status code is wrong %i" % rv.status_code, -# ) -# self.assertEqual( -# rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype -# ) -# -# # ERROR: Try to create a location as admin that already exists -# rv = self.server.post( -# URL_PREFIX + "/locations/test_location", -# data=json_dumps({"epsg": "4326"}), -# content_type="application/json", -# headers=self.admin_auth_header, -# ) -# print(rv.data) -# self.assertEqual( -# rv.status_code, -# 400, -# "HTML status code is wrong %i" % rv.status_code, -# ) -# self.assertEqual( -# rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype -# ) -# -# # Delete location -# rv = self.server.delete( -# URL_PREFIX + "/locations/test_location", -# headers=self.admin_auth_header, -# ) -# print(rv.data) -# self.assertEqual( -# rv.status_code, -# 200, -# "HTML status code is wrong %i" % rv.status_code, -# ) -# self.assertEqual( -# rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype -# ) -# -# # ERROR: Delete should fail, since location does not exists -# rv = self.server.delete( -# URL_PREFIX + "/locations/test_location", -# headers=self.admin_auth_header, -# ) -# print(rv.data) -# self.assertEqual( -# rv.status_code, -# 400, -# "HTML status code is wrong %i" % rv.status_code, -# ) -# self.assertEqual( -# rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype -# ) -# -# def test_location_creation_and_deletion_as_user(self): -# # Delete a potentially existing location -# rv = self.server.delete( -# URL_PREFIX + "/locations/test_location", -# headers=self.user_auth_header, -# ) -# -# # Create new location as user -# rv = self.server.post( -# URL_PREFIX + "/locations/test_location", -# data=json_dumps({"epsg": "4326"}), -# content_type="application/json", -# headers=self.user_auth_header, -# ) -# self.assertEqual( -# rv.status_code, -# 200, -# "Location creation by user: HTML status code is wrong %i" -# % rv.status_code, -# ) -# self.assertEqual( -# rv.mimetype, -# "application/json", -# "Location creation by user: Wrong mimetype %s" % rv.mimetype, -# ) -# -# # ERROR: Try to create a location as user that already exists -# rv = self.server.post( -# URL_PREFIX + "/locations/test_location", -# data=json_dumps({"epsg": "4326"}), -# content_type="application/json", -# headers=self.user_auth_header, -# ) -# self.assertEqual( -# rv.status_code, -# 400, -# "Location recreation by user: HTML status code is wrong %i" -# % rv.status_code, -# ) -# self.assertEqual( -# rv.mimetype, -# "application/json", -# "Location recreation by user: Wrong mimetype %s" % rv.mimetype, -# ) -# -# # Delete location -# rv = self.server.delete( -# URL_PREFIX + "/locations/test_location", -# headers=self.user_auth_header, -# ) -# self.assertEqual( -# rv.status_code, -# 200, -# "Location deletion by user: HTML status code is wrong %i" -# % rv.status_code, -# ) -# self.assertEqual( -# rv.mimetype, -# "application/json", -# "Location deletion by user: Wrong mimetype %s" % rv.mimetype, -# ) -# -# # ERROR: Delete should fail, since location does not exists -# rv = self.server.delete( -# URL_PREFIX + "/locations/test_location", -# headers=self.user_auth_header, -# ) -# self.assertEqual( -# rv.status_code, -# 400, -# "Location redeletion by user: HTML status code is wrong %i" -# % rv.status_code, -# ) -# self.assertEqual( -# rv.mimetype, -# "application/json", -# "Location redeletion by user: Wrong mimetype %s" % rv.mimetype, -# ) -# -# def test_location_creation_and_deletion_as_guest(self): -# # ERROR: Try to create a location as guest -# rv = self.server.post( -# URL_PREFIX + "/locations/test_location_user", -# data=json_dumps({"epsg": "4326"}), -# content_type="application/json", -# headers=self.guest_auth_header, -# ) -# print(rv.data) -# self.assertEqual( -# rv.status_code, -# 401, -# "HTML status code is wrong %i" % rv.status_code, -# ) -# self.assertEqual( -# rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype -# ) -# -# # ERROR: Delete should fail since the guest user is not authorized -# rv = self.server.delete( -# URL_PREFIX + "/locations/test_location_user", -# headers=self.guest_auth_header, -# ) -# print(rv.data) -# self.assertEqual( -# rv.status_code, -# 401, -# "HTML status code is wrong %i" % rv.status_code, -# ) -# self.assertEqual( -# rv.mimetype, "application/json", "Wrong mimetype %s" % rv.mimetype -# ) + def test_03_processing_ephemeral(self): + """Test job resumption with processing_async endpoint and stdout""" + endpoint = "/locations/nc_spm_08/processing_async" + rv = self.server.post( + f"{URL_PREFIX}{endpoint}", + data=json_dumps(PC), + content_type="application/json", + ) + resp = self.waitAsyncStatusAssertHTTP( + rv, + http_status=200, + status="finished", + ) + self.compare_stdout(resp) + + def test_04_processing_persistent(self): + """Test job resumption with persistent processing_async endpoint and + stdout + """ + endpoint = "/locations/nc_spm_08/mapsets/test/processing_async" + rv = self.server.post( + f"{URL_PREFIX}{endpoint}", + data=json_dumps(PC), + content_type="application/json", + ) + resp = self.waitAsyncStatusAssertHTTP( + rv, + http_status=200, + status="finished", + ) + self.compare_stdout(resp) + # check processing mapset + rv2 = self.server.get(f"{URL_PREFIX}/locations/nc_spm_08/mapsets") + self.assertEqual( + rv2.status_code, + 200, + f"HTML status code is wrong {rv2.status_code}", + ) + data = json_loads(rv2.data) + self.assertIn( + "test", data["process_results"], "Mapset 'test' does not exists" + ) + # check created raster + rv3 = self.server.get( + f"{URL_PREFIX}/locations/nc_spm_08/mapsets/test/raster_layers" + ) + self.assertEqual( + rv3.status_code, + 200, + f"HTML status code is wrong {rv3.status_code}", + ) + data = json_loads(rv3.data) + self.assertIn( + "baum", data["process_results"], "Raster 'baum' does not exists" + ) + # delete test mapset + self.admin_auth_header = None + self.delete_mapset("test", "nc_spm_08") + rv4 = self.server.get(f"{URL_PREFIX}/locations/nc_spm_08/mapsets") + self.assertEqual( + rv4.status_code, + 200, + f"HTML status code is wrong {rv4.status_code}", + ) + data = json_loads(rv4.data) + self.assertNotIn( + "test", data["process_results"], "Mapset 'test' exists" + ) + + def compare_stdout(self, resp): + proc_results = resp["process_results"] + self.assertIn( + "sld", proc_results, "'sld' not saved in process results" + ) + self.assertIn( + "r_info_map1", + proc_results, + "'r_info_map1' not saved in process results", + ) + self.assertIn( + "r_info_map2", + proc_results, + "'r_info_map2' not saved in process results", + ) + self.assertIsInstance( + proc_results["sld"], list, "'sld' result is not of type list" + ) + self.assertIsInstance( + proc_results["r_info_map1"], + dict, + "'r_info_map1' result is not of type dict", + ) + self.assertIsInstance( + proc_results["r_info_map2"], + dict, + "'r_info_map2' result is not of type dict", + ) + r_info_map1 = { + "cells": "2025000", + "cols": "1500", + "datatype": "FCELL", + "east": "645000", + "ewres": "10", + "ncats": "255", + "north": "228500", + "nsres": "10", + "rows": "1350", + "south": "215000", + "west": "630000", + } + self.assertEqual( + proc_results["r_info_map1"], + r_info_map1, + "'r_info_map1' is not equal to the region", + ) + r_info_map2 = { + "cells": "2025000", + "cols": "1500", + "datatype": "CELL", + "east": "645000", + "ewres": "10", + "ncats": "0", + "north": "228500", + "nsres": "10", + "rows": "1350", + "south": "215000", + "west": "630000", + } + self.assertEqual( + proc_results["r_info_map2"], + r_info_map2, + "'r_info_map2' is not equal to the region", + ) + if __name__ == "__main__": diff --git a/tests_with_redis.sh b/tests_with_redis.sh index 0e390dcb1..776307d93 100644 --- a/tests_with_redis.sh +++ b/tests_with_redis.sh @@ -1,5 +1,14 @@ #!/usr/bin/env sh +run_tests_notauth (){ + # change config run tests and change config bag + mv ${ACTINIA_CUSTOM_TEST_CFG} ${ACTINIA_CUSTOM_TEST_CFG}_tmp + cp /etc/default/actinia_test_noauth ${ACTINIA_CUSTOM_TEST_CFG} + pytest -m 'noauth' + mv ${ACTINIA_CUSTOM_TEST_CFG}_tmp ${ACTINIA_CUSTOM_TEST_CFG} +} + + # start redis server redis-server & sleep 1 @@ -19,7 +28,10 @@ then pytest -m 'dev' elif [ "$1" == "integrationtest" ] then - pytest -m 'not unittest' + pytest -m 'not unittest' -m 'not noauth' # + no auth +elif [ "$1" == "noauth" ] +then + run_tests_notauth else pytest fi From 29be56b1e6a5aa2d723e71b78e948130ecc1b040 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Fri, 25 Aug 2023 14:26:25 +0200 Subject: [PATCH 08/20] adjust test script --- tests_with_redis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests_with_redis.sh b/tests_with_redis.sh index 776307d93..cfb2d4980 100644 --- a/tests_with_redis.sh +++ b/tests_with_redis.sh @@ -28,7 +28,7 @@ then pytest -m 'dev' elif [ "$1" == "integrationtest" ] then - pytest -m 'not unittest' -m 'not noauth' # + no auth + pytest -m 'not unittest and not noauth' elif [ "$1" == "noauth" ] then run_tests_notauth From 35c75aa610a69f65aea9c6a2ea160690f028d699 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Fri, 25 Aug 2023 14:40:18 +0200 Subject: [PATCH 09/20] lintign --- tests/test_noauth.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_noauth.py b/tests/test_noauth.py index 452711975..c1dc0be7d 100644 --- a/tests/test_noauth.py +++ b/tests/test_noauth.py @@ -98,6 +98,7 @@ class ActiniaWithoutAuthentication(ActiniaTestCaseBase): """Test bsae class to test actinia without autentication""" + server_test = False if "ACTINIA_SERVER_TEST" in os.environ: @@ -190,7 +191,6 @@ def waitAsyncStatusAssertHTTP( @pytest.mark.noauth class TestActiniaWithoutAuthentication(ActiniaWithoutAuthentication): - def test_01_version(self): """Test version endpoint""" rv = self.server.get(f"{URL_PREFIX}/version") @@ -356,6 +356,5 @@ def compare_stdout(self, resp): ) - if __name__ == "__main__": unittest.main() From d5397741087c8eb43554543536f6ccbde31c1b91 Mon Sep 17 00:00:00 2001 From: Anika Weinmann <37300249+anikaweinmann@users.noreply.github.com> Date: Fri, 25 Aug 2023 15:22:12 +0200 Subject: [PATCH 10/20] Apply suggestions from code review Co-authored-by: Carmen Tawalika --- tests/test_noauth.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_noauth.py b/tests/test_noauth.py index c1dc0be7d..c9a010029 100644 --- a/tests/test_noauth.py +++ b/tests/test_noauth.py @@ -4,7 +4,7 @@ # performance processing of geographical data that uses GRASS GIS for # computational tasks. For details, see https://actinia.mundialis.de/ # -# Copyright (c) 2016-2018 Sören Gebbert and mundialis GmbH & Co. KG +# Copyright (c) 2023 Sören Gebbert and mundialis GmbH & Co. KG # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -97,7 +97,7 @@ class ActiniaWithoutAuthentication(ActiniaTestCaseBase): - """Test bsae class to test actinia without autentication""" + """Test base class to test actinia without autentication""" server_test = False From f55e41ddcec3c9d67716ad2ac3427daa6f46f8de Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Fri, 25 Aug 2023 15:31:37 +0200 Subject: [PATCH 11/20] CT review --- .../actinia-test-noauth.cfg | 20 ------------------- docker/actinia-core-tests/actinia-test.cfg | 20 ------------------- tests_with_redis.sh | 1 + 3 files changed, 1 insertion(+), 40 deletions(-) diff --git a/docker/actinia-core-tests/actinia-test-noauth.cfg b/docker/actinia-core-tests/actinia-test-noauth.cfg index b0a09f2fc..4eef9393c 100644 --- a/docker/actinia-core-tests/actinia-test-noauth.cfg +++ b/docker/actinia-core-tests/actinia-test-noauth.cfg @@ -1,16 +1,10 @@ [GRASS] -grass_gis_start_script = /usr/local/bin/grass grass_database = /actinia_core/grassdb grass_user_database = /actinia_core/userdata grass_tmp_database = /actinia_core/workspace/temp_db grass_resource_dir = /actinia_core/resources -grass_addon_path = /root/.grass8/addons/ -grass_gis_base = /usr/local/grass -grass_modules_xml_path = /usr/local/grass/gui/wxpython/xml/module_items.xml -grass_default_location = nc_spm_08 [API] -plugins = [] force_https_urls = True authentication = False @@ -18,27 +12,13 @@ authentication = False max_cell_limit = 22500000 process_time_limt = 60 process_num_limit = 20 -number_of_workers = 3 [REDIS] -redis_server_url = localhost -redis_server_port = 6379 -redis_queue_server_url = localhost -redis_queue_server_port = 6379 -worker_queue_name = actinia_job worker_logfile = /actinia_core/workspace/tmp/actinia_worker_test.log [LOGGING] -log_interface = fluentd -log_fluent_host = fluentd -log_fluent_port = 24224 log_level = 1 [MISC] tmp_workdir = /actinia_core/workspace/tmp download_cache = /actinia_core/workspace/download_cache -secret_key = token_signing_key_changeme - -[MANAGEMENT] -default_user = user -default_user_group = group diff --git a/docker/actinia-core-tests/actinia-test.cfg b/docker/actinia-core-tests/actinia-test.cfg index 004a9c9f2..728b0c712 100644 --- a/docker/actinia-core-tests/actinia-test.cfg +++ b/docker/actinia-core-tests/actinia-test.cfg @@ -1,43 +1,23 @@ [GRASS] -grass_gis_start_script = /usr/local/bin/grass grass_database = /actinia_core/grassdb grass_user_database = /actinia_core/userdata grass_tmp_database = /actinia_core/workspace/temp_db grass_resource_dir = /actinia_core/resources -grass_addon_path = /root/.grass8/addons/ -grass_gis_base = /usr/local/grass -grass_modules_xml_path = /usr/local/grass/gui/wxpython/xml/module_items.xml -grass_default_location = nc_spm_08 [API] -plugins = [] force_https_urls = True [LIMITS] max_cell_limit = 22500000 process_time_limt = 60 process_num_limit = 20 -number_of_workers = 3 [REDIS] -redis_server_url = localhost -redis_server_port = 6379 -redis_queue_server_url = localhost -redis_queue_server_port = 6379 -worker_queue_name = actinia_job worker_logfile = /actinia_core/workspace/tmp/actinia_worker_test.log [LOGGING] -log_interface = fluentd -log_fluent_host = fluentd -log_fluent_port = 24224 log_level = 1 [MISC] tmp_workdir = /actinia_core/workspace/tmp download_cache = /actinia_core/workspace/download_cache -secret_key = token_signing_key_changeme - -[MANAGEMENT] -default_user = user -default_user_group = group diff --git a/tests_with_redis.sh b/tests_with_redis.sh index cfb2d4980..3b1399b23 100644 --- a/tests_with_redis.sh +++ b/tests_with_redis.sh @@ -29,6 +29,7 @@ then elif [ "$1" == "integrationtest" ] then pytest -m 'not unittest and not noauth' + run_tests_notauth elif [ "$1" == "noauth" ] then run_tests_notauth From 1a04a29a9f5c68dba6a71b1a66367e9f44aaebdc Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Wed, 30 Aug 2023 07:53:31 +0200 Subject: [PATCH 12/20] MN review --- tests_with_redis.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests_with_redis.sh b/tests_with_redis.sh index 3b1399b23..c3b17d5c0 100644 --- a/tests_with_redis.sh +++ b/tests_with_redis.sh @@ -1,6 +1,6 @@ #!/usr/bin/env sh -run_tests_notauth (){ +run_tests_noauth (){ # change config run tests and change config bag mv ${ACTINIA_CUSTOM_TEST_CFG} ${ACTINIA_CUSTOM_TEST_CFG}_tmp cp /etc/default/actinia_test_noauth ${ACTINIA_CUSTOM_TEST_CFG} @@ -29,10 +29,10 @@ then elif [ "$1" == "integrationtest" ] then pytest -m 'not unittest and not noauth' - run_tests_notauth + run_tests_noauth elif [ "$1" == "noauth" ] then - run_tests_notauth + run_tests_noauth else pytest fi From bd2b9c32056641951b83b452a5eed7e084c64da8 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Wed, 30 Aug 2023 08:23:21 +0200 Subject: [PATCH 13/20] fix redis settings for tests --- docker/actinia-core-tests/actinia-test-noauth.cfg | 2 ++ docker/actinia-core-tests/actinia-test.cfg | 2 ++ tests/test_common_base.py | 1 - 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/actinia-core-tests/actinia-test-noauth.cfg b/docker/actinia-core-tests/actinia-test-noauth.cfg index 4eef9393c..3d8b88483 100644 --- a/docker/actinia-core-tests/actinia-test-noauth.cfg +++ b/docker/actinia-core-tests/actinia-test-noauth.cfg @@ -15,6 +15,8 @@ process_num_limit = 20 [REDIS] worker_logfile = /actinia_core/workspace/tmp/actinia_worker_test.log +redis_server_url = localhost +redis_server_port = 6379 [LOGGING] log_level = 1 diff --git a/docker/actinia-core-tests/actinia-test.cfg b/docker/actinia-core-tests/actinia-test.cfg index 728b0c712..472081fb0 100644 --- a/docker/actinia-core-tests/actinia-test.cfg +++ b/docker/actinia-core-tests/actinia-test.cfg @@ -14,6 +14,8 @@ process_num_limit = 20 [REDIS] worker_logfile = /actinia_core/workspace/tmp/actinia_worker_test.log +redis_server_url = localhost +redis_server_port = 6379 [LOGGING] log_level = 1 diff --git a/tests/test_common_base.py b/tests/test_common_base.py index a99c881a5..839eeeadd 100644 --- a/tests/test_common_base.py +++ b/tests/test_common_base.py @@ -103,7 +103,6 @@ class CommonTestCaseBase(unittest.TestCase): def setUpClass(cls): if custom_actinia_cfg is not False: global_config.read(custom_actinia_cfg) - print(global_config) else: global_config.REDIS_SERVER_URL = "localhost" global_config.REDIS_SERVER_PORT = 7000 From 864fe4d4cb368387de7e152e9b9f192014804ebd Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Wed, 30 Aug 2023 08:32:48 +0200 Subject: [PATCH 14/20] improve tests_with_redis.sh script --- tests_with_redis.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests_with_redis.sh b/tests_with_redis.sh index c3b17d5c0..49516b405 100644 --- a/tests_with_redis.sh +++ b/tests_with_redis.sh @@ -29,7 +29,12 @@ then elif [ "$1" == "integrationtest" ] then pytest -m 'not unittest and not noauth' - run_tests_noauth + if [ $? -eq 0] + then + run_tests_noauth + else + echo "Skipping tests without authentication" + fi elif [ "$1" == "noauth" ] then run_tests_noauth From 29a6ae73cb4e67a655349c0f3597af83aaf093f1 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Wed, 30 Aug 2023 09:28:13 +0200 Subject: [PATCH 15/20] echo returncode --- tests_with_redis.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests_with_redis.sh b/tests_with_redis.sh index 49516b405..6955af784 100644 --- a/tests_with_redis.sh +++ b/tests_with_redis.sh @@ -29,11 +29,12 @@ then elif [ "$1" == "integrationtest" ] then pytest -m 'not unittest and not noauth' - if [ $? -eq 0] + INTEGRATIONTEST_RETURN=$? + if [ ${INTEGRATIONTEST_RETURN} -eq 0] then run_tests_noauth else - echo "Skipping tests without authentication" + echo "Skipping tests without authentication scince other tests failed with ${INTEGRATIONTEST_RETURN}" fi elif [ "$1" == "noauth" ] then From 45ca12aa77ba91246b35223180d704c844931053 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Wed, 30 Aug 2023 12:56:59 +0200 Subject: [PATCH 16/20] try to fix --- tests_with_redis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests_with_redis.sh b/tests_with_redis.sh index 6955af784..630c0f7a0 100644 --- a/tests_with_redis.sh +++ b/tests_with_redis.sh @@ -30,7 +30,7 @@ elif [ "$1" == "integrationtest" ] then pytest -m 'not unittest and not noauth' INTEGRATIONTEST_RETURN=$? - if [ ${INTEGRATIONTEST_RETURN} -eq 0] + if [ ${INTEGRATIONTEST_RETURN} -eq 0 ] then run_tests_noauth else From c23c52bb6ab197a7bb82f5de59fb8d937ad6df8b Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Wed, 30 Aug 2023 13:56:07 +0200 Subject: [PATCH 17/20] remove return code from message --- tests_with_redis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests_with_redis.sh b/tests_with_redis.sh index 630c0f7a0..a4a4a1c63 100644 --- a/tests_with_redis.sh +++ b/tests_with_redis.sh @@ -34,7 +34,7 @@ then then run_tests_noauth else - echo "Skipping tests without authentication scince other tests failed with ${INTEGRATIONTEST_RETURN}" + echo "Skipping tests without authentication scince other tests failed" fi elif [ "$1" == "noauth" ] then From ac4587d91be41ff32239b9bdd7d6656c42b29191 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Wed, 30 Aug 2023 13:57:48 +0200 Subject: [PATCH 18/20] typo --- tests_with_redis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests_with_redis.sh b/tests_with_redis.sh index a4a4a1c63..6daa00034 100644 --- a/tests_with_redis.sh +++ b/tests_with_redis.sh @@ -34,7 +34,7 @@ then then run_tests_noauth else - echo "Skipping tests without authentication scince other tests failed" + echo "Skipping tests without authentication since other tests failed" fi elif [ "$1" == "noauth" ] then From d71f96e30ccb7b14bec9fe7b6bb528a41b613d28 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Wed, 30 Aug 2023 18:40:22 +0200 Subject: [PATCH 19/20] try pipeline failure --- tests/unittests/test_version.py | 1 + tests_with_redis.sh | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/unittests/test_version.py b/tests/unittests/test_version.py index f07ebf1d3..326781602 100644 --- a/tests/unittests/test_version.py +++ b/tests/unittests/test_version.py @@ -61,6 +61,7 @@ ) def test_valid_additional_version_info_key(key, status): test = valid_additional_version_info_key(key) + assert "b" == "v" if status is True: assert test, f"Key <{key}> is wrong" else: diff --git a/tests_with_redis.sh b/tests_with_redis.sh index 6daa00034..42e4b668d 100644 --- a/tests_with_redis.sh +++ b/tests_with_redis.sh @@ -22,29 +22,32 @@ sleep 10 echo $ACTINIA_CUSTOM_TEST_CFG echo $DEFAULT_CONFIG_PATH +TEST_RES=1 if [ "$1" == "dev" ] then echo "Executing only 'dev' tests ..." pytest -m 'dev' + TEST_RES=$? elif [ "$1" == "integrationtest" ] then pytest -m 'not unittest and not noauth' - INTEGRATIONTEST_RETURN=$? - if [ ${INTEGRATIONTEST_RETURN} -eq 0 ] + TEST_RES=$? + if [ ${TEST_RES} -eq 0 ] then run_tests_noauth + TEST_RES=$? else echo "Skipping tests without authentication since other tests failed" fi elif [ "$1" == "noauth" ] then run_tests_noauth + TEST_RES=$? else pytest + TEST_RES=$? fi -TEST_RES=$? - # stop redis server redis-cli shutdown From 382be42a7ffdc7ad2bbc673efd4d85db2694ac21 Mon Sep 17 00:00:00 2001 From: anikaweinmann Date: Wed, 30 Aug 2023 18:45:23 +0200 Subject: [PATCH 20/20] refix unittests --- tests/unittests/test_version.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unittests/test_version.py b/tests/unittests/test_version.py index 326781602..f07ebf1d3 100644 --- a/tests/unittests/test_version.py +++ b/tests/unittests/test_version.py @@ -61,7 +61,6 @@ ) def test_valid_additional_version_info_key(key, status): test = valid_additional_version_info_key(key) - assert "b" == "v" if status is True: assert test, f"Key <{key}> is wrong" else: