Add Debian support

alts/scheduler/monitoring.py

@@ -39,7 +39,7 @@ def get_available_test_tasks(self) -> List[dict]:
             response = requests.get(
                 urllib.parse.urljoin(
                     CONFIG.bs_host,
-                    CONFIG.bs_get_task_endpoint,
+                    CONFIG.bs_tasks_endpoint,
                 ),
                 headers={'Authorization': f'Bearer {CONFIG.bs_token}'},
             ).json()

alts/shared/models.py

@@ -254,6 +254,7 @@ def __init__(self, **data):
     azure_connection_string: typing.Optional[str] = None
     task_default_queue: str = 'default'
     task_acks_late: bool = True
+    enable_integrity_tests: bool = True
     task_track_started: bool = True
     worker_prefetch_multiplier: int = 1
     broker_pool_limit: int = 20
@@ -274,7 +275,7 @@ def __init__(self, **data):
     ssh_public_key_path: str = '~/.ssh/id_rsa.pub'
     # Build system settings
     bs_host: typing.Optional[str] = None
-    bs_get_task_endpoint: str = '/api/v1/tests/get_test_tasks/'
+    bs_tasks_endpoint: str = '/api/v1/tests/get_test_tasks/'
     bs_token: typing.Optional[str] = None
     # Log uploader settings
     logs_uploader_config: typing.Optional[

alts/worker/executors/command.py

@@ -35,7 +35,11 @@ def run_local_command(self, cmd_args: List[str]) -> CommandResult:
         return super().run_local_command(cmd_args)
 
     @measure_stage('run_signle_ssh_command')
-    def run_ssh_command(self, cmd_args: List[str]) -> CommandResult:
+    def run_ssh_command(
+        self,
+        cmd_args: List[str],
+        workdir: str = '',
+    ) -> CommandResult:
         return super().run_ssh_command(cmd_args)
 
     @measure_stage('run_signle_docker_command')

alts/worker/runners/base.py

@@ -8,6 +8,7 @@
 import shutil
 import tempfile
 import time
+import urllib.parse
 from functools import wraps
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Union
@@ -180,6 +181,7 @@ def __init__(
 
         # Package installation and test stuff
         self._repositories = repositories or []
+        self.add_credentials_in_deb_repos()
 
         self._artifacts = {}
         self._uploaded_logs = None
@@ -248,6 +250,24 @@ def default_ssh_params(self) -> Dict[str, Any]:
             'ignore_encrypted_keys': True,
         }
 
+    def add_credentials_in_deb_repos(self):
+        for repo in self._repositories:
+            if '-br' not in repo['name'] or 'amd64' not in repo['url']:
+                continue
+            parsed = urllib.parse.urlparse(repo['url'])
+            netloc = f'alts:{CONFIG.bs_token}@{parsed.netloc}'
+            url = urllib.parse.urlunparse(
+                (
+                    parsed.scheme,
+                    netloc,
+                    parsed.path,
+                    parsed.params,
+                    parsed.query,
+                    parsed.fragment,
+                )
+            )
+            repo['url'] = f'deb {url} ./'
+
     def __init_task_logger(self, log_file):
         """
         Task logger initialization, configures a test task   logger to write
@@ -703,7 +723,7 @@ def run_package_integrity_tests(
         full_pkg_name = package_name
         if package_version:
             full_pkg_name = f'{package_name}-{package_version}'
-            cmd_args.extend(['--package-version', package_version])
+            cmd_args.extend(('--package-version', package_version))
         cmd_args.append('tests')
         self._logger.info(
             'Running package integrity tests for %s on %s...',

alts/worker/tasks.py

@@ -142,7 +142,8 @@ def set_artifacts_when_stage_has_unexpected_exception(
             module_stream=module_stream,
             module_version=module_version,
         )
-        runner.run_package_integrity_tests(package_name, package_version)
+        if CONFIG.enable_integrity_tests:
+            runner.run_package_integrity_tests(package_name, package_version)
         runner.run_third_party_tests()
         runner.uninstall_package(
             package_name,

configs/example_config.yaml

@@ -43,7 +43,7 @@ opennebula_config:
   network:
 
 bs_host:
-bs_get_task_endpoint: '/api/v1/tests/get_test_tasks/'
+bs_tasks_endpoint: '/api/v1/tests/get_test_tasks/'
 bs_token:
 logs_uploader_config:
   pulp_host:

resources/package_tests/tests/test_package_is_correct.py

@@ -1,7 +1,16 @@
 import pytest_check as check
 from testinfra.modules.package import RpmPackage
 
-from ..base import *
+from ..base import (
+    get_package_files,
+    get_shared_libraries,
+    has_missing_shared_libraries,
+    is_debuginfo_package,
+    is_file_dynamically_linked,
+    is_package_empty,
+    is_rpath_correct,
+    resolve_symlink,
+)
 
 
 def test_package_is_installed(host, package_name, package_version):
@@ -26,15 +35,17 @@ def test_package_is_installed(host, package_name, package_version):
     check.is_true(package.is_installed)
     if package_version:
         check.is_in(
-            package.version, package_version,
-            'Version does not match: required: {0}, actual: {1}'.format(
-                package_version, package.version)
+            package.version,
+            package_version,
+            f'Version does not match: required: {package_version}, '
+            f'actual: {package.version}',
         )
         if isinstance(package, RpmPackage):
             check.is_in(
-                package.release, package_version,
-                'Release does not match: required: {0}, actual: {1}'.format(
-                    package_version, package.release)
+                package.release,
+                package_version,
+                f'Release does not match: required: {package_version}, '
+                f'actual: {package.release}',
             )
 
 
@@ -64,16 +75,20 @@ def test_all_package_files_exist(host, package_name):
         file_obj = host.file(file_)
         check.is_true(
             file_obj.exists,
-            'File is absent on the file system: {}'.format(file_)
+            f'File is absent on the file system: {file_}',
         )
         # Check all symlinks point to existing files
         if file_obj.is_symlink:
             resolved = resolve_symlink(host, file_obj)
             check.is_not_none(
-                resolved, 'Symlink cannot be resolved: {}'.format(file_))
+                resolved,
+                f'Symlink cannot be resolved: {file_}',
+            )
             file_obj = host.file(resolved)
             check.is_true(
-                file_obj.exists, 'Symlink is broken: {}'.format(file_))
+                file_obj.exists,
+                f'Symlink is broken: {file_}',
+            )
 
 
 def test_binaries_have_all_dependencies(host, package_name):
@@ -130,5 +145,7 @@ def test_check_rpath_is_correct(host, package_name):
         return
 
     for library in get_shared_libraries(package):
-        check.is_true(is_rpath_correct(host.file(library)),
-                      'RPATH is broken in {}'.format(library))
+        check.is_true(
+            is_rpath_correct(host.file(library)),
+            f'RPATH is broken in {library}',
+        )

resources/playbook.yml

@@ -35,6 +35,11 @@
             name: "*"
             state: latest
 
+        - name: Install additional packages
+          apt:
+            name: "python3-pip"
+            state: latest
+
         - name: Add required repositories
           apt_repository:
             repo: "{{ item.url }}"
@@ -118,6 +123,7 @@
       apt:
         name: "{{ pkg_name }}"
         state: present
+        allow_unauthenticated: true
       when: ansible_distribution_file_variety == 'Debian'
       tags:
         - install_package