diff --git a/devlib/__init__.py b/devlib/__init__.py index dceda0d62..e496299b1 100644 --- a/devlib/__init__.py +++ b/devlib/__init__.py @@ -22,8 +22,6 @@ ChromeOsTarget, ) -from devlib.target_runner import QEMUTargetRunner - from devlib.host import ( PACKAGE_BIN_DIRECTORY, LocalConnection, diff --git a/devlib/target_runner.py b/devlib/_target_runner.py similarity index 73% rename from devlib/target_runner.py rename to devlib/_target_runner.py index c08c3a1ab..b86c6b07c 100644 --- a/devlib/target_runner.py +++ b/devlib/_target_runner.py @@ -19,8 +19,6 @@ import logging import os -import signal -import subprocess import time from platform import machine @@ -37,13 +35,14 @@ class TargetRunner: It mainly aims to provide framework support for QEMU like target runners (e.g., :class:`QEMUTargetRunner`). - :param runner_cmd: The command to start runner process (e.g., - ``qemu-system-aarch64 -kernel Image -append "console=ttyAMA0" ...``). - :type runner_cmd: str - :param target: Specifies type of target per :class:`Target` based classes. :type target: Target + :param runner_cmd: The command to start runner process (e.g., + ``qemu-system-aarch64 -kernel Image -append "console=ttyAMA0" ...``). + Defaults to None. + :type runner_cmd: str, optional + :param connect: Specifies if :class:`TargetRunner` should try to connect target after launching it, defaults to True. :type connect: bool, optional @@ -58,15 +57,19 @@ class TargetRunner: """ def __init__(self, - runner_cmd, target, + runner_cmd=None, connect=True, boot_timeout=60): self.boot_timeout = boot_timeout self.target = target + self.runner_process = None self.logger = logging.getLogger(self.__class__.__name__) + if runner_cmd is None: + return + self.logger.info('runner_cmd: %s', runner_cmd) try: @@ -91,7 +94,7 @@ def __exit__(self, *_): """ Exit routine for contextmanager. - Ensure :attr:`TargetRunner.runner_process` is terminated on exit. + Ensure ``TargetRunner.runner_process`` is terminated on exit. """ self.terminate() @@ -99,7 +102,7 @@ def __exit__(self, *_): def wait_boot_complete(self): """ Wait for target OS to finish boot up and become accessible over SSH in at most - :attr:`TargetRunner.boot_timeout` seconds. + ``TargetRunner.boot_timeout`` seconds. :raises TargetStableError: In case of timeout. """ @@ -109,10 +112,10 @@ def wait_boot_complete(self): while self.boot_timeout >= elapsed: try: self.target.connect(timeout=self.boot_timeout - elapsed) - self.logger.info('Target is ready.') + self.logger.debug('Target is ready.') return # pylint: disable=broad-except - except BaseException as ex: + except Exception as ex: self.logger.info('Cannot connect target: %s', ex) time.sleep(1) @@ -123,25 +126,41 @@ def wait_boot_complete(self): def terminate(self): """ - Terminate :attr:`TargetRunner.runner_process`. + Terminate ``TargetRunner.runner_process``. """ if self.runner_process is None: return - try: - self.runner_process.stdin.close() - self.runner_process.stdout.close() - self.runner_process.stderr.close() + self.logger.debug('Killing target runner...') + self.runner_process.kill() + + +class NOPTargetRunner(TargetRunner): + """ + Class for implementing a target runner which does nothing except providing .target attribute. - if self.runner_process.poll() is None: - self.logger.debug('Terminating target runner...') - os.killpg(self.runner_process.pid, signal.SIGTERM) - # Wait 3 seconds before killing the runner. - self.runner_process.wait(timeout=3) - except subprocess.TimeoutExpired: - self.logger.info('Killing target runner...') - os.killpg(self.runner_process.pid, signal.SIGKILL) + :class:`NOPTargetRunner` is a subclass of :class:`TargetRunner` which is implemented only for + testing purposes. + + :param target: Specifies type of target per :class:`Target` based classes. + :type target: Target + """ + + def __init__(self, target): + super().__init__(target=target) + + def __enter__(self): + return self + + def __exit__(self, *_): + pass + + def wait_boot_complete(self): + pass + + def terminate(self): + pass class QEMUTargetRunner(TargetRunner): @@ -181,11 +200,10 @@ class QEMUTargetRunner(TargetRunner): :type qemu_settings: Dict :param connection_settings: the dictionary to store connection settings - of :attr:`Target.connection_settings`, defaults to None. + of ``Target.connection_settings``, defaults to None. :type connection_settings: Dict, optional - :param make_target: Lambda function for creating :class:`Target` based - object, defaults to :func:`lambda **kwargs: LinuxTarget(**kwargs)`. + :param make_target: Lambda function for creating :class:`Target` based object. :type make_target: func, optional :Variable positional arguments: Forwarded to :class:`TargetRunner`. @@ -196,8 +214,7 @@ class QEMUTargetRunner(TargetRunner): def __init__(self, qemu_settings, connection_settings=None, - # pylint: disable=unnecessary-lambda - make_target=lambda **kwargs: LinuxTarget(**kwargs), + make_target=LinuxTarget, **args): self.connection_settings = { 'host': '127.0.0.1', @@ -207,14 +224,11 @@ def __init__(self, 'strict_host_check': False, } - if connection_settings is not None: - self.connection_settings = self.connection_settings | connection_settings + self.connection_settings = {**self.connection_settings, **(connection_settings or {})} qemu_args = { - 'kernel_image': '', 'arch': 'aarch64', 'cpu_type': 'cortex-a72', - 'initrd_image': '', 'mem_size': 512, 'num_cores': 2, 'num_threads': 2, @@ -222,46 +236,41 @@ def __init__(self, 'enable_kvm': True, } - qemu_args = qemu_args | qemu_settings + qemu_args = {**qemu_args, **qemu_settings} qemu_executable = f'qemu-system-{qemu_args["arch"]}' qemu_path = which(qemu_executable) if qemu_path is None: raise FileNotFoundError(f'Cannot find {qemu_executable} executable!') - if not os.path.exists(qemu_args["kernel_image"]): - raise FileNotFoundError(f'{qemu_args["kernel_image"]} does not exist!') - - # pylint: disable=consider-using-f-string - qemu_cmd = '''\ -{} -kernel {} -append "{}" -m {} -smp cores={},threads={} -netdev user,id=net0,hostfwd=tcp::{}-:22 \ --device virtio-net-pci,netdev=net0 --nographic\ -'''.format( - qemu_path, - qemu_args["kernel_image"], - qemu_args["cmdline"], - qemu_args["mem_size"], - qemu_args["num_cores"], - qemu_args["num_threads"], - self.connection_settings["port"], - ) - - if qemu_args["initrd_image"]: + if qemu_args.get("kernel_image"): + if not os.path.exists(qemu_args["kernel_image"]): + raise FileNotFoundError(f'{qemu_args["kernel_image"]} does not exist!') + else: + raise KeyError('qemu_settings must have kernel_image!') + + qemu_cmd = f'''{qemu_path} -kernel {qemu_args["kernel_image"]} \ +-append "{qemu_args["cmdline"]}" -m {qemu_args["mem_size"]} -smp \ +cores={qemu_args["num_cores"]},threads={qemu_args["num_threads"]} \ +-netdev user,id=net0,hostfwd=tcp::{self.connection_settings["port"]}-:22 \ +-device virtio-net-pci,netdev=net0 --nographic''' + + if qemu_args.get("initrd_image"): if not os.path.exists(qemu_args["initrd_image"]): raise FileNotFoundError(f'{qemu_args["initrd_image"]} does not exist!') qemu_cmd += f' -initrd {qemu_args["initrd_image"]}' - if qemu_args["arch"] == machine(): + if qemu_args['arch'].startswith('x86') and machine().startswith('x86'): if qemu_args["enable_kvm"]: qemu_cmd += ' --enable-kvm' else: qemu_cmd += f' -machine virt -cpu {qemu_args["cpu_type"]}' - self.target = make_target(connect=False, - conn_cls=SshConnection, - connection_settings=self.connection_settings) + target = make_target(connect=False, + conn_cls=SshConnection, + connection_settings=self.connection_settings) - super().__init__(runner_cmd=qemu_cmd, - target=self.target, + super().__init__(target=target, + runner_cmd=qemu_cmd, **args) diff --git a/devlib/target.py b/devlib/target.py index 0f957201e..d171b1a95 100644 --- a/devlib/target.py +++ b/devlib/target.py @@ -1081,10 +1081,10 @@ def make_temp(self, is_directory=True, directory='', prefix='devlib-test'): :type is_directory: bool, optional :param directory: Temp object will be created under this directory, - defaults to :attr:`Target.working_directory`. + defaults to ``Target.working_directory``. :type directory: str, optional - :param prefix: Prefix of temp object's name, defaults to 'devlib-test'. + :param prefix: Prefix of temp object's name. :type prefix: str, optional :yield: Full path of temp object. @@ -1094,7 +1094,7 @@ def make_temp(self, is_directory=True, directory='', prefix='devlib-test'): directory = directory or self.working_directory temp_obj = None try: - cmd = f'mktemp -p {directory} {prefix}-XXXXXX' + cmd = f'mktemp -p {quote(directory)} {quote(prefix)}-XXXXXX' if is_directory: cmd += ' -d' diff --git a/tests/target_configs.yaml.example b/tests/target_configs.yaml.example index 1154ea8b9..497635693 100644 --- a/tests/target_configs.yaml.example +++ b/tests/target_configs.yaml.example @@ -31,7 +31,7 @@ QEMUTargetRunner: entry-1: connection_settings: - port : 8023 + port: 8023 qemu_settings: kernel_image: '/path/to/devlib/tools/buildroot/buildroot-v2023.11.1-x86_64/output/images/bzImage' diff --git a/tests/test_target.py b/tests/test_target.py index 2d59a2374..6874c4a78 100644 --- a/tests/test_target.py +++ b/tests/test_target.py @@ -14,19 +14,30 @@ # limitations under the License. # -"""Module for testing targets.""" +""" +Module for testing targets. +Sample run with log level is set to DEBUG (see +https://docs.pytest.org/en/7.1.x/how-to/logging.html#live-logs for logging details): + +$ python -m pytest --log-cli-level DEBUG test_target.py +""" + +import logging import os -from pprint import pp import pytest -from devlib import AndroidTarget, ChromeOsTarget, LinuxTarget, LocalLinuxTarget, QEMUTargetRunner +from devlib import AndroidTarget, ChromeOsTarget, LinuxTarget, LocalLinuxTarget +from devlib._target_runner import NOPTargetRunner, QEMUTargetRunner from devlib.utils.android import AdbConnection from devlib.utils.misc import load_struct_from_yaml -def build_targets(): - """Read targets from a YAML formatted config file""" +logger = logging.getLogger('test_target') + + +def build_target_runners(): + """Read targets from a YAML formatted config file and create runners for them""" config_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'target_configs.yaml') @@ -34,88 +45,79 @@ def build_targets(): if target_configs is None: raise ValueError(f'{config_file} looks empty!') - targets = [] + target_runners = [] if target_configs.get('AndroidTarget') is not None: - print('> Android targets:') + logger.info('> Android targets:') for entry in target_configs['AndroidTarget'].values(): - pp(entry) + logger.info('%s', repr(entry)) a_target = AndroidTarget( connect=False, connection_settings=entry['connection_settings'], conn_cls=lambda **kwargs: AdbConnection(adb_as_root=True, **kwargs), ) a_target.connect(timeout=entry.get('timeout', 60)) - targets.append((a_target, None)) + target_runners.append(NOPTargetRunner(a_target)) if target_configs.get('LinuxTarget') is not None: - print('> Linux targets:') + logger.info('> Linux targets:') for entry in target_configs['LinuxTarget'].values(): - pp(entry) + logger.info('%s', repr(entry)) l_target = LinuxTarget(connection_settings=entry['connection_settings']) - targets.append((l_target, None)) + target_runners.append(NOPTargetRunner(l_target)) if target_configs.get('ChromeOsTarget') is not None: - print('> ChromeOS targets:') + logger.info('> ChromeOS targets:') for entry in target_configs['ChromeOsTarget'].values(): - pp(entry) + logger.info('%s', repr(entry)) c_target = ChromeOsTarget( connection_settings=entry['connection_settings'], working_directory='/tmp/devlib-target', ) - targets.append((c_target, None)) + target_runners.append(NOPTargetRunner(c_target)) if target_configs.get('LocalLinuxTarget') is not None: - print('> LocalLinux targets:') + logger.info('> LocalLinux targets:') for entry in target_configs['LocalLinuxTarget'].values(): - pp(entry) + logger.info('%s', repr(entry)) ll_target = LocalLinuxTarget(connection_settings=entry['connection_settings']) - targets.append((ll_target, None)) + target_runners.append(NOPTargetRunner(ll_target)) if target_configs.get('QEMUTargetRunner') is not None: - print('> QEMU target runners:') + logger.info('> QEMU target runners:') for entry in target_configs['QEMUTargetRunner'].values(): - pp(entry) - qemu_settings = entry.get('qemu_settings') and entry['qemu_settings'] - connection_settings = entry.get( - 'connection_settings') and entry['connection_settings'] + logger.info('%s', repr(entry)) qemu_runner = QEMUTargetRunner( - qemu_settings=qemu_settings, - connection_settings=connection_settings, + qemu_settings=entry.get('qemu_settings'), + connection_settings=entry.get('connection_settings'), ) - if entry.get('ChromeOsTarget') is None: - targets.append((qemu_runner.target, qemu_runner)) - continue + if entry.get('ChromeOsTarget') is not None: + # Leave termination of QEMU runner to ChromeOS target. + target_runners.append(NOPTargetRunner(qemu_runner.target)) - # Leave termination of QEMU runner to ChromeOS target. - targets.append((qemu_runner.target, None)) + logger.info('>> ChromeOS target: %s', repr(entry["ChromeOsTarget"])) + qemu_runner.target = ChromeOsTarget( + connection_settings={ + **entry['ChromeOsTarget']['connection_settings'], + **qemu_runner.target.connection_settings, + }, + working_directory='/tmp/devlib-target', + ) - print('> ChromeOS targets:') - pp(entry['ChromeOsTarget']) - c_target = ChromeOsTarget( - connection_settings={ - **entry['ChromeOsTarget']['connection_settings'], - **qemu_runner.target.connection_settings, - }, - working_directory='/tmp/devlib-target', - ) - targets.append((c_target, qemu_runner)) + target_runners.append(qemu_runner) - return targets + return target_runners -@pytest.mark.parametrize("target, target_runner", build_targets()) -def test_read_multiline_values(target, target_runner): +@pytest.mark.parametrize("target_runner", build_target_runners()) +def test_read_multiline_values(target_runner): """ Test Target.read_tree_values_flat() - :param target: Type of target per :class:`Target` based classes. - :type target: Target - - :param target_runner: Target runner object to terminate target (if necessary). - :type target: TargetRunner + :param target_runner: TargetRunner object to test. + :type target_runner: TargetRunner """ data = { @@ -124,26 +126,27 @@ def test_read_multiline_values(target, target_runner): 'test3': '3\n\n4\n\n', } - print(f'target={target.__class__.__name__} os={target.os} hostname={target.hostname}') + target = target_runner.target + + logger.info('target=%s os=%s hostname=%s', + target.__class__.__name__, target.os, target.hostname) with target.make_temp() as tempdir: - print(f'Created {tempdir}.') + logger.debug('Created %s.', tempdir) for key, value in data.items(): path = os.path.join(tempdir, key) - print(f'Writing {value!r} to {path}...') + logger.debug('Writing %s to %s...', repr(value), path) target.write_value(path, value, verify=False, as_root=target.conn.connected_as_root) - print('Reading values from target...') + logger.debug('Reading values from target...') raw_result = target.read_tree_values_flat(tempdir) result = {os.path.basename(k): v for k, v in raw_result.items()} - print(f'Removing {target.working_directory}...') + logger.debug('Removing %s...', target.working_directory) target.remove(target.working_directory) - if target_runner is not None: - print('Terminating target runner...') - target_runner.terminate() + target_runner.terminate() assert {k: v.strip() for k, v in data.items()} == result diff --git a/tools/docker/run_tests.sh b/tools/docker/run_tests.sh index 226639b54..f49dd4e62 100755 --- a/tools/docker/run_tests.sh +++ b/tools/docker/run_tests.sh @@ -38,4 +38,5 @@ sleep 30 cd /devlib cp -f tools/docker/target_configs.yaml tests/ -python3 -m pytest -v -s ./tests/test_target.py +python3 -m pytest --log-cli-level DEBUG ./tests/test_target.py +