Integrate buildroot and add support for QEMU Linux target

devlib/__init__.py

@@ -22,6 +22,8 @@
         ChromeOsTarget,
 )
 
+from devlib.target_runner import QEMUTargetRunner
+
 from devlib.host import (
         PACKAGE_BIN_DIRECTORY,
         LocalConnection,

devlib/target.py

@@ -14,6 +14,7 @@
 #
 
 import asyncio
+from contextlib import contextmanager
 import io
 import base64
 import functools
@@ -493,7 +494,7 @@ async def setup(self, executables=None):
         # Check for platform dependent setup procedures
         self.platform.setup(self)
 
-        # Initialize modules which requires Buxybox (e.g. shutil dependent tasks)
+        # Initialize modules which requires Busybox (e.g. shutil dependent tasks)
         self._update_modules('setup')
 
         await self.execute.asyn('mkdir -p {}'.format(quote(self._file_transfer_cache)))
@@ -1071,6 +1072,38 @@ async def write_value(self, path, value, verify=True, as_root=True):
             else:
                 raise
 
+    @contextmanager
+    def make_temp(self, is_directory=True, directory='', prefix='devlib-test'):
+        """
+        Creates temporary file/folder on target and deletes it once it's done.
+
+        :param is_directory: Specifies if temporary object is a directory, defaults to True.
+        :type is_directory: bool, optional
+
+        :param directory: Temp object will be created under this directory,
+            defaults to :attr:`Target.working_directory`.
+        :type directory: str, optional
+
+        :param prefix: Prefix of temp object's name, defaults to 'devlib-test'.
+        :type prefix: str, optional
+
+        :yield: Full path of temp object.
+        :rtype: str
+        """
+
+        directory = directory or self.working_directory
+        temp_obj = None
+        try:
+            cmd = f'mktemp -p {directory} {prefix}-XXXXXX'
+            if is_directory:
+                cmd += ' -d'
+
+            temp_obj = self.execute(cmd).strip()
+            yield temp_obj
+        finally:
+            if temp_obj is not None:
+                self.remove(temp_obj)
+
     def reset(self):
         try:
             self.execute('reboot', as_root=self.needs_su, timeout=2)

devlib/target_runner.py

@@ -0,0 +1,267 @@
+#    Copyright 2024 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+Target runner and related classes are implemented here.
+"""
+
+import logging
+import os
+import signal
+import subprocess
+import time
+from platform import machine
+
+from devlib.exception import (TargetStableError, HostError)
+from devlib.target import LinuxTarget
+from devlib.utils.misc import get_subprocess, which
+from devlib.utils.ssh import SshConnection
+
+
+class TargetRunner:
+    """
+    A generic class for interacting with targets runners.
+
+    It mainly aims to provide framework support for QEMU like target runners
+    (e.g., :class:`QEMUTargetRunner`).
+
+    :param runner_cmd: The command to start runner process (e.g.,
+        ``qemu-system-aarch64 -kernel Image -append "console=ttyAMA0" ...``).
+    :type runner_cmd: str
+
+    :param target: Specifies type of target per :class:`Target` based classes.
+    :type target: Target
+
+    :param connect: Specifies if :class:`TargetRunner` should try to connect
+        target after launching it, defaults to True.
+    :type connect: bool, optional
+
+    :param boot_timeout: Timeout for target's being ready for SSH access in
+        seconds, defaults to 60.
+    :type boot_timeout: int, optional
+
+    :raises HostError: if it cannot execute runner command successfully.
+
+    :raises TargetStableError: if Target is inaccessible.
+    """
+
+    def __init__(self,
+                 runner_cmd,
+                 target,
+                 connect=True,
+                 boot_timeout=60):
+        self.boot_timeout = boot_timeout
+        self.target = target
+
+        self.logger = logging.getLogger(self.__class__.__name__)
+
+        self.logger.info('runner_cmd: %s', runner_cmd)
+
+        try:
+            self.runner_process = get_subprocess(list(runner_cmd.split()))
+        except Exception as ex:
+            raise HostError(f'Error while running "{runner_cmd}": {ex}') from ex
+
+        if connect:
+            self.wait_boot_complete()
+
+    def __enter__(self):
+        """
+        Complementary method for contextmanager.
+
+        :return: Self object.
+        :rtype: TargetRunner
+        """
+
+        return self
+
+    def __exit__(self, *_):
+        """
+        Exit routine for contextmanager.
+
+        Ensure :attr:`TargetRunner.runner_process` is terminated on exit.
+        """
+
+        self.terminate()
+
+    def wait_boot_complete(self):
+        """
+        Wait for target OS to finish boot up and become accessible over SSH in at most
+        :attr:`TargetRunner.boot_timeout` seconds.
+
+        :raises TargetStableError: In case of timeout.
+        """
+
+        start_time = time.time()
+        elapsed = 0
+        while self.boot_timeout >= elapsed:
+            try:
+                self.target.connect(timeout=self.boot_timeout - elapsed)
+                self.logger.info('Target is ready.')
+                return
+            # pylint: disable=broad-except
+            except BaseException as ex:
+                self.logger.info('Cannot connect target: %s', ex)
+
+            time.sleep(1)
+            elapsed = time.time() - start_time
+
+        self.terminate()
+        raise TargetStableError(f'Target is inaccessible for {self.boot_timeout} seconds!')
+
+    def terminate(self):
+        """
+        Terminate :attr:`TargetRunner.runner_process`.
+        """
+
+        if self.runner_process is None:
+            return
+
+        try:
+            self.runner_process.stdin.close()
+            self.runner_process.stdout.close()
+            self.runner_process.stderr.close()
+
+            if self.runner_process.poll() is None:
+                self.logger.debug('Terminating target runner...')
+                os.killpg(self.runner_process.pid, signal.SIGTERM)
+                # Wait 3 seconds before killing the runner.
+                self.runner_process.wait(timeout=3)
+        except subprocess.TimeoutExpired:
+            self.logger.info('Killing target runner...')
+            os.killpg(self.runner_process.pid, signal.SIGKILL)
+
+
+class QEMUTargetRunner(TargetRunner):
+    """
+    Class for interacting with QEMU runners.
+
+    :class:`QEMUTargetRunner` is a subclass of :class:`TargetRunner` which performs necessary
+    groundwork for launching a guest OS on QEMU.
+
+    :param qemu_settings: A dictionary which has QEMU related parameters. The full list
+        of QEMU parameters is below:
+        * ``kernel_image``: This is the location of kernel image (e.g., ``Image``) which
+            will be used as target's kernel.
+
+        * ``arch``: Architecture type. Defaults to ``aarch64``.
+
+        * ``cpu_types``: List of CPU ids for QEMU. The list only contains ``cortex-a72`` by
+            default. This parameter is valid for Arm architectures only.
+
+        * ``initrd_image``: This points to the location of initrd image (e.g.,
+            ``rootfs.cpio.xz``) which will be used as target's root filesystem if kernel
+            does not include one already.
+
+        * ``mem_size``: Size of guest memory in MiB.
+
+        * ``num_cores``: Number of CPU cores. Guest will have ``2`` cores by default.
+
+        * ``num_threads``: Number of CPU threads. Set to ``2`` by defaults.
+
+        * ``cmdline``: Kernel command line parameter. It only specifies console device in
+            default (i.e., ``console=ttyAMA0``) which is valid for Arm architectures.
+            May be changed to ``ttyS0`` for x86 platforms.
+
+        * ``enable_kvm``: Specifies if KVM will be used as accelerator in QEMU or not.
+            Enabled by default if host architecture matches with target's for improving
+            QEMU performance.
+    :type qemu_settings: Dict
+
+    :param connection_settings: the dictionary to store connection settings
+        of :attr:`Target.connection_settings`, defaults to None.
+    :type connection_settings: Dict, optional
+
+    :param make_target: Lambda function for creating :class:`Target` based
+        object, defaults to :func:`lambda **kwargs: LinuxTarget(**kwargs)`.
+    :type make_target: func, optional
+
+    :Variable positional arguments: Forwarded to :class:`TargetRunner`.
+
+    :raises FileNotFoundError: if QEMU executable, kernel or initrd image cannot be found.
+    """
+
+    def __init__(self,
+                 qemu_settings,
+                 connection_settings=None,
+                 # pylint: disable=unnecessary-lambda
+                 make_target=lambda **kwargs: LinuxTarget(**kwargs),
+                 **args):
+        self.connection_settings = {
+            'host': '127.0.0.1',
+            'port': 8022,
+            'username': 'root',
+            'password': 'root',
+            'strict_host_check': False,
+        }
+
+        if connection_settings is not None:
+            self.connection_settings = self.connection_settings | connection_settings
+
+        qemu_args = {
+            'kernel_image': '',
+            'arch': 'aarch64',
+            'cpu_type': 'cortex-a72',
+            'initrd_image': '',
+            'mem_size': 512,
+            'num_cores': 2,
+            'num_threads': 2,
+            'cmdline': 'console=ttyAMA0',
+            'enable_kvm': True,
+        }
+
+        qemu_args = qemu_args | qemu_settings
+
+        qemu_executable = f'qemu-system-{qemu_args["arch"]}'
+        qemu_path = which(qemu_executable)
+        if qemu_path is None:
+            raise FileNotFoundError(f'Cannot find {qemu_executable} executable!')
+
+        if not os.path.exists(qemu_args["kernel_image"]):
+            raise FileNotFoundError(f'{qemu_args["kernel_image"]} does not exist!')
+
+        # pylint: disable=consider-using-f-string
+        qemu_cmd = '''\
+{} -kernel {} -append "{}" -m {} -smp cores={},threads={} -netdev user,id=net0,hostfwd=tcp::{}-:22 \
+-device virtio-net-pci,netdev=net0 --nographic\
+'''.format(
+            qemu_path,
+            qemu_args["kernel_image"],
+            qemu_args["cmdline"],
+            qemu_args["mem_size"],
+            qemu_args["num_cores"],
+            qemu_args["num_threads"],
+            self.connection_settings["port"],
+        )
+
+        if qemu_args["initrd_image"]:
+            if not os.path.exists(qemu_args["initrd_image"]):
+                raise FileNotFoundError(f'{qemu_args["initrd_image"]} does not exist!')
+
+            qemu_cmd += f' -initrd {qemu_args["initrd_image"]}'
+
+        if qemu_args["arch"] == machine():
+            if qemu_args["enable_kvm"]:
+                qemu_cmd += ' --enable-kvm'
+        else:
+            qemu_cmd += f' -machine virt -cpu {qemu_args["cpu_type"]}'
+
+        self.target = make_target(connect=False,
+                                  conn_cls=SshConnection,
+                                  connection_settings=self.connection_settings)
+
+        super().__init__(runner_cmd=qemu_cmd,
+                         target=self.target,
+                         **args)

devlib/utils/ssh.py

@@ -367,25 +367,32 @@ def __init__(self,
         else:
             logger.debug('Using SFTP for file transfer')
 
-        self.client = self._make_client()
-        atexit.register(self.close)
-
-        # Use a marker in the output so that we will be able to differentiate
-        # target connection issues with "password needed".
-        # Also, sudo might not be installed at all on the target (but
-        # everything will work as long as we login as root). If sudo is still
-        # needed, it will explode when someone tries to use it. After all, the
-        # user might not be interested in being root at all.
-        self._sudo_needs_password = (
-            'NEED_PASSWORD' in
-            self.execute(
-                # sudo -n is broken on some versions on MacOSX, revisit that if
-                # someone ever cares
-                'sudo -n true || echo NEED_PASSWORD',
-                as_root=False,
-                check_exit_code=False,
+        self.client = None
+        try:
+            self.client = self._make_client()
+            atexit.register(self.close)
+
+            # Use a marker in the output so that we will be able to differentiate
+            # target connection issues with "password needed".
+            # Also, sudo might not be installed at all on the target (but
+            # everything will work as long as we login as root). If sudo is still
+            # needed, it will explode when someone tries to use it. After all, the
+            # user might not be interested in being root at all.
+            self._sudo_needs_password = (
+                'NEED_PASSWORD' in
+                self.execute(
+                    # sudo -n is broken on some versions on MacOSX, revisit that if
+                    # someone ever cares
+                    'sudo -n true || echo NEED_PASSWORD',
+                    as_root=False,
+                    check_exit_code=False,
+                )
             )
-        )
+
+        except BaseException:
+            if self.client is not None:
+                self.client.close()
+            raise
 
     def _make_client(self):
         if self.strict_host_check:

doc/index.rst

@@ -25,6 +25,7 @@ Contents:
    derived_measurements
    platform
    connection
+   tools
 
 Indices and tables
 ==================