Update docs/laptop

docs/laptop/README.md

@@ -52,7 +52,7 @@ docker network create paddles
 Start postgres containers in order to use paddles:
 
 ```bash
-mkdir $HOME/.teuthology/postgres
+mkdir -p $HOME/.teuthology/postgres
 docker run -d -p 5432:5432 --network paddles --name paddles-postgres \
     -e POSTGRES_PASSWORD=secret \
     -e POSTGRES_USER=paddles \
@@ -61,6 +61,10 @@ docker run -d -p 5432:5432 --network paddles --name paddles-postgres \
     -v $HOME/.teuthology/postgres:/var/lib/postgresql/data postgres
 ```
 
+NOTE. When running container on MacOS X using podman postgres may experience
+troubles with volume directory binds because of podman machine,  thus use regular
+volumes like `-v paddlesdb:/var/lib/postrgresql/data`.
+
 ### Run paddles
 
 Checkout paddles and build the image:
@@ -72,7 +76,7 @@ cd ~/paddles && docker build . --file Dockerfile --tag paddles
 Run the container with previously created network:
 
 ```bash
-docker run -d --network paddles --name api -p 80:8080 \
+docker run -d --network paddles --name api -p 8080:8080 \
 	-e PADDLES_SERVER_HOST=0.0.0.0 \
 	-e PADDLES_SQLALCHEMY_URL=postgresql+psycopg2://paddles:secret@paddles-postgres/paddles \
 	paddles

docs/siteconfig.rst

@@ -251,6 +251,17 @@ Here is a sample configuration with many of the options set and documented::
       endpoint: http://head.ses.suse.de:5000/
       machine_types: ['type1', 'type2', 'type3']
 
+    # Define a list of ssh tunnels for a various group of test nodes.
+    # Notice: provided domain names for the nodes must be resolvable
+    # in your network and jump host (bastion) must be accessible.
+    tunnel:
+      - hosts: ['example1.domain', 'example2.domain', 'example3.domain']
+        bastion:
+          host: ssh_host_name         # must be resolvable and reachable
+          user: ssh_user_name         # (optional)
+          port: ssh_port              # (optional)
+          identity: ~/.ssh/id_ed25519 # (optional)
+
     # Do not allow more than that many jobs in a single run by default.
     # To disable this check use 0.
     job_threshold: 500

setup.cfg

@@ -56,6 +56,7 @@ install_requires =
     python-dateutil
     requests>2.13.0
     sentry-sdk
+    sshtunnel
     types-psutil
     urllib3>=1.25.4,<1.27  # For botocore
 scripts =

teuthology/misc.py

@@ -1112,6 +1112,11 @@ def _ssh_keyscan(hostname):
     :returns: The host key
     """
     args = ['ssh-keyscan', '-T', '1', hostname]
+    if config.tunnel:
+        for tunnel in config.tunnel:
+            if hostname in tunnel.get('hosts'):
+                bastion = tunnel.get('bastion')
+                args = ['ssh', bastion.get('host')] + args
     p = subprocess.Popen(
         args=args,
         stdout=subprocess.PIPE,
@@ -1127,7 +1132,8 @@ def _ssh_keyscan(hostname):
     for line in p.stdout:
         host, key = line.strip().decode().split(' ', 1)
         keys.append(key)
-    return sorted(keys)[0]
+    if len(keys) > 0:
+        return sorted(keys)[0]
 
 
 def ssh_keyscan_wait(hostname):

teuthology/orchestra/connection.py

@@ -8,6 +8,7 @@
 from teuthology.config import config
 from teuthology.contextutil import safe_while
 from paramiko.hostkeys import HostKeyEntry
+import sshtunnel
 
 log = logging.getLogger(__name__)
 
@@ -50,7 +51,20 @@ def connect(user_at_host, host_key=None, keep_alive=False, timeout=60,
     :param retry:       Whether or not to retry failed connection attempts
                         (eventually giving up if none succeed). Default is True
     :param key_filename:  Optionally override which private key to use.
+
     :return: ssh connection.
+
+    The connection is going to be established via tunnel if corresponding options
+    are provided in teuthology configuration file. For example:
+
+    tunnel:
+      - hosts: ['hostname1.domain', 'hostname2.domain']
+        bastion:
+          host: ssh_host_name
+          user: ssh_user_name
+          port: 22
+          identity: ~/.ssh/id_ed25519
+
     """
     user, host = split_user(user_at_host)
     if _SSHClient is None:
@@ -79,6 +93,28 @@ def connect(user_at_host, host_key=None, keep_alive=False, timeout=60,
         timeout=timeout
     )
 
+    if config.tunnel:
+        for tunnel in config.tunnel:
+            if host in tunnel.get('hosts'):
+                bastion = tunnel.get('bastion')
+                if not bastion:
+                    log.error("The 'tunnel' config must include 'bastion' entry")
+                    continue
+                bastion_host = bastion.get('host')
+                server = sshtunnel.SSHTunnelForwarder(
+                    bastion_host,
+                    ssh_username=bastion.get('user', None),
+                    ssh_password=bastion.get('word', None),
+                    ssh_pkey=bastion.get('identity'),
+                    remote_bind_address=(host, 22))
+                log.info(f'Starting tunnel to {bastion_host} for host {host}')
+                server.start()
+                local_port = server.local_bind_port
+                log.debug(f"Local port for host {host} is {local_port}")
+                connect_args['hostname'] = '127.0.0.1'
+                connect_args['port'] = local_port
+                break
+
     key_filename = key_filename or config.ssh_key
     ssh_config_path = config.ssh_config_path or "~/.ssh/config"
     ssh_config_path = os.path.expanduser(ssh_config_path)

teuthology/orchestra/run.py

@@ -9,7 +9,7 @@
 import gevent
 import gevent.event
 import socket
-import pipes
+import shlex
 import logging
 import shutil
 
@@ -252,7 +252,7 @@ def _quote(args):
             if isinstance(a, Raw):
                 yield a.value
             else:
-                yield pipes.quote(a)
+                yield shlex.quote(a)
     if isinstance(args, list):
         return ' '.join(_quote(args))
     else:
@@ -400,7 +400,7 @@ def run(
     """
     Run a command remotely.  If any of 'args' contains shell metacharacters
     that you want to pass unquoted, pass it as an instance of Raw(); otherwise
-    it will be quoted with pipes.quote() (single quote, and single quotes
+    it will be quoted with shlex.quote() (single quote, and single quotes
     enclosed in double quotes).
 
     :param client: SSHConnection to run the command with

teuthology/provision/downburst.py

@@ -44,6 +44,7 @@ def downburst_executable():
 
 def downburst_environment():
     env = dict()
+    env['PATH'] = os.environ.get('PATH')
     discover_url = os.environ.get('DOWNBURST_DISCOVER_URL')
     if config.downburst and not discover_url:
         if isinstance(config.downburst, dict):
@@ -215,7 +216,7 @@ def build_config(self):
             'additional-disks-size': machine['volumes']['size'],
             'arch': 'x86_64',
         }
-        fqdn = self.name.split('@')[1]
+        fqdn = self.name.split('@')[-1]
         file_out = {
             'downburst': file_info,
             'local-hostname': fqdn,