Add initial Windows support (#1675)

* Only client (not server)
* Only MSYS2 OpenSSH client is supported (expected to be installed with
  Git for Windows)

Issue: #1644

.github/workflows/build.yml

@@ -60,7 +60,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ macos-latest, ubuntu-latest]
+        os: [ macos-latest, ubuntu-latest, windows-latest ]
         python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
     steps:
       - uses: actions/checkout@v4
@@ -75,7 +75,8 @@ jobs:
         with:
           name: frontend-build
           path: src/dstack/_internal/server/statics
-      - name: Run pytest
+      - name: Run pytest on POSIX
+        if: matrix.os != 'windows-latest'
         # Skip Postgres tests on macos since macos runner doesn't have Docker.
         # Skip Postgres tests for Python 3.8 since testcontainers<4 doesn't support asyncpg correctly.
         run: |
@@ -84,6 +85,10 @@ jobs:
             RUNPOSTGRES="--runpostgres"
           fi
           pytest src/tests --runui $RUNPOSTGRES
+      - name: Run pytest on Windows
+        if: matrix.os == 'windows-latest'
+        run: |
+          pytest src/tests --runui --runpostgres
 
   update-get-dstack:
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository

.github/workflows/release.yml

@@ -51,7 +51,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ macos-latest, ubuntu-latest]
+        os: [ macos-latest, ubuntu-latest, windows-latest ]
         python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
     steps:
       - uses: actions/checkout@v4
@@ -66,7 +66,8 @@ jobs:
         with:
           name: frontend-build
           path: src/dstack/_internal/server/statics
-      - name: Run pytest
+      - name: Run pytest on POSIX
+        if: matrix.os != 'windows-latest'
         # Skip Postgres tests on macos since macos runner doesn't have Docker.
         # Skip Postgres tests for Python 3.8 since testcontainers<4 doesn't support asyncpg correctly.
         run: |
@@ -75,6 +76,10 @@ jobs:
             RUNPOSTGRES="--runpostgres"
           fi
           pytest src/tests --runui $RUNPOSTGRES
+      - name: Run pytest on Windows
+        if: matrix.os == 'windows-latest'
+        run: |
+          pytest src/tests --runui --runpostgres
 
   runner-test:
     defaults:

docs/docs/installation/index.md

@@ -9,6 +9,18 @@ To use the open-source version of `dstack` with your own cloud accounts or on-pr
 > If you don't want to host the `dstack` server (or want to access GPU marketplace),
 > skip installation and proceed to [dstack Sky :material-arrow-top-right-thin:{ .external }](https://sky.dstack.ai){:target="_blank"}.
 
+## Prerequisites
+
+`dstack` works on Linux, macOS, and Windows, with one exception — the `dstack server` functionality is not currently supported on Windows.
+
+`dstack` requires Git and OpenSSH client to operate.
+
+On Windows, install [Git for Windows](https://git-scm.com/download/win), it contains both Git and OpenSSH. During the installation,
+make sure the following options are checked:
+
+- _“Git from the command line and also from 3-rd party software”_ or _“Use Git and optional Unix tools from the Command Prompt”_
+- _“Use bundled OpenSSH”_
+
 ## Configure backends
 
 To use `dstack` with your own cloud accounts, or Kubernetes,

src/dstack/_internal/cli/main.py

@@ -18,7 +18,8 @@
 from dstack._internal.cli.commands.volume import VolumeCommand
 from dstack._internal.cli.utils.common import _colors, console
 from dstack._internal.cli.utils.updates import check_for_updates
-from dstack._internal.core.errors import ClientError, CLIError, ConfigurationError
+from dstack._internal.core.errors import ClientError, CLIError, ConfigurationError, SSHError
+from dstack._internal.core.services.ssh.client import get_ssh_client_info
 from dstack._internal.utils.logging import get_logger
 from dstack.version import __version__ as version
 
@@ -72,8 +73,9 @@ def main():
     args.unknown = unknown_args
     try:
         check_for_updates()
+        get_ssh_client_info()
         args.func(args)
-    except (ClientError, CLIError, ConfigurationError) as e:
+    except (ClientError, CLIError, ConfigurationError, SSHError) as e:
         console.print(f"[error]{escape(str(e))}[/]")
         logger.debug(e, exc_info=True)
         exit(1)

src/dstack/_internal/compat.py

@@ -0,0 +1,3 @@
+import os
+
+IS_WINDOWS = os.name == "nt"

src/dstack/_internal/core/services/ssh/attach.py

@@ -2,40 +2,64 @@
 import re
 import subprocess
 import time
-from typing import Optional, Tuple
+from pathlib import Path
+from typing import Optional
 
+from dstack._internal.compat import IS_WINDOWS
 from dstack._internal.core.errors import SSHError
 from dstack._internal.core.models.instances import SSHConnectionParams
 from dstack._internal.core.services.configs import ConfigManager
+from dstack._internal.core.services.ssh.client import get_ssh_client_info
 from dstack._internal.core.services.ssh.ports import PortsLock
-from dstack._internal.core.services.ssh.tunnel import (
-    FilePath,
-    SSHTunnel,
-    ports_to_forwarded_sockets,
+from dstack._internal.core.services.ssh.tunnel import SSHTunnel, ports_to_forwarded_sockets
+from dstack._internal.utils.path import FilePath, PathLike
+from dstack._internal.utils.ssh import (
+    include_ssh_config,
+    normalize_path,
+    update_ssh_config,
 )
-from dstack._internal.utils.path import PathLike
-from dstack._internal.utils.ssh import get_ssh_config, include_ssh_config, update_ssh_config
 
 
 class SSHAttach:
-    @staticmethod
-    def reuse_control_sock_path_and_port_locks(run_name: str) -> Optional[Tuple[str, PortsLock]]:
-        ssh_config_path = str(ConfigManager().dstack_ssh_config_path)
-        host_config = get_ssh_config(ssh_config_path, run_name)
-        if host_config and host_config.get("ControlPath"):
+    @classmethod
+    def get_control_sock_path(cls, run_name: str) -> Path:
+        return ConfigManager().dstack_ssh_dir / f"{run_name}.control.sock"
+
+    @classmethod
+    def reuse_ports_lock(cls, run_name: str) -> Optional[PortsLock]:
+        if not get_ssh_client_info().supports_control_socket:
+            raise SSHError("Unsupported SSH client")
+        control_sock_path = normalize_path(cls.get_control_sock_path(run_name))
+        filter_prefix: str
+        output: bytes
+        if IS_WINDOWS:
+            filter_prefix = "powershell"
+            output = subprocess.check_output(
+                [
+                    "powershell",
+                    "-c",
+                    f"""Get-CimInstance Win32_Process \
+                        -filter "commandline like '%-S {control_sock_path}%'" \
+                        | select -ExpandProperty CommandLine \
+                    """,
+                ]
+            )
+        else:
+            filter_prefix = "grep"
             ps = subprocess.Popen(("ps", "-A", "-o", "command"), stdout=subprocess.PIPE)
-            control_sock_path = host_config.get("ControlPath")
-            output = subprocess.check_output(("grep", control_sock_path), stdin=ps.stdout)
+            output = subprocess.check_output(
+                ["grep", "--", f"-S {control_sock_path}"], stdin=ps.stdout
+            )
             ps.wait()
-            commands = list(
-                filter(lambda s: not s.startswith("grep"), output.decode().strip().split("\n"))
+        commands = list(
+            filter(lambda s: not s.startswith(filter_prefix), output.decode().strip().split("\n"))
+        )
+        if commands:
+            port_pattern = r"-L (?:[\w.-]+:)?(\d+):localhost:(\d+)"
+            matches = re.findall(port_pattern, commands[0])
+            return PortsLock(
+                {int(target_port): int(local_port) for local_port, target_port in matches}
             )
-            if commands:
-                port_pattern = r"-L (?:[\w.-]+:)?(\d+):localhost:(\d+)"
-                matches = re.findall(port_pattern, commands[0])
-                return control_sock_path, PortsLock(
-                    {int(target_port): int(local_port) for local_port, target_port in matches}
-                )
         return None
 
     def __init__(
@@ -48,17 +72,21 @@ def __init__(
         run_name: str,
         dockerized: bool,
         ssh_proxy: Optional[SSHConnectionParams] = None,
-        control_sock_path: Optional[str] = None,
         local_backend: bool = False,
         bind_address: Optional[str] = None,
     ):
         self._ports_lock = ports_lock
         self.ports = ports_lock.dict()
         self.run_name = run_name
-        self.ssh_config_path = str(ConfigManager().dstack_ssh_config_path)
+        self.ssh_config_path = ConfigManager().dstack_ssh_config_path
+        control_sock_path = self.get_control_sock_path(run_name)
+        # Cast all path-like values used in configs to FilePath instances for automatic
+        # path normalization in :func:`update_ssh_config`.
+        self.control_sock_path = FilePath(control_sock_path)
+        self.identity_file = FilePath(id_rsa_path)
         self.tunnel = SSHTunnel(
             destination=run_name,
-            identity=FilePath(id_rsa_path),
+            identity=self.identity_file,
             forwarded_sockets=ports_to_forwarded_sockets(
                 ports=self.ports,
                 bind_local=bind_address or "localhost",
@@ -72,7 +100,7 @@ def __init__(
                 "HostName": hostname,
                 "Port": ssh_port,
                 "User": user,
-                "IdentityFile": id_rsa_path,
+                "IdentityFile": self.identity_file,
                 "IdentitiesOnly": "yes",
                 "StrictHostKeyChecking": "no",
                 "UserKnownHostsFile": "/dev/null",
@@ -82,7 +110,7 @@ def __init__(
                 "HostName": ssh_proxy.hostname,
                 "Port": ssh_proxy.port,
                 "User": ssh_proxy.username,
-                "IdentityFile": id_rsa_path,
+                "IdentityFile": self.identity_file,
                 "IdentitiesOnly": "yes",
                 "StrictHostKeyChecking": "no",
                 "UserKnownHostsFile": "/dev/null",
@@ -92,31 +120,32 @@ def __init__(
                 "HostName": "localhost",
                 "Port": 10022,
                 "User": "root",  # TODO(#1535): support non-root images properly
-                "IdentityFile": id_rsa_path,
+                "IdentityFile": self.identity_file,
                 "IdentitiesOnly": "yes",
                 "StrictHostKeyChecking": "no",
                 "UserKnownHostsFile": "/dev/null",
-                "ControlPath": self.tunnel.control_sock_path,
-                "ControlMaster": "auto",
-                "ControlPersist": "yes",
                 "ProxyJump": f"{run_name}-host",
             }
         elif ssh_proxy is not None:
             self.container_config = {
                 "HostName": hostname,
                 "Port": ssh_port,
                 "User": user,
-                "IdentityFile": id_rsa_path,
+                "IdentityFile": self.identity_file,
                 "IdentitiesOnly": "yes",
                 "StrictHostKeyChecking": "no",
                 "UserKnownHostsFile": "/dev/null",
-                "ControlPath": self.tunnel.control_sock_path,
-                "ControlMaster": "auto",
-                "ControlPersist": "yes",
                 "ProxyJump": f"{run_name}-jump-host",
             }
         else:
             self.container_config = None
+        if self.container_config is not None and get_ssh_client_info().supports_multiplexing:
+            self.container_config.update(
+                {
+                    "ControlMaster": "auto",
+                    "ControlPath": self.control_sock_path,
+                }
+            )
 
     def attach(self):
         include_ssh_config(self.ssh_config_path)