Add download/3 DSL verb. (#192)

lib/bootleg/config.ex

@@ -10,7 +10,7 @@ defmodule Bootleg.Config do
     quote do
       import Bootleg.Config, only: [role: 2, role: 3, config: 2, config: 1, config: 0,
         before_task: 2, after_task: 2, invoke: 1, task: 2, remote: 1, remote: 2,
-        remote: 3, load: 1, upload: 3]
+        remote: 3, load: 1, upload: 3, download: 3]
     end
   end
 
@@ -444,10 +444,10 @@ defmodule Bootleg.Config do
   # runs for hosts found in :build first, then for hosts in :app
   remote [:build, :app], do: "hostname"
 
-  # only runs on `host1.example.com`
   role :build, "host2.example.com"
-  role :build, "host1.example.com", filter: [primary: true, another_attr: :cat]
+  role :build, "host1.example.com", primary: true, another_attr: :cat
 
+  # only runs on `host1.example.com`
   remote :build, filter: [primary: true] do
     "hostname"
   end
@@ -536,6 +536,59 @@ defmodule Bootleg.Config do
     end
   end
 
+  @doc """
+  Downloads files from remote hosts to the local machine.
+
+  Downloading works much like `remote/3`, but instead of transferring shell commands over SSH,
+  it transfers files via SCP. The remote host does need to support SCP, which should be provided
+  by most SSH implementations automatically.
+
+  `role` can either be a single role name, a list of roles, or a list of roles and filter
+  attributes. The special `:all` role is also supported. See `remote/3` for details. Note that
+  if multiple hosts match, files will be downloaded from all matching hosts, and any duplicate
+  file names will result in collisions. The exact semantics of how that works are handled by
+  `SSHKit.SCP`, but in general the file transfered last wins.
+
+  `local_path` is a path to local directory or file where the downloaded files(s) should be placed.
+  Absolute paths will be respected, relative paths will be resolved relative to the current working
+  directory of the invoking shell. If the `local_path` does not exist in the local file system, an
+  attempt will be made to create the missing directory. This does not handle nested directories,
+  and a `File.Error` will be raised.
+
+  `remote_path` is the file or directory to be copied from the remote hosts. If a directory is
+  specified, its contents will be recursively copied. Relative paths will be resolved relative to
+  the remote workspace, absolute paths will be respected.
+
+  The files on the local host are created using the current user's `uid`/`gid` and `umask`.
+
+  ```
+  use Bootleg.Config
+
+  # copies ./my_file from the remote host to ./new_name locally
+  download :app, "my_file", "new_name"
+
+  # copies ./my_file from the remote host to the file ./a_dir/my_file locally
+  download :app, "my_file", "a_dir"
+
+  # recursively copies ./some_dir on the remote host to ./new_dir locally, ./new_dir
+  # will be created if missing
+  download :app, "some_dir", "new_dir"
+
+  # copies /foo/my_file on the remote host to /tmp/foo locally
+  download :app, "/foo/my_file", "/tmp/foo"
+  """
+  defmacro download(role, remote_path, local_path) do
+    {roles, filters} = split_roles_and_filters(role)
+    roles = unpack_role(roles)
+    quote bind_quoted: binding() do
+      Enum.each(roles, fn role ->
+        role
+        |> SSH.init([], filters)
+        |> SSH.download(remote_path, local_path)
+      end)
+    end
+  end
+
   @doc false
   @spec get_config(atom, any) :: any
   def get_config(key, default \\ nil) do

lib/bootleg/ssh.ex

@@ -171,8 +171,9 @@ defmodule Bootleg.SSH do
 
   def download(conn, remote_path, local_path) do
     UI.puts_download conn, remote_path, local_path
-    case SSHKit.download(conn, remote_path, as: local_path) do
+    case SSHKit.download(conn, remote_path, as: local_path, recursive: true) do
       [:ok|_] -> :ok
+      [] -> :ok
       [{_, msg}|_] -> raise "SCP download error: #{msg}"
     end
   end

test/bootleg/config_functional_test.exs

@@ -121,6 +121,22 @@ defmodule Bootleg.ConfigFunctionalTest do
     end)
   end
 
+  @tag boot: 3
+  test "remote/2 multiple roles/hosts" do
+    capture_io(fn ->
+      use Bootleg.Config
+
+      assert [{:ok, [stdout: host_1], 0, _}, {:ok, [stdout: host_2], 0, _}] = remote :build, do: "hostname"
+      refute host_1 == host_2
+
+      assert [
+        [{:ok, [stdout: host_1], 0, _},
+        {:ok, [stdout: host_2], 0, _}],
+        [{:ok, [stdout: host_3], 0, _}]] = remote [:build, :app], do: "hostname"
+      refute host_1 == host_2 == host_3
+    end)
+  end
+
   test "remote/2 fails remotely" do
     use Bootleg.Config
 
@@ -219,7 +235,6 @@ defmodule Bootleg.ConfigFunctionalTest do
 
       invoke :upload_role_filtered
       assert_raise SSHError, fn ->
-        # credo:disable-for-next-line Credo.Check.Consistency.MultiAliasImportRequireUse
         use Bootleg.Config
         remote :all, do: "grep '^upload_role_filtered$' role_filtered"
       end
@@ -237,4 +252,139 @@ defmodule Bootleg.ConfigFunctionalTest do
       invoke :upload_preserve_name
     end)
   end
+
+  @tag boot: 3
+  test "download/3" do
+    capture_io(fn ->
+      # credo:disable-for-next-line Credo.Check.Consistency.MultiAliasImportRequireUse
+      use Bootleg.Config
+
+      path = Temp.mkdir!("download")
+
+      # single file, single role and host
+      remote :app, do: "echo -n download_single_role >> download_single_role"
+      download :app, "download_single_role", path
+
+      assert {:ok, "download_single_role"} = File.read(Path.join(path, "download_single_role"))
+
+      # single file, single role, multiple hosts
+      remote :build, do: "hostname >> download_single_role_multi_host"
+      [_, {:ok, [stdout: host], 0, _}] = remote :build, do: "hostname"
+      download :build, "download_single_role_multi_host", path
+
+      assert {:ok, ^host} = File.read(Path.join(path, "download_single_role_multi_host"))
+
+      # single file, multiple roles/hosts
+      remote [:build, :app], do: "hostname >> download_multi_role"
+      [[_, _], [{:ok, [stdout: host], 0, _}]] = remote [:build, :app], do: "hostname"
+      download :build, "download_multi_role", path
+
+      assert {:ok, ^host} = File.read(Path.join(path, "download_multi_role"))
+
+      # single file, :all role (multiple hosts)
+      remote :all, do: "hostname >> download_all_role"
+      [[_, _], [{:ok, [stdout: host], 0, _}]] = remote :all, do: "hostname"
+      download :all, "download_all_role", path
+      assert {:ok, ^host} = File.read(Path.join(path, "download_all_role"))
+
+      # single file, filtered role
+      remote :all, [filter: [foo: :bar]], "hostname >> download_role_filtered"
+      [{:ok, [stdout: host], 0, _}] = remote :all, [filter: [foo: :bar]], "hostname"
+      download [:all, foo: :bar], "download_role_filtered", path
+      assert {:ok, ^host} = File.read(Path.join(path, "download_role_filtered"))
+
+      # recursively download directory
+      remote :app do
+        "mkdir -p to_download"
+        "mkdir -p to_download/deep/deeper"
+        "touch to_download/foo"
+        "touch to_download/bar"
+        "hostname >> to_download/hostname"
+        "uname -a >> to_download/deep/deeper/the_depths"
+      end
+      [{:ok, [stdout: host], 0, _}] = remote :app, "hostname"
+      download :app, "to_download", path
+      to_download_path = Path.join(path, "to_download")
+      assert {:ok, ^host} = File.read(Path.join(to_download_path, "hostname"))
+      assert {:ok, ""} = File.read(Path.join(to_download_path, "foo"))
+      assert {:ok, ""} = File.read(Path.join(to_download_path, "bar"))
+      assert {:ok, uname} = to_download_path
+        |> Path.join("deep")
+        |> Path.join("deeper")
+        |> Path.join("the_depths")
+        |> File.read
+      assert String.match?(uname, ~r{Linux})
+
+      # remote absolute path
+      remote :app, "hostname >> /tmp/download_abs"
+      [{:ok, [stdout: host], 0, _}] = remote :app, "hostname"
+      download :app, "/tmp/download_abs", path
+      assert {:ok, ^host} = File.read(Path.join(path, "download_abs"))
+
+      # remote absolute path with local rename
+      remote :app, "hostname >> /tmp/download_alt_name"
+      [{:ok, [stdout: host], 0, _}] = remote :app, "hostname"
+      download :app, "/tmp/download_alt_name", Path.join(path, "new_name")
+      assert {:ok, ^host} = File.read(Path.join(path, "new_name"))
+
+      # a single missing directory will be created
+      remote :app do
+        "mkdir -p /tmp/download_dir"
+        "hostname >> /tmp/download_dir/a_file"
+      end
+
+      [{:ok, [stdout: host], 0, _}] = remote :app, "hostname"
+
+      to_download_path = Path.join(path, "not_a_dir")
+      download :app, "/tmp/download_dir", to_download_path
+      assert {:ok, ^host} = File.read(Path.join(to_download_path, "a_file"))
+
+      # nested local directories are not created
+      remote :app do
+        "mkdir -p /tmp/download_dir_deep"
+        "hostname >> /tmp/download_dir_deep/a_file"
+      end
+
+      to_download_path = path
+        |> Path.join("not_a_dir_error")
+        |> Path.join("deeper_still")
+      assert_raise File.Error, fn ->
+        download :app, "/tmp/download_dir_deep", to_download_path
+      end
+      assert_raise File.Error, fn ->
+        download :app, "/tmp/download_dir_deep/a_file", to_download_path
+      end
+
+      # local file is clobbered
+      remote :app, "hostname >> /tmp/download_dir_local_exists"
+      [{:ok, [stdout: host], 0, _}] = remote :app, "hostname"
+      to_download_path = Path.join(path, "i_exist")
+      File.write!(to_download_path, "some content")
+      download :app, "/tmp/download_dir_local_exists", to_download_path
+      assert {:ok, ^host} = File.read(to_download_path)
+
+      # local directories will be respected
+      remote :app, "hostname >> /tmp/download_dir_exists"
+      [{:ok, [stdout: host], 0, _}] = remote :app, "hostname"
+      to_download_path = Path.join(path, "dir_exists")
+      File.mkdir!(to_download_path)
+      download :app, "/tmp/download_dir_exists", to_download_path
+      assert {:ok, ^host} = File.read(Path.join(to_download_path, "download_dir_exists"))
+
+      # trailing slashes are ignored
+      remote :app, "hostname >> /tmp/download_force_dir"
+      [{:ok, [stdout: host], 0, _}] = remote :app, "hostname"
+      to_download_path = Path.join(path, "forced_dir") <> "/"
+      download :app, "/tmp/download_force_dir", to_download_path
+      assert {:ok, ^host} = File.read(Path.join(path, "forced_dir"))
+
+      # trailing current directory characters are not respected
+      remote :app, "hostname >> /tmp/download_force_current_dir"
+      [{:ok, [stdout: host], 0, _}] = remote :app, "hostname"
+      to_download_path = Path.join(path, "forced_current_dir")
+      download :app, "/tmp/download_force_current_dir", to_download_path <> "/."
+      assert {:error, :enotdir} = File.read(Path.join(to_download_path, "download_force_current_dir"))
+      assert {:ok, ^host} = File.read(Path.join(path, "forced_current_dir"))
+    end)
+  end
 end

test/bootleg/config_test.exs

@@ -596,7 +596,6 @@ defmodule Bootleg.ConfigTest do
   end
 
   test "config/1" do
-    # credo:disable-for-next-line Credo.Check.Consistency.MultiAliasImportRequireUse
     use Bootleg.Config
 
     refute config(:foo)
@@ -608,4 +607,60 @@ defmodule Bootleg.ConfigTest do
     config(:foo, nil)
     assert config({:foo, :bar}) == nil
   end
+
+  test_with_mock "download/3", SSH, [:passthrough], [
+      init: fn(role, options, filter) -> {role, options, filter} end,
+      download: fn(_conn, _remote, _local) -> :ok end,
+    ] do
+    # credo:disable-for-next-line Credo.Check.Consistency.MultiAliasImportRequireUse
+    use Bootleg.Config
+
+    role :foo, "never-used-foo.example.com"
+    role :car, "never-used-bar.example.com"
+
+    download :foo, "some/remote/path", "the/local/path"
+
+    assert called SSH.init(:foo, [], [])
+    assert called SSH.download({:foo, [], []}, "some/remote/path", "the/local/path")
+
+    download [:foo, :bar], "some/remote/path", "the/local/path"
+
+    assert called SSH.init(:foo, [], [])
+    assert called SSH.init(:bar, [], [])
+    assert called SSH.download({:foo, [], []}, "some/remote/path", "the/local/path")
+    assert called SSH.download({:bar, [], []}, "some/remote/path", "the/local/path")
+
+    download :all, "some/remote/path", "the/local/path"
+
+    assert called SSH.init(:foo, [], [])
+    assert called SSH.init(:car, [], [])
+    assert called SSH.download({:foo, [], []}, "some/remote/path", "the/local/path")
+    assert called SSH.download({:car, [], []}, "some/remote/path", "the/local/path")
+
+    download [:foo, primary: true], "some/remote/path", "the/local/path"
+
+    assert called SSH.init(:foo, [], [primary: true])
+    assert called SSH.download({:foo, [], [primary: true]}, "some/remote/path", "the/local/path")
+
+    download [:foo, :bar, primary: true], "some/remote/path", "the/local/path"
+
+    assert called SSH.init(:foo, [], [primary: true])
+    assert called SSH.init(:bar, [], [primary: true])
+    assert called SSH.download({:foo, [], [primary: true]}, "some/remote/path", "the/local/path")
+    assert called SSH.download({:bar, [], [primary: true]}, "some/remote/path", "the/local/path")
+
+    download [:foo, :bar, primary: true, db: :mysql], "some/remote/path", "the/local/path"
+
+    assert called SSH.init(:foo, [], [primary: true, db: :mysql])
+    assert called SSH.init(:bar, [], [primary: true, db: :mysql])
+    assert called SSH.download({:foo, [], [primary: true, db: :mysql]}, "some/remote/path", "the/local/path")
+    assert called SSH.download({:bar, [], [primary: true, db: :mysql]}, "some/remote/path", "the/local/path")
+
+    download [:all, db: :mysql], "some/remote/path", "the/local/path"
+
+    assert called SSH.init(:foo, [], [db: :mysql])
+    assert called SSH.init(:car, [], [db: :mysql])
+    assert called SSH.download({:foo, [], [db: :mysql]}, "some/remote/path", "the/local/path")
+    assert called SSH.download({:car, [], [db: :mysql]}, "some/remote/path", "the/local/path")
+  end
 end