add httpmirror, truncate harder when retrying

include/powerloader/context.hpp

@@ -17,7 +17,7 @@ namespace powerloader
     namespace fs = std::filesystem;
 
     class Context;
-    struct Mirror;
+    class Mirror;
 
     using mirror_set
         = std::vector<std::shared_ptr<Mirror>>;  // TODO: replace by std::flat_set once available.

include/powerloader/mirror.hpp

@@ -63,10 +63,15 @@ namespace powerloader
     };
 
     // mirrors should be dict -> urls mapping
-    struct POWERLOADER_API Mirror
+    class POWERLOADER_API Mirror
     {
-        Mirror(MirrorID id, const Context& ctx, const std::string& url);
-        Mirror(const Context& ctx, const std::string& url);
+    public:
+        Mirror(const MirrorID& id, const Context& ctx, const std::string& url)
+            : m_id(id)
+            , m_url(url)
+        {
+        }
+
         virtual ~Mirror();
 
         Mirror(const Mirror&) = delete;
@@ -156,8 +161,8 @@ namespace powerloader
         }
 
     private:
-        std::string m_url;
         const MirrorID m_id;
+        const std::string m_url;
 
         Protocol m_protocol = Protocol::kHTTP;
         MirrorState m_state = MirrorState::READY;
@@ -180,6 +185,36 @@ namespace powerloader
         std::size_t m_retry_counter = 0;
     };
 
+    class POWERLOADER_API HTTPMirror : public Mirror
+    {
+    public:
+        HTTPMirror(const Context& ctx, const std::string& url)
+            : Mirror(HTTPMirror::id(url), ctx, url)
+        {
+        }
+
+        static MirrorID id(const std::string& url)
+        {
+            return MirrorID{ fmt::format("HTTPMirror[{}]", url) };
+        }
+
+        void set_auth(const std::string& user, const std::string& password);
+
+        bool prepare(Target* target) override;
+        bool prepare(const std::string& path, CURLHandle& handle) override;
+
+        bool needs_preparation(Target* target) const override;
+        bool authenticate(CURLHandle& handle, const std::string& path) override;
+
+        std::vector<std::string> get_auth_headers(const std::string& path) const override;
+
+        std::string format_url(Target* target) const override;
+
+    private:
+        std::string m_auth_user;
+        std::string m_auth_password;
+    };
+
     bool sort_mirrors(std::vector<std::shared_ptr<Mirror>>& mirrors,
                       const std::shared_ptr<Mirror>& mirror,
                       bool success,

src/cli/main.cpp

@@ -392,7 +392,7 @@ parse_mirrors(const Context& ctx, const YAML::Node& node)
             else if (kof == KindOf::kHTTP)
             {
                 spdlog::info("Adding HTTP mirror: {} -> {}", mirror_name, creds.url.url());
-                result.create_unique_mirror<Mirror>(mirror_name, ctx, creds.url.url());
+                result.create_unique_mirror<HTTPMirror>(mirror_name, ctx, creds.url.url());
             }
         }
     }

src/download_target.cpp

@@ -46,7 +46,7 @@ namespace powerloader
             URLHandler uh{ target_url };
             if (uh.scheme() == "file")
             {
-                ctx.mirror_map.create_unique_mirror<Mirror>("[file]", ctx, "file://");
+                ctx.mirror_map.create_unique_mirror<HTTPMirror>("[file]", ctx, "file://");
                 return std::make_shared<DownloadTarget>(uh.path(), "[file]", destination_path);
             }
 
@@ -57,7 +57,7 @@ namespace powerloader
             const fs::path dst = destination_path.empty() ? fs::path{ rsplit(path, "/", 1).back() }
                                                           : destination_path;
 
-            ctx.mirror_map.create_unique_mirror<Mirror>(host, ctx, mirror_url);
+            ctx.mirror_map.create_unique_mirror<HTTPMirror>(host, ctx, mirror_url);
             return std::make_shared<DownloadTarget>(path.substr(1, std::string::npos), host, dst);
         }
         else

src/downloader.cpp

@@ -571,7 +571,6 @@ namespace powerloader
                 bool retry = false;
 
                 spdlog::error("Error during transfer");
-
                 // Call mirrorfailure callback
                 // LrMirrorFailureCb mf_cb = target->target().mirrorfailurecb;
                 // if (mf_cb)

src/mirror.cpp

@@ -7,24 +7,6 @@
 
 namespace powerloader
 {
-    Mirror::Mirror(MirrorID id, const Context& ctx, const std::string& url)
-        : m_url(url)
-        , m_id(id)
-    {
-        if (url.back() == '/' && url != "file://")
-            m_url = m_url.substr(0, m_url.size() - 1);
-
-        if (ctx.max_downloads_per_mirror > 0)
-        {
-            m_stats.allowed_parallel_connections = ctx.max_downloads_per_mirror;
-        }
-    }
-
-    Mirror::Mirror(const Context& ctx, const std::string& url)
-        : Mirror(Mirror::id(url), ctx, url)
-    {
-    }
-
     Mirror::~Mirror() = default;
 
     void Mirror::change_max_ranges(int new_value)
@@ -202,4 +184,49 @@ namespace powerloader
 
         return true;
     }
+
+    bool HTTPMirror::prepare(Target* target)
+    {
+        return true;
+    }
+
+    bool HTTPMirror::prepare(const std::string& path, CURLHandle& handle)
+    {
+        return true;
+    }
+
+    bool HTTPMirror::needs_preparation(Target* target) const
+    {
+        return false;
+    }
+
+    bool HTTPMirror::authenticate(CURLHandle& handle, const std::string& path)
+    {
+        if (!m_auth_password.empty())
+        {
+            spdlog::warn(
+                "Setting HTTP authentication for {} to {}:{}", path, m_auth_user, m_auth_password);
+            handle.setopt(CURLOPT_USERNAME, m_auth_user.c_str());
+            handle.setopt(CURLOPT_PASSWORD, m_auth_password.c_str());
+        }
+        return true;
+    }
+
+    void HTTPMirror::set_auth(const std::string& user, const std::string& password)
+    {
+        m_auth_user = user;
+        m_auth_password = password;
+    }
+
+
+    std::vector<std::string> HTTPMirror::get_auth_headers(const std::string& path) const
+    {
+        return {};
+    }
+
+    std::string HTTPMirror::format_url(Target* target) const
+    {
+        return join_url(url(), target->target().path());
+    }
+
 }

src/python/main.cpp

@@ -24,13 +24,14 @@ PYBIND11_MODULE(pypowerloader, m)
 
     py::class_<DownloadTarget, std::shared_ptr<DownloadTarget>>(m, "DownloadTarget")
         .def(py::init<const std::string&, const std::string&, const fs::path&>())
-        .def_property_readonly("complete_url", &DownloadTarget::complete_url)
         .def_property("progress_callback",
                       &DownloadTarget::progress_callback,
                       &DownloadTarget::set_progress_callback);
 
+    py::class_<MirrorID>(m, "MirrorID").def(py::init<const std::string&>());
+
     py::class_<Mirror, std::shared_ptr<Mirror>>(m, "Mirror")
-        .def(py::init<const Context&, const std::string&>());
+        .def(py::init<MirrorID, const Context&, const std::string&>());
 
     py::class_<mirror_map_type>(m, "MirrorMap")
         .def(py::init<>())

src/target.cpp

@@ -38,6 +38,7 @@ namespace powerloader
 
     void Target::reset()
     {
+        spdlog::warn("Resetting target {}", m_target->destination_path().string());
         if (m_target->outfile() && !zck_running())
         {
             std::error_code ec;
@@ -108,15 +109,29 @@ namespace powerloader
 
     bool Target::truncate_transfer_file()
     {
+        spdlog::warn("Truncating transfer file ... ");
         std::ptrdiff_t offset = 0;
         std::error_code ec;
 
-        if (!m_target->outfile() || !m_target->outfile()->open())
+        spdlog::warn("NOT TRUNCATING BECAUSE FILE IS {} or {}",
+                     (bool) m_target->outfile(),
+                     m_target->outfile()->open());
+        if (!m_target->outfile())
+        {
             return true;
+        }
+
+        if (!m_target->outfile()->open())
+        {
+            fs::remove(m_target->outfile()->path());
+            return true;
+        }
 
         if (m_original_offset >= 0)
             offset = m_original_offset;
 
+        spdlog::warn("Truncating transfer file offset {}", offset);
+
         m_target->outfile()->truncate(offset, ec);
         if (ec)
         {

test.py

@@ -10,7 +10,6 @@
 baseurl = "https://conda.anaconda.org/conda-forge"
 filename = "python3.9_test"
 downTarg = pypowerloader.DownloadTarget(path, baseurl, filename)
-print("complete url: " + downTarg.complete_url)
 
 
 def progress(total, done):