Merge pull request #25 from rizo/src-extra

Fetch extra-source files
rizo · May 30, 2024 · 2af576a · 2af576a
2 parents 623a937 + be0df49
commit 2af576a
Show file tree

Hide file tree

Showing 5 changed files with 117 additions and 27 deletions.
diff --git a/nix/core.nix b/nix/core.nix
@@ -99,13 +99,21 @@ let
         in lib.lists.toList (lib.attrsets.attrByPath pkgPath [ ] pkgs))
         (dep.depexts or [ ]);
 
+      # Main src of the dep
       src = if dep ? src then
         fetchSrc {
           inherit rootPath name;
           inherit (dep) src;
         }
-      else
-        null;
+      else null;
+
+      # The extra-source files
+      srcExtraCommands =
+        lib.attrsets.mapAttrsToList
+          (name: src: ''
+            ln -s ${pkgs.fetchurl src} "${name}"
+          '')
+          (dep.src-extra or {});
 
       opam = getOpamFile {
         inherit repoPath src name;
@@ -150,7 +158,12 @@ let
       ONIX_LOG_LEVEL = verbosity;
       ONIXPATH = lib.strings.concatStringsSep ":" dependsAndBuildPkgs;
 
+      # Steps:
+      # - Link extra-sources
+      # Run the opam patch actions
       prePatch = ''
+        ${lib.concatStrings srcExtraCommands}
+
         ${onix}/bin/onix opam-patch \
           --ocaml-version=${ocaml.version} \
           --opam=${opam} \

diff --git a/src/onix_core/Lock_pkg.ml b/src/onix_core/Lock_pkg.ml
@@ -21,15 +21,17 @@ module Resolvers = struct
       ]
 end
 
+type http_src = {
+  url : OpamUrl.t;
+  hash : OpamHash.kind * string;
+}
+
 type src =
   | Git of {
       url : string;
       rev : string;
     }
-  | Http of {
-      url : OpamUrl.t;
-      hash : OpamHash.kind * string;
-    }
+  | Http of http_src
 
 let src_is_git src =
   match src with
@@ -53,6 +55,7 @@ type t = {
   depexts_unknown : String_set.t;
   vars : Opam_utils.dep_vars;
   flags : string list;
+  extra_src : (string * http_src) list;
 }
 
 let check_is_zip_src src =
@@ -95,6 +98,18 @@ let select_opam_hash hashes =
   | Some hash, _, _ -> Some (`MD5, OpamHash.contents hash)
   | _ -> None
 
+let http_src_of_opam_url ~hashes url =
+  let hash =
+    match select_opam_hash hashes with
+    | Some hash -> hash
+    | None ->
+      Logs.warn (fun log ->
+          log "Prefetching url without hash: %a" Opam_utils.pp_url url);
+      ( `SHA256,
+        Nix_utils.prefetch_url ~hash_type:`sha256 (OpamUrl.to_string url) )
+  in
+  { url; hash }
+
 let src_of_opam_url opam_url =
   let url = OpamFile.URL.url opam_url in
   match url.OpamUrl.backend with
@@ -104,16 +119,8 @@ let src_of_opam_url opam_url =
     | _ -> Error (`Msg ("Missing rev in git url: " ^ OpamUrl.to_string url)))
   | `http ->
     let hashes = OpamFile.URL.checksum opam_url in
-    let hash =
-      match select_opam_hash hashes with
-      | Some hash -> hash
-      | None ->
-        Logs.warn (fun log ->
-            log "Prefetching url without hash: %a" Opam_utils.pp_url url);
-        ( `SHA256,
-          Nix_utils.prefetch_url ~hash_type:`sha256 (OpamUrl.to_string url) )
-    in
-    Ok (Http { url; hash })
+    let http_src = http_src_of_opam_url ~hashes url in
+    Ok (Http http_src)
   | _ -> Error (`Msg ("Unsupported url: " ^ OpamUrl.to_string url))
 
 let get_src ~package opam_url_opt =
@@ -203,6 +210,22 @@ let get_depexts ~package ~is_zip_src ~env depexts =
   in
   (nix_depexts, unknown_depexts)
 
+let get_extra_sources (srcs : (OpamTypes.basename * OpamFile.URL.t) list) =
+  List.map
+    (fun (basename, opam_url) ->
+      let url = OpamFile.URL.url opam_url in
+      let name = OpamFilename.Base.to_string basename in
+      let http_src =
+        match url.OpamUrl.backend with
+        | `git | `hg | `rsync | `darcs ->
+          Fmt.failwith "extra-source: %S: invalid url, must be http" name
+        | `http ->
+          let hashes = OpamFile.URL.checksum opam_url in
+          http_src_of_opam_url ~hashes url
+      in
+      (name, http_src))
+    srcs
+
 (* Given required and optional deps, compute a union of all installed deps. *)
 let only_installed ~installed req opt =
   (* All req deps MUST be installed. *)
@@ -288,6 +311,8 @@ let of_opam ~installed ~with_test ~with_doc ~with_dev_setup opam_details =
     List.map OpamTypesBase.string_of_pkg_flag (OpamFile.OPAM.flags opam)
   in
 
+  let extra_src = get_extra_sources (OpamFile.OPAM.extra_sources opam) in
+
   Some
     {
       src;
@@ -302,4 +327,5 @@ let of_opam ~installed ~with_test ~with_doc ~with_dev_setup opam_details =
       depexts_unknown;
       flags;
       vars = { Opam_utils.test; doc; dev_setup };
+      extra_src;
     }
diff --git a/src/onix_core/Lock_pkg.mli b/src/onix_core/Lock_pkg.mli
@@ -1,14 +1,16 @@
 open Utils
 
+type http_src = {
+  url : OpamUrl.t;
+  hash : OpamHash.kind * string;
+}
+
 type src =
   | Git of {
       url : string;
       rev : string;
     }
-  | Http of {
-      url : OpamUrl.t;
-      hash : OpamHash.kind * string;
-    }
+  | Http of http_src
 
 type t = {
   src : src option;
@@ -22,6 +24,7 @@ type t = {
   depexts_unknown : String_set.t;
   vars : Opam_utils.dep_vars;
   flags : string list;
+  extra_src : (string * http_src) list;
 }
 
 val src_is_git : src -> bool

diff --git a/src/onix_lock_json/Pp.ml b/src/onix_lock_json/Pp.ml
@@ -13,6 +13,9 @@ let pp_hash f (kind, hash) =
   | `SHA512 -> Fmt.pf f "\"sha512\": %S" hash
   | `MD5 -> Fmt.pf f "\"md5\": %S" hash
 
+let pp_http_src f ({ url; hash } : Lock_pkg.http_src) =
+  Fmt.pf f "\"url\": %a,@,%a" (Fmt.quote Opam_utils.pp_url) url pp_hash hash
+
 let pp_src f (t : Lock_pkg.t) =
   if Opam_utils.Opam_details.check_has_absolute_path t.opam_details then
     (* Absolute path: use src. *)
@@ -26,10 +29,8 @@ let pp_src f (t : Lock_pkg.t) =
     | Some (Http { url; hash = `MD5, _ }) ->
       Fmt.invalid_arg "Unexpected md5 hash: package=%a url=%a"
         Opam_utils.pp_package t.opam_details.package Opam_utils.pp_url url
-    | Some (Http { url; hash }) ->
-      Fmt.pf f ",@,@[<v2>\"src\": {@,\"url\": %a,@,%a@]@,}"
-        (Fmt.quote Opam_utils.pp_url)
-        url pp_hash hash
+    | Some (Http hsrc) ->
+      Fmt.pf f ",@,@[<v2>\"src\": {@,%a@]@,}" pp_http_src hsrc
   else
     (* Relative path: use file scheme. *)
     let path =
@@ -43,6 +44,18 @@ let pp_src f (t : Lock_pkg.t) =
     in
     Fmt.pf f ",@,\"src\": { \"url\": \"file://%s\" }" path
 
+let pp_extra_src =
+  let pp_item f (name, hsrc) =
+    Fmt.pf f "@[<v2>%S: {@,%a@]@,}" name pp_http_src hsrc
+  in
+  fun f (srcs : (string * Lock_pkg.http_src) list) ->
+    if List.is_empty srcs then ()
+    else begin
+      Fmt.pf f ",@,@[<v2>\"src-extra\": {@ %a@]@ }"
+        (Fmt.iter ~sep:Fmt.comma List.iter pp_item)
+        srcs
+    end
+
 let pp_depends =
   let pp_deps = Fmt.iter ~sep:Fmt.comma Name_set.iter pp_name_quoted in
   fun key f deps ->
@@ -72,9 +85,9 @@ let pp_pkg_vars f vars =
     Fmt.pf f ",@,@[<v2>\"vars\": { %s }@]" vars_str
 
 let pp_pkg ppf (t : Lock_pkg.t) =
-  Fmt.pf ppf "\"version\": %S%a%a%a%a%a%a%a%a"
+  Fmt.pf ppf "\"version\": %S%a%a%a%a%a%a%a%a%a"
     (OpamPackage.version_to_string t.opam_details.package)
-    pp_src t (pp_depends "depends") t.depends
+    pp_src t pp_extra_src t.extra_src (pp_depends "depends") t.depends
     (pp_depends "build-depends")
     t.depends_build
     (pp_depends "test-depends")

diff --git a/tests/Test_lock.ml b/tests/Test_lock.ml
@@ -26,6 +26,26 @@ depexts: [
   ["libogg"] {os-distribution = "nixos"}
   ["libogg"] {os = "macos" & os-distribution = "homebrew"}
 ]
+url {
+  src: "https://github.com/xavierleroy/camlzip/archive/rel110.zip"
+  checksum: "sha256=a5541cbc38c14467a8abcbdcb54c1d2ed12515c1c4c6da0eb3bdafb44aff7996"
+}
+extra-source "gui_gtk_dir.patch" {
+  src:
+    "https://raw.githubusercontent.com/ocaml/opam-source-archives/main/patches/0install/gui_gtk_dir.patch"
+  checksum: [
+    "sha256=ef4c291794ed4ca7f024c671f48a8aaa2dcd9d12c1ab73829373a7d904e537e1"
+    "md5=0a14e57ca2b2a914a5433b3a2ca2abb1"
+  ]
+}
+extra-source "0install.install" {
+  src:
+    "https://raw.githubusercontent.com/ocaml/opam-source-archives/main/patches/0install/0install.install"
+  checksum: [
+    "sha256=db9ef395b376617d963fd4c097ebdfe005978f9a3282810f858f89207fa85ab2"
+    "md5=db6ee7a35da5d98136e5a56bad08496e"
+  ]
+}
 |}
 
 let dev_opam =
@@ -116,6 +136,20 @@ let test_complex_opam () =
   let actual = Fmt.str "@[<v>%a@]@." Onix_lock_json.Pp.pp_pkg lock_pkg in
   let expected =
     {|"version": "root",
+"src": {
+  "url": "https://github.com/xavierleroy/camlzip/archive/rel110.zip",
+  "sha256": "a5541cbc38c14467a8abcbdcb54c1d2ed12515c1c4c6da0eb3bdafb44aff7996"
+},
+"src-extra": {
+  "0install.install": {
+    "url": "https://raw.githubusercontent.com/ocaml/opam-source-archives/main/patches/0install/0install.install",
+    "sha256": "db9ef395b376617d963fd4c097ebdfe005978f9a3282810f858f89207fa85ab2"
+  },
+  "gui_gtk_dir.patch": {
+    "url": "https://raw.githubusercontent.com/ocaml/opam-source-archives/main/patches/0install/gui_gtk_dir.patch",
+    "sha256": "ef4c291794ed4ca7f024c671f48a8aaa2dcd9d12c1ab73829373a7d904e537e1"
+  }
+},
 "depends": [
   "bos",
   "cmdliner",
@@ -135,7 +169,8 @@ let test_complex_opam () =
   "odoc"
 ],
 "depexts": [
-  "libogg"
+  "libogg",
+  "unzip"
 ],
 "vars": { "with-test": true, "with-doc": true, "with-dev-setup": true }
 |}