WIP: Add support for Go workspaces

This commit changes the resolve_gomod function so that it can handle a json stream coming from `go list -m`, which happens when workspaces are used in a Go project. It also removes the previous restrictions on using workspaces. Signed-off-by: Bruno Pimentel <[email protected]>
containerbuildsystem · May 21, 2024 · 3acda43 · 3acda43
1 parent 59967a1
commit 3acda43
Show file tree

Hide file tree

Showing 15 changed files with 17,944 additions and 71 deletions.
diff --git a/cachito/workers/pkg_managers/gomod.py b/cachito/workers/pkg_managers/gomod.py
@@ -375,7 +375,14 @@ def resolve_gomod(app_source_path, request, dep_replacements=None, git_dir_path=
             # Make Go ignore the vendor dir even if there is one
             go_list.extend(["-mod", "readonly"])
 
-        main_module_name = go([*go_list, "-m"], run_params).rstrip()
+        modules_json_stream = go([*go_list, "-m", "-json"], run_params).rstrip()
+
+        main_module, workspace_modules = _process_modules_json_stream(
+            app_source_path, modules_json_stream
+        )
+
+        main_module_name = main_module.path
+
         main_module_version = get_golang_version(
             main_module_name,
             git_dir_path,
@@ -387,6 +394,10 @@ def resolve_gomod(app_source_path, request, dep_replacements=None, git_dir_path=
                 else str(app_source_path).replace(f"{git_dir_path}/", "")
             ),
         )
+
+        for module in workspace_modules:
+            module.version = main_module_version
+
         main_module = {
             "type": "gomod",
             "name": main_module_name,
@@ -408,7 +419,7 @@ def go_list_deps(pattern: Literal["./...", "all"]) -> Iterator[GoPackage]:
             mod for pkg in go_list_deps("all") if (mod := pkg.module) and not mod.main
         )
         main_module_deps = _deduplicate_to_gomod_dicts(
-            chain(package_modules, downloaded_modules), deps_to_replace
+            chain(package_modules, downloaded_modules, workspace_modules), deps_to_replace
         )
 
         log.info("Retrieving the list of packages")
@@ -473,6 +484,34 @@ def go_list_deps(pattern: Literal["./...", "all"]) -> Iterator[GoPackage]:
         }
 
 
+def _process_modules_json_stream(
+    app_dir: Path, modules_json_stream: str
+) -> tuple[GoModule, list[GoModule]]:
+    """Process the json stream returned by "go list -m -json".
+
+    The stream will contain the module currently being processed, or a list of all workspaces in
+    case a go.work file is present in the repository.
+
+    :param app_dir: the path to the module directory
+    :param modules_json_stream: the json stream returned by "go list -m -json"
+    :return: A tuple containing the main module and a list of workspaces
+    """
+    module_list = []
+    main_module = None
+
+    for module in load_json_stream(modules_json_stream):
+        if module["Dir"] == str(app_dir):
+            main_module = GoModule.parse_obj(module)
+        else:
+            module_list.append(GoModule.parse_obj(module))
+
+    # should never happen, since the main module will always be a part of the json stream
+    if not main_module:
+        raise RuntimeError('Failed to find the main module info in the "go list -m" output.')
+
+    return main_module, module_list
+
+
 def _deduplicate_to_gomod_dicts(
     modules: Iterable[GoModule], user_specified_deps_to_replace: set[str]
 ) -> list[dict[str, Any]]:

diff --git a/cachito/workers/tasks/gomod.py b/cachito/workers/tasks/gomod.py
@@ -65,12 +65,6 @@ def _is_workspace(repo_root: Path, subpath: str):
     return False
 
 
-def _fail_if_bundle_dir_has_workspaces(bundle_dir: RequestBundleDir, subpaths: list[str]):
-    for subpath in subpaths:
-        if _is_workspace(bundle_dir.source_root_dir, subpath):
-            raise InvalidRepoStructure("Go workspaces are not supported by Cachito.")
-
-
 def _fail_if_parent_replacement_not_included(packages_json_data: PackagesData) -> None:
     """
     Fail if any dependency replacement refers to a parent dir that isn't included in this request.
@@ -139,8 +133,6 @@ def fetch_gomod_source(request_id, dep_replacements=None, package_configs=None):
         # Default to the root of the application source
         subpaths = [os.curdir]
 
-    _fail_if_bundle_dir_has_workspaces(bundle_dir, subpaths)
-
     invalid_gomod_files = _find_missing_gomod_files(bundle_dir, subpaths)
     if invalid_gomod_files:
         invalid_files_print = "; ".join(invalid_gomod_files)

diff --git a/hack/mock-unittest-data/gomod.sh b/hack/mock-unittest-data/gomod.sh
@@ -25,6 +25,35 @@ $(
     cd "$tmpdir/gomod-pandemonium"
     export GOMODCACHE="$tmpdir/cachito-mock-gomodcache"
 
+    git switch workspaces
+
+    echo "generating $mocked_data_dir/workspaces/go_list_modules.json"
+    go work edit -json > \
+        "$mocked_data_dir_abspath/workspaces/go_work.json"
+
+    echo "generating $mocked_data_dir/workspaces/go_list_modules.json"
+    go list -m -json > \
+        "$mocked_data_dir_abspath/workspaces/go_list_modules.json"
+
+    echo "generating $mocked_data_dir/workspaces/go_mod_download.json"
+    go mod download -json > \
+        "$mocked_data_dir_abspath/workspaces/go_mod_download.json"
+
+    echo "generating $mocked_data_dir/workspaces/go_list_deps_all.json"
+    go list -deps -json=ImportPath,Module,Standard,Deps all > \
+        "$mocked_data_dir_abspath/workspaces/go_list_deps_all.json"
+
+    echo "generating $mocked_data_dir/workspaces/go_list_deps_threedot.json"
+    go list -deps -json=ImportPath,Module,Standard,Deps ./... > \
+        "$mocked_data_dir_abspath/workspaces/go_list_deps_threedot.json"
+
+    git restore .
+    git switch main
+
+    echo "generating $mocked_data_dir/non-vendored/go_list_modules.json"
+    go list -m -json > \
+        "$mocked_data_dir_abspath/non-vendored/go_list_modules.json"
+
     echo "generating $mocked_data_dir/non-vendored/go_mod_download.json"
     go mod download -json > \
         "$mocked_data_dir_abspath/non-vendored/go_mod_download.json"
@@ -52,7 +81,9 @@ $(
 --------------------------------------------------------------------------------
 banner-end
 
-find "$mocked_data_dir/non-vendored" "$mocked_data_dir/vendored" -type f |
+find "$mocked_data_dir/non-vendored" \
+     "$mocked_data_dir/vendored" \
+     "$mocked_data_dir/workspaces" -type f |
     while read -r f; do
         sed "s|$tmpdir.cachito-mock-gomodcache|{gomodcache_dir}|" -i "$f"
         sed "s|$tmpdir.gomod-pandemonium|{repo_dir}|" -i "$f"

diff --git a/tests/integration/test_gomod_packages.py b/tests/integration/test_gomod_packages.py
@@ -69,33 +69,6 @@ def test_gomod_vendor_check_fail(env_name, test_env):
     )
 
 
-def test_gomod_workspace_check(test_env):
-    """
-    Validate failing of gomod requests that contain workspaces.
-
-    Checks:
-    * The request fails with expected error message
-    """
-    env_data = utils.load_test_data("gomod_packages.yaml")["with_workspace"]
-    client = utils.Client(test_env["api_url"], test_env["api_auth_type"], test_env.get("timeout"))
-    initial_response = client.create_new_request(
-        payload={
-            "repo": env_data["repo"],
-            "ref": env_data["ref"],
-            "pkg_managers": env_data["pkg_managers"],
-        },
-    )
-    completed_response = client.wait_for_complete_request(initial_response)
-    assert completed_response.status == 200
-    assert completed_response.data["state"] == "failed"
-    error_msg = "Go workspaces are not supported by Cachito."
-
-    assert error_msg in completed_response.data["state_reason"], (
-        f"#{completed_response.id}: Request failed correctly, but with unexpected message: "
-        f"{completed_response.data['state_reason']}. Expected message was: {error_msg}"
-    )
-
-
 def test_gomod_with_local_replacements_in_parent_dir_missing(test_env):
     """
     Test that a gomod local replacement from a parent directory includes the parent module.