From 96301b1074e04350c87de2f0706f51fd114334dd Mon Sep 17 00:00:00 2001 From: Tobias Nilsson Date: Wed, 17 Jul 2024 01:14:16 +0200 Subject: [PATCH] feat: Infer file dependencies via jsconfig --- .../javascript/dependency_inference/rules.py | 53 +++++++++++---- .../dependency_inference/rules_test.py | 31 ++++++++- .../pants/backend/typescript/tsconfig.py | 66 +++++++++++++++---- .../pants/backend/typescript/tsconfig_test.py | 10 +-- .../pants/engine/internals/native_engine.pyi | 5 +- 5 files changed, 130 insertions(+), 35 deletions(-) diff --git a/src/python/pants/backend/javascript/dependency_inference/rules.py b/src/python/pants/backend/javascript/dependency_inference/rules.py index e455e077e430..98feff22be6f 100644 --- a/src/python/pants/backend/javascript/dependency_inference/rules.py +++ b/src/python/pants/backend/javascript/dependency_inference/rules.py @@ -17,16 +17,20 @@ PackageJsonEntryPoints, PackageJsonImports, PackageJsonSourceField, + find_owning_package, + subpath_imports_for_source, ) from pants.backend.javascript.subsystems.nodejs_infer import NodeJSInfer from pants.backend.javascript.target_types import JSDependenciesField, JSSourceField +from pants.backend.typescript import tsconfig +from pants.backend.typescript.tsconfig import ParentTSConfigRequest, TSConfig, find_parent_ts_config from pants.build_graph.address import Address from pants.engine.addresses import Addresses from pants.engine.internals.graph import Owners, OwnersRequest from pants.engine.internals.native_dep_inference import NativeParsedJavascriptDependencies from pants.engine.internals.native_engine import InferenceMetadata, NativeDependenciesRequest -from pants.engine.internals.selectors import Get -from pants.engine.rules import Rule, collect_rules, rule +from pants.engine.internals.selectors import Get, concurrently +from pants.engine.rules import Rule, collect_rules, implicitly, rule from pants.engine.target import ( FieldSet, HydratedSources, @@ -101,22 +105,43 @@ async def map_candidate_node_packages( ) +@dataclass(frozen=True) +class InferenceMetadataRequest: + imports: PackageJsonImports + config: TSConfig | None + + @rule -async def prepare_inference_metadata(imports: PackageJsonImports) -> InferenceMetadata: +async def prepare_inference_metadata(req: InferenceMetadataRequest) -> InferenceMetadata: return InferenceMetadata.javascript( - imports.root_dir, - {pattern: list(replacements) for pattern, replacements in imports.imports.items()}, - None, - {} + req.imports.root_dir, + dict(req.imports.imports), + req.config.resolution_root_dir if req.config else None, + dict(req.config.paths or {}) if req.config else {}, ) -async def _prepare_inference_metadata(address: Address) -> InferenceMetadata: - owning_pkg = await Get(OwningNodePackage, OwningNodePackageRequest(address)) +async def _prepare_inference_metadata(address: Address, file_path: str) -> InferenceMetadata: + owning_pkg, maybe_config = await concurrently( + find_owning_package(OwningNodePackageRequest(address)), + find_parent_ts_config(ParentTSConfigRequest(file_path, "jsconfig.json"), **implicitly()), + ) if not owning_pkg.target: - return InferenceMetadata.javascript(address.spec_path, {}, None, {}) - return await Get( - InferenceMetadata, PackageJsonSourceField, owning_pkg.target[PackageJsonSourceField] + return InferenceMetadata.javascript( + ( + os.path.dirname(maybe_config.ts_config.path) + if maybe_config.ts_config + else address.spec_path + ), + {}, + maybe_config.ts_config.resolution_root_dir if maybe_config.ts_config else None, + dict(maybe_config.ts_config.paths or {}) if maybe_config.ts_config else {}, + ) + return await prepare_inference_metadata( + InferenceMetadataRequest( + await subpath_imports_for_source(owning_pkg.target[PackageJsonSourceField]), + maybe_config.ts_config, + ) ) @@ -132,13 +157,12 @@ async def infer_js_source_dependencies( sources = await Get( HydratedSources, HydrateSourcesRequest(source, for_sources_types=[JSSourceField]) ) - metadata = await _prepare_inference_metadata(request.field_set.address) + metadata = await _prepare_inference_metadata(request.field_set.address, source.file_path) import_strings = await Get( NativeParsedJavascriptDependencies, NativeDependenciesRequest(sources.snapshot.digest, metadata), ) - owners = await Get(Owners, OwnersRequest(tuple(import_strings.file_imports))) owning_targets = await Get(Targets, Addresses(owners)) @@ -167,6 +191,7 @@ def rules() -> Iterable[Rule | UnionRule]: return [ *collect_rules(), *package_json.rules(), + *tsconfig.rules(), UnionRule(InferDependenciesRequest, InferNodePackageDependenciesRequest), UnionRule(InferDependenciesRequest, InferJSDependenciesRequest), ] diff --git a/src/python/pants/backend/javascript/dependency_inference/rules_test.py b/src/python/pants/backend/javascript/dependency_inference/rules_test.py index b210e9e19735..055bd6cf1b97 100644 --- a/src/python/pants/backend/javascript/dependency_inference/rules_test.py +++ b/src/python/pants/backend/javascript/dependency_inference/rules_test.py @@ -29,7 +29,6 @@ def rule_runner() -> RuleRunner: rule_runner = RuleRunner( rules=[ - *package_json.rules(), *dependency_inference_rules(), QueryRule(AllPackageJson, ()), QueryRule(Owners, (OwnersRequest,)), @@ -130,6 +129,36 @@ def test_infers_commonjs_js_dependencies_from_ancestor_files(rule_runner: RuleRu assert set(addresses) == {Address("src/js", relative_file_path="xes.cjs")} +def test_infers_js_dependencies_via_config(rule_runner: RuleRunner) -> None: + rule_runner.write_files( + { + "root/project/BUILD": "package_json()", + "root/project/package.json": given_package("ham", "0.0.1", main="./src/index.js"), + "root/project/jsconfig.json": json.dumps( + {"compilerOptions": {"paths": {"*": ["./src/*"]}}} + ), + "root/project/src/BUILD": "javascript_sources()", + "root/project/src/index.js": dedent( + """\ + import button from "components/button.js"; + """ + ), + "root/project/src/components/BUILD": "javascript_sources()", + "root/project/src/components/button.js": "", + } + ) + + index_tgt = rule_runner.get_target(Address("root/project/src", relative_file_path="index.js")) + addresses = rule_runner.request( + InferredDependencies, + [InferJSDependenciesRequest(JSSourceInferenceFieldSet.create(index_tgt))], + ).include + + assert set(addresses) == { + Address("root/project/src/components", relative_file_path="button.js") + } + + def test_infers_main_package_json_field_js_source_dependency(rule_runner: RuleRunner) -> None: rule_runner.write_files( { diff --git a/src/python/pants/backend/typescript/tsconfig.py b/src/python/pants/backend/typescript/tsconfig.py index 42cf2bd66adc..4dee47eef171 100644 --- a/src/python/pants/backend/typescript/tsconfig.py +++ b/src/python/pants/backend/typescript/tsconfig.py @@ -5,17 +5,16 @@ import json import os from dataclasses import dataclass -from typing import Iterable +from pathlib import PurePath +from typing import Iterable, Literal from pants.engine.collection import Collection from pants.engine.fs import DigestContents, FileContent, PathGlobs -from pants.engine.internals.selectors import concurrently, Get +from pants.engine.internals.selectors import Get, concurrently from pants.engine.intrinsics import directory_digest_to_digest_contents, path_globs_to_digest from pants.engine.rules import Rule, collect_rules, rule from pants.util.frozendict import FrozenDict -_CONFIG = "tsconfig.json" # should be configurable - @dataclass(frozen=True) class TSConfig: @@ -34,11 +33,16 @@ def parse_from_content(cls, content: FileContent) -> TSConfig: return TSConfig( content.path, module_resolution=compiler_options.get("moduleResolution"), - paths=compiler_options.get("path"), + paths=compiler_options.get("paths"), base_url=compiler_options.get("baseUrl"), extends=compiler_options.get("extends"), ) + @property + def resolution_root_dir(self) -> str: + directory = os.path.dirname(self.path) + return os.path.join(directory, self.base_url) if self.base_url else directory + class AllTSConfigs(Collection[TSConfig]): pass @@ -48,25 +52,29 @@ class AllTSConfigs(Collection[TSConfig]): class ParseTSConfigRequest: content: FileContent others: DigestContents + target_file: Literal["tsconfig.json", "jsconfig.json"] async def _read_parent_config( - child_path: str, extends_path: str, others: DigestContents + child_path: str, + extends_path: str, + others: DigestContents, + target_file: Literal["tsconfig.json", "jsconfig.json"], ) -> TSConfig: - if child_path.endswith(_CONFIG): + if child_path.endswith(".json"): relative = os.path.dirname(child_path) else: relative = child_path relative = os.path.normpath(os.path.join(relative, extends_path)) - if not relative.endswith(_CONFIG): - relative = os.path.join(relative, _CONFIG) + if not extends_path.endswith(".json"): + relative = os.path.join(relative, target_file) parent = next((other for other in others if other.path == relative), None) if not parent: raise ValueError( f"pants could not locate {child_path}'s parent at {relative}. Found: {[other.path for other in others]}." ) return await Get( # Must be a Get until https://github.com/pantsbuild/pants/pull/21174 lands - TSConfig, ParseTSConfigRequest(parent, others) + TSConfig, ParseTSConfigRequest(parent, others, target_file) ) @@ -75,7 +83,7 @@ async def parse_extended_ts_config(request: ParseTSConfigRequest) -> TSConfig: ts_config = TSConfig.parse_from_content(request.content) if ts_config.extends: extended_parent = await _read_parent_config( - ts_config.path, ts_config.extends, request.others + ts_config.path, ts_config.extends, request.others, request.target_file ) else: extended_parent = TSConfig(ts_config.path) @@ -88,18 +96,48 @@ async def parse_extended_ts_config(request: ParseTSConfigRequest) -> TSConfig: ) +@dataclass(frozen=True) +class TSConfigsRequest: + target_file: Literal["tsconfig.json", "jsconfig.json"] + + @rule -async def construct_effective_ts_configs() -> AllTSConfigs: - all_files = await path_globs_to_digest(PathGlobs([f"**/{_CONFIG}"])) # should be configurable +async def construct_effective_ts_configs(req: TSConfigsRequest) -> AllTSConfigs: + all_files = await path_globs_to_digest( + PathGlobs([f"**/{req.target_file}"]) + ) # should be configurable digest_contents = await directory_digest_to_digest_contents(all_files) return AllTSConfigs( await concurrently( - parse_extended_ts_config(ParseTSConfigRequest(digest_content, digest_contents)) + parse_extended_ts_config( + ParseTSConfigRequest(digest_content, digest_contents, req.target_file) + ) for digest_content in digest_contents ) ) +@dataclass(frozen=True) +class ClosestTSConfig: + ts_config: TSConfig | None + + +@dataclass(frozen=True) +class ParentTSConfigRequest: + file: str + target_file: Literal["tsconfig.json", "jsconfig.json"] + + +@rule(desc="Finding parent tsconfig.json") +async def find_parent_ts_config(req: ParentTSConfigRequest) -> ClosestTSConfig: + all_configs = await construct_effective_ts_configs(TSConfigsRequest(req.target_file)) + configs_by_longest_path = sorted(all_configs, key=lambda config: config.path, reverse=True) + for config in configs_by_longest_path: + if PurePath(req.file).is_relative_to(os.path.dirname(config.path)): + return ClosestTSConfig(config) + return ClosestTSConfig(None) + + def rules() -> Iterable[Rule]: return collect_rules() diff --git a/src/python/pants/backend/typescript/tsconfig_test.py b/src/python/pants/backend/typescript/tsconfig_test.py index 0662808dec13..480a0704ed6f 100644 --- a/src/python/pants/backend/typescript/tsconfig_test.py +++ b/src/python/pants/backend/typescript/tsconfig_test.py @@ -8,7 +8,7 @@ from pants.backend.typescript import tsconfig from pants.backend.typescript.target_types import TypeScriptSourceTarget -from pants.backend.typescript.tsconfig import AllTSConfigs, TSConfig +from pants.backend.typescript.tsconfig import AllTSConfigs, TSConfig, TSConfigsRequest from pants.core.target_types import TargetGeneratorSourcesHelperTarget from pants.engine.rules import QueryRule from pants.testutil.rule_runner import RuleRunner @@ -17,7 +17,7 @@ @pytest.fixture def rule_runner() -> RuleRunner: return RuleRunner( - rules=[*tsconfig.rules(), QueryRule(AllTSConfigs, ())], + rules=[*tsconfig.rules(), QueryRule(AllTSConfigs, (TSConfigsRequest,))], target_types=[TypeScriptSourceTarget, TargetGeneratorSourcesHelperTarget], ) @@ -30,7 +30,7 @@ def test_parses_tsconfig(rule_runner: RuleRunner) -> None: "project/tsconfig.json": "{}", } ) - [ts_config] = rule_runner.request(AllTSConfigs, []) + [ts_config] = rule_runner.request(AllTSConfigs, [TSConfigsRequest("tsconfig.json")]) assert ts_config == TSConfig("project/tsconfig.json") @@ -43,7 +43,7 @@ def test_parses_extended_tsconfig(rule_runner: RuleRunner) -> None: "project/lib/tsconfig.json": json.dumps({"compilerOptions": {"extends": ".."}}), } ) - configs = rule_runner.request(AllTSConfigs, []) + configs = rule_runner.request(AllTSConfigs, [TSConfigsRequest("tsconfig.json")]) assert set(configs) == { TSConfig("project/tsconfig.json", base_url="./"), TSConfig("project/lib/tsconfig.json", base_url="./", extends=".."), @@ -59,7 +59,7 @@ def test_parses_extended_tsconfig_with_overrides(rule_runner: RuleRunner) -> Non "project/lib/tsconfig.json": json.dumps({"compilerOptions": {"baseUrl": "./src", "extends": ".."}}), } ) - configs = rule_runner.request(AllTSConfigs, []) + configs = rule_runner.request(AllTSConfigs, [TSConfigsRequest("tsconfig.json")]) assert set(configs) == { TSConfig("project/tsconfig.json", base_url="./"), TSConfig("project/lib/tsconfig.json", base_url="./src", extends=".."), diff --git a/src/python/pants/engine/internals/native_engine.pyi b/src/python/pants/engine/internals/native_engine.pyi index ee1019940528..6d2910577d1b 100644 --- a/src/python/pants/engine/internals/native_engine.pyi +++ b/src/python/pants/engine/internals/native_engine.pyi @@ -703,7 +703,10 @@ class PyStubCAS: class InferenceMetadata: @staticmethod def javascript( - package_root: str, import_patterns: dict[str, list[str]], config_root: str | None, paths: dict[str, list[str]] + package_root: str, + import_patterns: dict[str, Sequence[str]], + config_root: str | None, + paths: dict[str, Sequence[str]], ) -> InferenceMetadata: ... def __eq__(self, other: InferenceMetadata | Any) -> bool: ... def __hash__(self) -> int: ...