Plumb reproducible build env vars more thoroughly.

This helps reduce cache sizes where sdists are built into wheels as well
as expanding the set of sdists Pex can lock reproducibly.

CHANGES.md

@@ -1,5 +1,20 @@
 # Release Notes
 
+## 2.20.3
+
+This release fixes both PEX building and lock creation via
+`pex3 lock {create,sync}` to be reproducible in more cases. Previously,
+if a requirement only available in source form (an sdist, a local
+project or a VCS requirement) had a build that was not reproducible due
+to either file timestamps (where the `SOURCE_DATE_EPOCH` standard was
+respected) or random iteration order (e.g.: the `setup.py` used sets in
+certain in-opportune ways), Pex's outputs would mirror the problematic
+requirement's non-reproducibility. Now Pex plumbs a fixed
+`SOURCE_DATE_EPOCH` and `PYTHONHASHSEED` to all places sources are
+built.
+
+* Plumb reproducible build env vars more thoroughly. (#2554)
+
 ## 2.20.2
 
 This release fixes an old bug handling certain sdist zips under

pex/bin/pex.py

@@ -1254,7 +1254,9 @@ def main(args=None):
     try:
         with global_environment(options) as env:
             try:
-                resolver_configuration = resolver_options.configure(options)
+                resolver_configuration = resolver_options.configure(
+                    options, use_system_time=options.use_system_time
+                )
             except resolver_options.InvalidConfigurationError as e:
                 die(str(e))
 

pex/build_system/pep_517.py

@@ -75,6 +75,7 @@ def _default_build_system(
                     resolved=resolved_dists,
                     build_backend=DEFAULT_BUILD_BACKEND,
                     backend_path=(),
+                    use_system_time=resolver.use_system_time(),
                     **extra_env
                 )
             )

pex/build_system/pep_518.py

@@ -7,6 +7,7 @@
 import subprocess
 
 from pex.build_system import DEFAULT_BUILD_BACKEND
+from pex.common import REPRODUCIBLE_BUILDS_ENV
 from pex.dist_metadata import Distribution
 from pex.interpreter import PythonInterpreter
 from pex.pex import PEX
@@ -80,13 +81,16 @@ def create(
         build_backend,  # type: str
         backend_path,  # type: Tuple[str, ...]
         extra_requirements=None,  # type: Optional[Iterable[str]]
+        use_system_time=False,  # type: bool
         **extra_env  # type: str
     ):
         # type: (...) -> Union[BuildSystem, Error]
         pex_builder = PEXBuilder()
         pex_builder.info.venv = True
         pex_builder.info.venv_site_packages_copies = True
         pex_builder.info.venv_bin_path = BinPath.PREPEND
+        # Allow REPRODUCIBLE_BUILDS_ENV PYTHONHASHSEED env var to take effect.
+        pex_builder.info.venv_hermetic_scripts = False
         for req in requires:
             pex_builder.add_requirement(req)
         for dist in resolved:
@@ -144,6 +148,8 @@ def create(
                 env.update(extra_env)
             if backend_path:
                 env.update(PEX_EXTRA_SYS_PATH=os.pathsep.join(backend_path))
+            if not use_system_time:
+                env.update(REPRODUCIBLE_BUILDS_ENV)
             return cls(
                 venv_pex=venv_pex, build_backend=build_backend, requires=tuple(requires), env=env
             )
@@ -190,4 +196,5 @@ def load_build_system(
             build_backend=build_system_table.build_backend,
             backend_path=build_system_table.backend_path,
             extra_requirements=extra_requirements,
+            use_system_time=resolver.use_system_time(),
         )

pex/cache/dirs.py

@@ -136,7 +136,7 @@ def iter_transitive_dependents(self):
 
     PIP = Value(
         "pip",
-        version=0,
+        version=1,
         name="Pip Versions",
         description="Isolated Pip caches and Pip PEXes Pex uses to resolve distributions.",
     )

pex/cli/commands/lock.py

@@ -799,7 +799,10 @@ def _resolve_targets(
         # type: (...) -> Union[Targets, Error]
 
         target_config = target_configuration or target_options.configure(
-            self.options, pip_configuration=resolver_options.create_pip_configuration(self.options)
+            self.options,
+            pip_configuration=resolver_options.create_pip_configuration(
+                self.options, use_system_time=False
+            ),
         )
         if style is not LockStyle.UNIVERSAL:
             return target_config.resolve_targets()
@@ -867,7 +870,9 @@ def _gather_requirements(
     def _create(self):
         # type: () -> Result
 
-        pip_configuration = resolver_options.create_pip_configuration(self.options)
+        pip_configuration = resolver_options.create_pip_configuration(
+            self.options, use_system_time=False
+        )
         target_configuration = target_options.configure(
             self.options, pip_configuration=pip_configuration
         )
@@ -963,7 +968,9 @@ def _export(self, requirement_configuration=RequirementConfiguration()):
             )
 
         lockfile_path, lock_file = self._load_lockfile()
-        pip_configuration = resolver_options.create_pip_configuration(self.options)
+        pip_configuration = resolver_options.create_pip_configuration(
+            self.options, use_system_time=False
+        )
         targets = target_options.configure(
             self.options, pip_configuration=pip_configuration
         ).resolve_targets()
@@ -1091,7 +1098,9 @@ def _create_lock_update_request(
     ):
         # type: (...) -> Union[LockUpdateRequest, Error]
 
-        pip_configuration = resolver_options.create_pip_configuration(self.options)
+        pip_configuration = resolver_options.create_pip_configuration(
+            self.options, use_system_time=False
+        )
         lock_updater = LockUpdater.create(
             lock_file=lock_file,
             repos_configuration=pip_configuration.repos_configuration,
@@ -1506,7 +1515,8 @@ def _sync(self):
         # type: () -> Result
 
         resolver_configuration = cast(
-            LockRepositoryConfiguration, resolver_options.configure(self.options)
+            LockRepositoryConfiguration,
+            resolver_options.configure(self.options, use_system_time=False),
         )
         production_assert(isinstance(resolver_configuration, LockRepositoryConfiguration))
         pip_configuration = resolver_configuration.pip_configuration

pex/common.py

@@ -54,6 +54,15 @@
 _UNIX_EPOCH = datetime(year=1970, month=1, day=1, hour=0, minute=0, second=0, tzinfo=None)
 DETERMINISTIC_DATETIME_TIMESTAMP = (DETERMINISTIC_DATETIME - _UNIX_EPOCH).total_seconds()
 
+# N.B.: The `SOURCE_DATE_EPOCH` env var is semi-standard magic for controlling
+# build tools. Wheel, for example, has supported this since 2016.
+# See:
+# + https://reproducible-builds.org/docs/source-date-epoch/
+# + https://github.com/pypa/wheel/blob/1b879e53fed1f179897ed47e55a68bc51df188db/wheel/archive.py#L36-L39
+REPRODUCIBLE_BUILDS_ENV = dict(
+    PYTHONHASHSEED="0", SOURCE_DATE_EPOCH=str(int(DETERMINISTIC_DATETIME_TIMESTAMP))
+)
+
 
 def is_pyc_dir(dir_path):
     # type: (Text) -> bool

pex/pip/installation.py

@@ -11,7 +11,7 @@
 from pex import pex_warnings, third_party
 from pex.atomic_directory import atomic_directory
 from pex.cache.dirs import CacheDir
-from pex.common import pluralize, safe_mkdtemp
+from pex.common import REPRODUCIBLE_BUILDS_ENV, pluralize, safe_mkdtemp
 from pex.dist_metadata import Requirement
 from pex.interpreter import PythonInterpreter
 from pex.orderedset import OrderedSet
@@ -42,6 +42,7 @@ def _pip_installation(
     iter_distribution_locations,  # type: Callable[[], Iterator[str]]
     fingerprint,  # type: str
     interpreter=None,  # type: Optional[PythonInterpreter]
+    use_system_time=False,  # type: bool
 ):
     # type: (...) -> Pip
     pip_root = CacheDir.PIP.path(str(version))
@@ -54,6 +55,8 @@ def _pip_installation(
 
             isolated_pip_builder = PEXBuilder(path=chroot.work_dir)
             isolated_pip_builder.info.venv = True
+            # Allow REPRODUCIBLE_BUILDS_ENV PYTHONHASHSEED env var to take effect if needed.
+            isolated_pip_builder.info.venv_hermetic_scripts = False
             for dist_location in iter_distribution_locations():
                 isolated_pip_builder.add_dist_location(dist=dist_location)
             with named_temporary_file(prefix="", suffix=".py", mode="w") as fp:
@@ -78,7 +81,11 @@ def _pip_installation(
             isolated_pip_builder.freeze()
     pip_cache = os.path.join(pip_root, "pip_cache")
     pip_pex = ensure_venv(PEX(pip_pex_path, interpreter=pip_interpreter))
-    pip_venv = PipVenv(venv_dir=pip_pex.venv_dir, execute_args=tuple(pip_pex.execute_args()))
+    pip_venv = PipVenv(
+        venv_dir=pip_pex.venv_dir,
+        execute_env=REPRODUCIBLE_BUILDS_ENV if not use_system_time else {},
+        execute_args=tuple(pip_pex.execute_args()),
+    )
     return Pip(pip=pip_venv, version=version, pip_cache=pip_cache)
 
 
@@ -98,6 +105,7 @@ def _vendored_installation(
     interpreter=None,  # type: Optional[PythonInterpreter]
     resolver=None,  # type: Optional[Resolver]
     extra_requirements=(),  # type: Tuple[Requirement, ...]
+    use_system_time=False,  # type: bool
 ):
     # type: (...) -> Pip
 
@@ -111,6 +119,7 @@ def expose_vendored():
             iter_distribution_locations=expose_vendored,
             interpreter=interpreter,
             fingerprint=_fingerprint(extra_requirements),
+            use_system_time=use_system_time,
         )
 
     if not resolver:
@@ -171,6 +180,7 @@ def iter_distribution_locations():
         iter_distribution_locations=iter_distribution_locations,
         interpreter=interpreter,
         fingerprint=_fingerprint(extra_requirements),
+        use_system_time=use_system_time,
     )
 
 
@@ -204,6 +214,7 @@ def _resolved_installation(
     resolver=None,  # type: Optional[Resolver]
     interpreter=None,  # type: Optional[PythonInterpreter]
     extra_requirements=(),  # type: Tuple[Requirement, ...]
+    use_system_time=False,  # type: bool
 ):
     # type: (...) -> Pip
     targets = Targets.from_target(LocalInterpreter.create(interpreter))
@@ -222,6 +233,7 @@ def _resolved_installation(
             iter_distribution_locations=_bootstrap_pip(version, interpreter=interpreter),
             interpreter=interpreter,
             fingerprint=_fingerprint(extra_requirements),
+            use_system_time=use_system_time,
         )
 
     requirements_by_project_name = OrderedDict(
@@ -269,6 +281,7 @@ def resolve_distribution_locations():
         iter_distribution_locations=resolve_distribution_locations,
         interpreter=interpreter,
         fingerprint=_fingerprint(extra_requirements),
+        use_system_time=use_system_time,
     )
 
 
@@ -277,6 +290,7 @@ class PipInstallation(object):
     interpreter = attr.ib()  # type: PythonInterpreter
     version = attr.ib()  # type: PipVersionValue
     extra_requirements = attr.ib()  # type: Tuple[Requirement, ...]
+    use_system_time = attr.ib()  # type: bool
 
     def check_python_applies(self):
         # type: () -> None
@@ -396,6 +410,7 @@ def get_pip(
         interpreter=interpreter or PythonInterpreter.get(),
         version=calculated_version,
         extra_requirements=extra_requirements,
+        use_system_time=resolver.use_system_time() if resolver else False,
     )
     pip = _PIP.get(installation)
     if pip is None:
@@ -405,13 +420,15 @@ def get_pip(
                 interpreter=interpreter,
                 resolver=resolver,
                 extra_requirements=installation.extra_requirements,
+                use_system_time=installation.use_system_time,
             )
         else:
             pip = _resolved_installation(
                 version=installation.version,
                 resolver=resolver,
                 interpreter=interpreter,
                 extra_requirements=installation.extra_requirements,
+                use_system_time=installation.use_system_time,
             )
         _PIP[installation] = pip
     return pip

pex/pip/tool.py

@@ -272,6 +272,7 @@ def analyze(self, line):
 @attr.s(frozen=True)
 class PipVenv(object):
     venv_dir = attr.ib()  # type: str
+    execute_env = attr.ib()  # type: Mapping[str, str]
     _execute_args = attr.ib()  # type: Tuple[str, ...]
 
     def execute_args(self, *args):
@@ -431,6 +432,7 @@ def _spawn_pip_isolated(
                 popen_kwargs["stdout"] = sys.stderr.fileno()
             popen_kwargs.update(stderr=subprocess.PIPE)
 
+            env.update(self._pip.execute_env)
             args = self._pip.execute_args(*command)
 
             rendered_env = " ".join(

pex/resolve/configured_resolver.py

@@ -45,8 +45,13 @@ def default(cls):
     pip_configuration = attr.ib()  # type: PipConfiguration
 
     def is_default_repos(self):
+        # type: () -> bool
         return self.pip_configuration.repos_configuration == _DEFAULT_REPOS
 
+    def use_system_time(self):
+        # type: () -> bool
+        return self.pip_configuration.build_configuration.use_system_time
+
     def resolve_lock(
         self,
         lock,  # type: Lockfile

pex/resolve/lockfile/json_codec.py

@@ -231,6 +231,8 @@ def parse_version_specifier(
         for index, constraint in enumerate(get("constraints", list))
     ]
 
+    use_system_time = get("use_system_time", bool, optional=True)
+
     excluded = [
         parse_requirement(req, path=".excluded[{index}]".format(index=index))
         for index, req in enumerate(get("excluded", list, optional=True) or ())
@@ -352,6 +354,10 @@ def assemble_tag(
             prefer_older_binary=get("prefer_older_binary", bool),
             use_pep517=get("use_pep517", bool, optional=True),
             build_isolation=get("build_isolation", bool),
+            # N.B.: Although locks are now always generated under SOURCE_DATE_EPOCH=fixed and
+            # PYTHONHASHSEED=0 (aka: `use_system_time=False`), that did not use to be the case. In
+            # those old locks there was no "use_system_time" field.
+            use_system_time=use_system_time if use_system_time is not None else True,
         ),
         transitive=get("transitive", bool),
         excluded=excluded,
@@ -399,6 +405,7 @@ def as_json_data(
         "prefer_older_binary": lockfile.prefer_older_binary,
         "use_pep517": lockfile.use_pep517,
         "build_isolation": lockfile.build_isolation,
+        "use_system_time": lockfile.use_system_time,
         "transitive": lockfile.transitive,
         "excluded": [str(exclude) for exclude in lockfile.excluded],
         "overridden": [str(override) for override in lockfile.overridden],

pex/resolve/lockfile/model.py

@@ -106,6 +106,7 @@ def extract_requirement(req):
             prefer_older_binary=build_configuration.prefer_older_binary,
             use_pep517=build_configuration.use_pep517,
             build_isolation=build_configuration.build_isolation,
+            use_system_time=build_configuration.use_system_time,
             transitive=transitive,
             excluded=SortedTuple(excluded),
             overridden=SortedTuple(overridden),
@@ -130,6 +131,7 @@ def extract_requirement(req):
     prefer_older_binary = attr.ib()  # type: bool
     use_pep517 = attr.ib()  # type: Optional[bool]
     build_isolation = attr.ib()  # type: bool
+    use_system_time = attr.ib()  # type: bool
     transitive = attr.ib()  # type: bool
     excluded = attr.ib()  # type: SortedTuple[Requirement]
     overridden = attr.ib()  # type: SortedTuple[Requirement]
@@ -147,6 +149,7 @@ def build_configuration(self):
             prefer_older_binary=self.prefer_older_binary,
             use_pep517=self.use_pep517,
             build_isolation=self.build_isolation,
+            use_system_time=self.use_system_time,
         )
 
     def dependency_configuration(self):

pex/resolve/resolver_configuration.py

@@ -98,6 +98,7 @@ def create(
         prefer_older_binary=False,  # type: bool
         use_pep517=None,  # type: Optional[bool]
         build_isolation=True,  # type: bool
+        use_system_time=False,  # type: bool
     ):
         # type: (...) -> BuildConfiguration
         return cls(
@@ -108,6 +109,7 @@ def create(
             prefer_older_binary=prefer_older_binary,
             use_pep517=use_pep517,
             build_isolation=build_isolation,
+            use_system_time=use_system_time,
         )
 
     allow_builds = attr.ib(default=True)  # type: bool
@@ -117,6 +119,7 @@ def create(
     prefer_older_binary = attr.ib(default=False)  # type: bool
     use_pep517 = attr.ib(default=None)  # type: Optional[bool]
     build_isolation = attr.ib(default=True)  # type: bool
+    use_system_time = attr.ib(default=False)  # type: bool
 
     def __attrs_post_init__(self):
         # type: () -> None