Filter out incompatible dists

.github/workflows/ci.yaml

@@ -48,6 +48,10 @@ jobs:
     name: "cargo test | ${{ matrix.os }}"
     steps:
       - uses: actions/checkout@v4
+      # TODO(konstin): Mock the venv in the installer test so we don't need this anymore
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
       - name: "Install Python"
         uses: actions/setup-python@v4
         with:

.gitignore

@@ -13,3 +13,6 @@ target/
 
 # Use e.g. `--cache-dir cache-docker` to keep a cache across container invocations
 cache-*
+
+# python tmp files
+__pycache__

crates/pep440-rs/src/version_specifier.rs

@@ -540,6 +540,9 @@ impl Display for VersionSpecifier {
 /// ```
 pub fn parse_version_specifiers(spec: &str) -> Result<Vec<VersionSpecifier>, Pep440Error> {
     let mut version_ranges = Vec::new();
+    if spec.is_empty() {
+        return Ok(version_ranges);
+    }
     let mut start: usize = 0;
     let separator = ",";
     for version_range_spec in spec.split(separator) {
@@ -1301,4 +1304,11 @@ mod test {
             ">=3.7, <4.0, !=3.9.0"
         );
     }
+
+    /// These occur in the simple api, e.g.
+    /// <https://pypi.org/simple/geopandas/?format=application/vnd.pypi.simple.v1+json>
+    #[test]
+    fn test_version_specifiers_empty() {
+        assert_eq!(VersionSpecifiers::from_str("").unwrap().to_string(), "");
+    }
 }

crates/puffin-cli/src/commands/pip_compile.rs

@@ -125,6 +125,11 @@ pub(crate) async fn pip_compile(
         || Cow::Borrowed(venv.interpreter_info().markers()),
         |python_version| Cow::Owned(python_version.markers(venv.interpreter_info().markers())),
     );
+    // Inject the fake python version if necessary
+    let interpreter_info = venv
+        .interpreter_info()
+        .clone()
+        .patch_markers(markers.clone().into_owned());
 
     // Instantiate a client.
     let client = {
@@ -143,7 +148,7 @@ pub(crate) async fn pip_compile(
     let build_dispatch = BuildDispatch::new(
         client.clone(),
         cache.to_path_buf(),
-        venv.interpreter_info().clone(),
+        interpreter_info,
         fs::canonicalize(venv.python_executable())?,
         no_build,
     );

crates/puffin-cli/src/commands/pip_sync.rs

@@ -128,8 +128,9 @@ pub(crate) async fn sync_requirements(
     } else {
         let start = std::time::Instant::now();
 
-        let wheel_finder = puffin_resolver::DistFinder::new(&tags, &client)
-            .with_reporter(FinderReporter::from(printer).with_length(remote.len() as u64));
+        let wheel_finder =
+            puffin_resolver::DistFinder::new(&tags, &client, venv.interpreter_info())
+                .with_reporter(FinderReporter::from(printer).with_length(remote.len() as u64));
         let resolution = wheel_finder.resolve(&remote).await?;
 
         let s = if resolution.len() == 1 { "" } else { "s" };

crates/puffin-cli/tests/pip_compile.rs

@@ -630,6 +630,57 @@ fn compile_python_37() -> Result<()> {
     Ok(())
 }
 
+/// Test that we select the last 3.8 compatible numpy version instead of trying to compile an
+/// incompatible sdist <https://github.com/astral-sh/puffin/issues/388>
+#[test]
+fn compile_numpy_py38() -> Result<()> {
+    let temp_dir = assert_fs::TempDir::new()?;
+    let cache_dir = assert_fs::TempDir::new()?;
+    let venv = temp_dir.child(".venv");
+
+    Command::new(get_cargo_bin(BIN_NAME))
+        .arg("venv")
+        .arg(venv.as_os_str())
+        .arg("--cache-dir")
+        .arg(cache_dir.path())
+        .current_dir(&temp_dir)
+        .assert()
+        .success();
+    venv.assert(predicates::path::is_dir());
+
+    let requirements_in = temp_dir.child("requirements.in");
+    requirements_in.touch()?;
+    requirements_in.write_str("numpy")?;
+
+    insta::with_settings!({
+            filters => INSTA_FILTERS.to_vec()
+        }, {
+            assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME))
+                .arg("pip-compile")
+                .arg("requirements.in")
+                .arg("--python-version")
+                .arg("py38")
+                .arg("--cache-dir")
+                .arg(cache_dir.path())
+    // Check that we select the wheel and not the sdist
+                .arg("--no-build")
+                .env("VIRTUAL_ENV", venv.as_os_str())
+                .current_dir(&temp_dir), @r###"
+        success: true
+        exit_code: 0
+        ----- stdout -----
+        # This file was autogenerated by Puffin v0.0.1 via the following command:
+        #    [BIN_PATH] pip-compile requirements.in --python-version py38 --cache-dir [CACHE_DIR]
+        numpy==1.24.4
+
+        ----- stderr -----
+        Resolved 1 package in [TIME]
+        "###);
+        });
+
+    Ok(())
+}
+
 /// Resolve a specific Flask wheel via a URL dependency.
 #[test]
 fn compile_wheel_url_dependency() -> Result<()> {

crates/puffin-cli/tests/pip_sync.rs

@@ -2,7 +2,7 @@
 
 use std::process::Command;
 
-use anyhow::Result;
+use anyhow::{Context, Result};
 use assert_cmd::prelude::*;
 use assert_fs::prelude::*;
 use insta_cmd::_macro_support::insta;
@@ -928,3 +928,50 @@ fn install_version_then_install_url() -> Result<()> {
 
     Ok(())
 }
+
+/// Test that we select the last 3.8 compatible numpy version instead of trying to compile an
+/// incompatible sdist <https://github.com/astral-sh/puffin/issues/388>
+#[test]
+fn install_numpy_py38() -> Result<()> {
+    let temp_dir = assert_fs::TempDir::new()?;
+    let cache_dir = assert_fs::TempDir::new()?;
+    let venv = temp_dir.child(".venv");
+
+    Command::new(get_cargo_bin(BIN_NAME))
+        .arg("venv")
+        .arg(venv.as_os_str())
+        .arg("--python")
+        // TODO(konstin): Mock the venv in the installer test so we don't need this anymore
+        .arg(which::which("python3.8").context("python3.8 must be installed")?)
+        .arg("--cache-dir")
+        .arg(cache_dir.path())
+        .current_dir(&temp_dir)
+        .assert()
+        .success();
+    venv.assert(predicates::path::is_dir());
+
+    let requirements_txt = temp_dir.child("requirements.txt");
+    requirements_txt.touch()?;
+    requirements_txt.write_str("numpy")?;
+
+    insta::with_settings!({
+        filters => INSTA_FILTERS.to_vec()
+    }, {
+        assert_cmd_snapshot!(Command::new(get_cargo_bin(BIN_NAME))
+            .arg("pip-sync")
+            .arg("requirements.txt")
+            .arg("--cache-dir")
+            .arg(cache_dir.path())
+            .env("VIRTUAL_ENV", venv.as_os_str())
+            .current_dir(&temp_dir));
+    });
+
+    Command::new(venv.join("bin").join("python"))
+        .arg("-c")
+        .arg("import numpy")
+        .current_dir(&temp_dir)
+        .assert()
+        .success();
+
+    Ok(())
+}

crates/puffin-cli/tests/snapshots/pip_compile__compile_constraints_markers.snap

@@ -8,21 +8,23 @@ info:
     - "--constraint"
     - constraints.txt
     - "--cache-dir"
-    - /var/folders/bc/qlsk3t6x7c9fhhbvvcg68k9c0000gp/T/.tmp1Ts53o
+    - /tmp/.tmpzpwtXx
   env:
-    VIRTUAL_ENV: /var/folders/bc/qlsk3t6x7c9fhhbvvcg68k9c0000gp/T/.tmp1vzBQa/.venv
+    VIRTUAL_ENV: /tmp/.tmpwo1pOb/.venv
 ---
 success: true
 exit_code: 0
 ----- stdout -----
 # This file was autogenerated by Puffin v0.0.1 via the following command:
 #    [BIN_PATH] pip-compile requirements.in --constraint constraints.txt --cache-dir [CACHE_DIR]
 anyio==4.0.0
+exceptiongroup==1.1.3
+    # via anyio
 idna==3.4
     # via anyio
 sniffio==1.3.0
     # via anyio
 
 ----- stderr -----
-Resolved 3 packages in [TIME]
+Resolved 4 packages in [TIME]
 

crates/puffin-cli/tests/snapshots/pip_compile__compile_constraints_txt.snap

@@ -8,9 +8,9 @@ info:
     - "--constraint"
     - constraints.txt
     - "--cache-dir"
-    - /var/folders/nt/6gf2v7_s3k13zq_t3944rwz40000gn/T/.tmpvjdHOb
+    - /tmp/.tmp9F1Ljq
   env:
-    VIRTUAL_ENV: /var/folders/nt/6gf2v7_s3k13zq_t3944rwz40000gn/T/.tmpSAaWi3/.venv
+    VIRTUAL_ENV: /tmp/.tmpNJjhiS/.venv
 ---
 success: true
 exit_code: 0
@@ -22,7 +22,9 @@ asgiref==3.7.2
 django==5.0b1
 sqlparse==0.4.3
     # via django
+typing-extensions==4.8.0
+    # via asgiref
 
 ----- stderr -----
-Resolved 3 packages in [TIME]
+Resolved 4 packages in [TIME]
 

crates/puffin-cli/tests/snapshots/pip_compile__compile_pyproject_toml.snap

@@ -6,9 +6,9 @@ info:
     - pip-compile
     - pyproject.toml
     - "--cache-dir"
-    - /var/folders/nt/6gf2v7_s3k13zq_t3944rwz40000gn/T/.tmpOOTFwj
+    - /tmp/.tmphqsdmA
   env:
-    VIRTUAL_ENV: /var/folders/nt/6gf2v7_s3k13zq_t3944rwz40000gn/T/.tmpU0VXyY/.venv
+    VIRTUAL_ENV: /tmp/.tmpvlroQ4/.venv
 ---
 success: true
 exit_code: 0
@@ -20,7 +20,9 @@ asgiref==3.7.2
 django==5.0b1
 sqlparse==0.4.4
     # via django
+typing-extensions==4.8.0
+    # via asgiref
 
 ----- stderr -----
-Resolved 3 packages in [TIME]
+Resolved 4 packages in [TIME]
 

crates/puffin-cli/tests/snapshots/pip_compile__compile_pyproject_toml_all_extras.snap

@@ -7,9 +7,9 @@ info:
     - pyproject.toml
     - "--all-extras"
     - "--cache-dir"
-    - /var/folders/nt/6gf2v7_s3k13zq_t3944rwz40000gn/T/.tmpFzJKRe
+    - /tmp/.tmpLq97XV
   env:
-    VIRTUAL_ENV: /var/folders/nt/6gf2v7_s3k13zq_t3944rwz40000gn/T/.tmpdadcmu/.venv
+    VIRTUAL_ENV: /tmp/.tmpj8K855/.venv
 ---
 success: true
 exit_code: 0
@@ -23,6 +23,8 @@ asgiref==3.7.2
 certifi==2023.7.22
     # via httpcore
 django==5.0b1
+exceptiongroup==1.1.3
+    # via anyio
 h11==0.14.0
     # via httpcore
 httpcore==0.18.0
@@ -34,7 +36,9 @@ sniffio==1.3.0
     #   httpcore
 sqlparse==0.4.4
     # via django
+typing-extensions==4.8.0
+    # via asgiref
 
 ----- stderr -----
-Resolved 9 packages in [TIME]
+Resolved 11 packages in [TIME]
 

crates/puffin-cli/tests/snapshots/pip_compile__compile_pyproject_toml_extra.snap

@@ -8,9 +8,9 @@ info:
     - "--extra"
     - foo
     - "--cache-dir"
-    - /var/folders/bc/qlsk3t6x7c9fhhbvvcg68k9c0000gp/T/.tmpAYEAdM
+    - /tmp/.tmpvkwe04
   env:
-    VIRTUAL_ENV: /var/folders/bc/qlsk3t6x7c9fhhbvvcg68k9c0000gp/T/.tmp1xuOcV/.venv
+    VIRTUAL_ENV: /tmp/.tmpMFDom0/.venv
 ---
 success: true
 exit_code: 0
@@ -22,7 +22,9 @@ asgiref==3.7.2
 django==5.0b1
 sqlparse==0.4.4
     # via django
+typing-extensions==4.8.0
+    # via asgiref
 
 ----- stderr -----
-Resolved 3 packages in [TIME]
+Resolved 4 packages in [TIME]
 

crates/puffin-cli/tests/snapshots/pip_compile__compile_pyproject_toml_extra_name_normalization.snap

@@ -8,9 +8,9 @@ info:
     - "--extra"
     - FRiENDlY-...-_-BARd
     - "--cache-dir"
-    - /var/folders/bc/qlsk3t6x7c9fhhbvvcg68k9c0000gp/T/.tmpFyLXQn
+    - /tmp/.tmpVf4Xxi
   env:
-    VIRTUAL_ENV: /var/folders/bc/qlsk3t6x7c9fhhbvvcg68k9c0000gp/T/.tmpxfZatv/.venv
+    VIRTUAL_ENV: /tmp/.tmpM4Yfd8/.venv
 ---
 success: true
 exit_code: 0
@@ -22,7 +22,9 @@ asgiref==3.7.2
 django==5.0b1
 sqlparse==0.4.4
     # via django
+typing-extensions==4.8.0
+    # via asgiref
 
 ----- stderr -----
-Resolved 3 packages in [TIME]
+Resolved 4 packages in [TIME]
 

crates/puffin-cli/tests/snapshots/pip_compile__compile_python_37.snap

@@ -8,38 +8,16 @@ info:
     - "--python-version"
     - py37
     - "--cache-dir"
-    - /var/folders/nt/6gf2v7_s3k13zq_t3944rwz40000gn/T/.tmpzwzUVe
+    - /tmp/.tmpHz7Dc2
   env:
-    VIRTUAL_ENV: /var/folders/nt/6gf2v7_s3k13zq_t3944rwz40000gn/T/.tmpqFv4YL/.venv
+    VIRTUAL_ENV: /tmp/.tmpoMjDJT/.venv
 ---
-success: true
-exit_code: 0
+success: false
+exit_code: 1
 ----- stdout -----
-# This file was autogenerated by Puffin v0.0.1 via the following command:
-#    [BIN_PATH] pip-compile requirements.in --python-version py37 --cache-dir [CACHE_DIR]
-black==23.10.1
-click==8.1.7
-    # via black
-importlib-metadata==6.8.0
-    # via click
-mypy-extensions==1.0.0
-    # via black
-packaging==23.2
-    # via black
-pathspec==0.11.2
-    # via black
-platformdirs==4.0.0
-    # via black
-tomli==2.0.1
-    # via black
-typing-extensions==4.8.0
-    # via
-    #   black
-    #   importlib-metadata
-    #   platformdirs
-zipp==3.17.0
-    # via importlib-metadata
 
 ----- stderr -----
-Resolved 10 packages in [TIME]
+  × No solution found when resolving dependencies:
+  ╰─▶ Because there is no version of black available matching ==23.10.1 and
+      root depends on black==23.10.1, version solving failed.