Merge branch 'main' into release-cleanup

geopandas · Dec 23, 2024 · cdd73f3 · cdd73f3
2 parents d31a006 + 07416f4
commit cdd73f3
Show file tree

Hide file tree

Showing 41 changed files with 1,229 additions and 429 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -20,7 +20,7 @@ jobs:
       - run:
           name: Build the Linux aarch64 wheels.
           command: |
-            python3 -m pip install --user cibuildwheel==2.20.0
+            python3 -m pip install --user cibuildwheel==2.22.0
             python3 -m cibuildwheel --output-dir wheelhouse
       - run:
           name: Test the wheels

diff --git a/.github/workflows/docker-gdal.yml b/.github/workflows/docker-gdal.yml
@@ -21,12 +21,13 @@ jobs:
       matrix:
         container:
           - "ghcr.io/osgeo/gdal:ubuntu-small-latest" # >= python 3.12.3
+          - "ghcr.io/osgeo/gdal:ubuntu-small-3.10.0" # python 3.12.3
           - "ghcr.io/osgeo/gdal:ubuntu-small-3.9.2" # python 3.12.3
           - "ghcr.io/osgeo/gdal:ubuntu-small-3.8.5" # python 3.10.12
           - "ghcr.io/osgeo/gdal:ubuntu-small-3.7.3" # python 3.10.12
           - "ghcr.io/osgeo/gdal:ubuntu-small-3.6.4" # python 3.10.6
-          - "osgeo/gdal:ubuntu-small-3.5.3" # python 3.8.10
-          - "osgeo/gdal:ubuntu-small-3.4.3" # python 3.8.10
+          - "osgeo/gdal:ubuntu-small-3.5.3" # python 3.9.20 (installed manually)
+          - "osgeo/gdal:ubuntu-small-3.4.3" # python 3.9.20 (installed manually)
 
     container:
       image: ${{ matrix.container }}
@@ -36,6 +37,15 @@ jobs:
         run: |
           apt-get update && apt-get install -y build-essential git python3-dev
 
+      - name: Install Python
+        # the GDAL 3.4 and 3.5 images do have Python 3.8 installed, so have to
+        # install a more recent Python version manually
+        if: matrix.container == 'osgeo/gdal:ubuntu-small-3.5.3' || matrix.container == 'osgeo/gdal:ubuntu-small-3.4.3'
+        run: |
+          apt-get update && apt-get install -y software-properties-common
+          add-apt-repository -y ppa:deadsnakes/ppa
+          apt-get update && apt-get install -y python3.9-dev
+
       - uses: actions/checkout@v4
 
       - name: Create virtual environment
@@ -44,7 +54,7 @@ jobs:
         # used for tests below
         run: |
           curl -LsSf https://astral.sh/uv/install.sh | sh
-          . $HOME/.cargo/env
+          . $HOME/.local/bin/env
           uv venv .venv
           echo "VIRTUAL_ENV=.venv" >> $GITHUB_ENV
           echo "$PWD/.venv/bin" >> $GITHUB_PATH

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -54,7 +54,7 @@ jobs:
     needs: [build-sdist]
     runs-on: ubuntu-latest
     container:
-      image: "ghcr.io/osgeo/gdal:ubuntu-small-3.9.2"
+      image: "ghcr.io/osgeo/gdal:ubuntu-small-3.10.0"
 
     steps:
       - name: Install packages
@@ -67,7 +67,7 @@ jobs:
         # used for tests below
         run: |
           curl -LsSf https://astral.sh/uv/install.sh | sh
-          . $HOME/.cargo/env
+          . $HOME/.local/bin/env
           uv venv .venv
           echo "VIRTUAL_ENV=.venv" >> $GITHUB_ENV
           echo "$PWD/.venv/bin" >> $GITHUB_PATH
@@ -136,7 +136,7 @@ jobs:
           BUILDKIT_PROGRESS: plain
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.20.0
+        uses: pypa/cibuildwheel@v2.22.0
 
       - uses: actions/upload-artifact@v4
         with:
@@ -151,7 +151,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - os: "macos-12"
+          - os: "macos-13"
             triplet: "x64-osx-dynamic-release"
             arch: x86_64
             vcpkg_cache: "/Users/runner/.cache/vcpkg/archives"
@@ -176,6 +176,15 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: Cache vcpkg
+        uses: actions/cache@v4
+        id: vcpkgcache
+        with:
+          path: |
+            ${{ matrix.vcpkg_cache }}
+          # bump the last digit to avoid using previous build cache
+          key: ${{ matrix.os }}-${{ matrix.arch }}-vcpkg-gdal3.10.0-cache0
+
       # MacOS build requires aclocal, which is part of automake, but appears
       # to be missing in default image
       - name: Reinstall automake
@@ -188,7 +197,7 @@ jobs:
         uses: lukka/run-vcpkg@v11
         with:
           vcpkgDirectory: '${{ github.workspace }}/vcpkg'
-          vcpkgGitCommitId: 73794ce5f63fd138fab999a22959ca7c6305d93c
+          vcpkgGitCommitId: 0857a4b08c14030bbe41e80accb2b1fddb047a74
 
       - name: Install GDAL
         env:
@@ -207,7 +216,7 @@ jobs:
           path: ${{ matrix.vcpkg_logs }}
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.20.0
+        uses: pypa/cibuildwheel@v2.22.0
         env:
           # CIBW needs to know triplet for the correct install path
           VCPKG_DEFAULT_TRIPLET: ${{ matrix.triplet }}
@@ -232,10 +241,10 @@ jobs:
             "ubuntu-latest",
             "ubuntu-20.04",
             "windows-latest",
-            "macos-12",
+            "macos-13",
             "macos-latest",
           ]
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
         include:
           - os: "ubuntu-latest"
             artifact: pyogrio-wheel-linux-manylinux2014_x86_64
@@ -245,7 +254,7 @@ jobs:
             artifact: pyogrio-wheel-linux-manylinux_2_28_x86_64
           - os: "windows-latest"
             artifact: pyogrio-wheel-x64-windows-dynamic-release
-          - os: "macos-12"
+          - os: "macos-13"
             artifact: pyogrio-wheel-x64-osx-dynamic-release
           - os: "macos-latest"
             artifact: pyogrio-wheel-arm64-osx-dynamic-release
@@ -263,21 +272,20 @@ jobs:
           python-version: ${{ matrix.python-version }}
           allow-prereleases: true
 
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
       - name: Create virtual environment (Linux / MacOS)
-        # install uv and use it to create a virtual environment, then add it to
-        # environment variables so that it is automatically activated and can be
-        # used for tests below
+        # use uv to create a virtual environment, then add it to environment
+        # variables so that it is automatically activated and can be used for
+        # tests below
         if: ${{ runner.os != 'Windows' }}
-        shell: bash
         run: |
-          curl -LsSf https://astral.sh/uv/install.sh | sh
-          . $HOME/.cargo/env
           uv venv .venv
 
       - name: Create virtual environment (Windows)
         if: ${{ runner.os == 'Windows' }}
         run: |
-          irm https://astral.sh/uv/install.ps1 | iex
           uv venv .venv
           "VIRTUAL_ENV=.venv" | Out-File -FilePath $env:GITHUB_ENV -Append
           "$PWD/.venv/Scripts" | Out-File -FilePath $env:GITHUB_PATH -Append
@@ -291,9 +299,13 @@ jobs:
       - name: Install dependencies and pyogrio wheel
         shell: bash
         run: |
-          uv pip install -r ci/requirements-wheel-test.txt
+          if [ ${{ matrix.python-version }} != "3.13" ]; then
+            uv pip install -r ci/requirements-wheel-test.txt
+          else
+            uv pip install pytest numpy certifi packaging
+          fi
           uv pip install --no-cache --pre --no-index --find-links wheelhouse pyogrio
-          if [ ${{ matrix.python-version }} != "3.12" ]; then
+          if [ ${{ matrix.python-version }} != "3.13" ]; then
             uv pip install --no-deps geopandas
           fi
           uv pip list

diff --git a/.github/workflows/tests-conda.yml b/.github/workflows/tests-conda.yml
@@ -50,7 +50,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Install Conda environment with Micromamba
-        uses: mamba-org/setup-micromamba@v1
+        uses: mamba-org/setup-micromamba@v2
         with:
           environment-file: ci/envs/${{ matrix.env }}.yml
           create-args: >-
@@ -62,7 +62,7 @@ jobs:
         run: |
           echo "GDAL_INCLUDE_PATH=$MAMBA_ROOT_PREFIX/envs/test/Library/include." >> $GITHUB_ENV
           echo "GDAL_LIBRARY_PATH=$MAMBA_ROOT_PREFIX/envs/test/Library/lib" >> $GITHUB_ENV
-          echo "GDAL_VERSION=$(gdalinfo --version | cut -c 6-10)" >> $GITHUB_ENV
+          echo "GDAL_VERSION=$(gdalinfo --version | awk '{print $2}' | tr -d ",")" >> $GITHUB_ENV
 
       - name: Install pyogrio
         run: pip install -e .

diff --git a/CHANGES.md b/CHANGES.md
@@ -1,12 +1,30 @@
 # CHANGELOG
 
-## 0.10.0 (yyyy-mm-dd)
+## 0.11.0 (TBD)
+
+### Improvements
+
+-   Capture all errors logged by gdal when opening a file fails (#495).
+
+### Bug fixes
+
+-   Fix WKB writing on big-endian systems (#497).
+
+### Packaging
+
+-   The GDAL library included in the wheels is upgraded from 3.9.2 to 3.10.0 (#499).
+
+## 0.10.0 (2024-09-28)
+
+### Improvements
+
+-   Add support to read, write, list, and remove `/vsimem/` files (#457).
 
 ### Bug fixes
 
 -   Silence warning from `write_dataframe` with `GeoSeries.notna()` (#435).
 -   Enable mask & bbox filter when geometry column not read (#431).
--   Raise NotImplmentedError when user attempts to write to an open file handle (#442).
+-   Raise `NotImplementedError` when user attempts to write to an open file handle (#442).
 -   Prevent seek on read from compressed inputs (#443).
 
 ### Packaging
@@ -18,6 +36,8 @@
     This also leads to `pyproj` becoming an optional dependency; you will need
     to install `pyproj` in order to support spatial reference systems (#452).
 -   The GDAL library included in the wheels is updated from 3.8.5 to GDAL 3.9.2 (#466).
+-   pyogrio now requires a minimum version of Python >= 3.9 (#473).
+-   Wheels are now available for Python 3.13.
 
 ## 0.9.0 (2024-06-17)
 

diff --git a/README.md b/README.md
@@ -1,38 +1,32 @@
-# pyogrio - Vectorized spatial vector file format I/O using GDAL/OGR
-
-Pyogrio provides a
-[GeoPandas](https://github.com/geopandas/geopandas)-oriented API to OGR vector
-data sources, such as ESRI Shapefile, GeoPackage, and GeoJSON. Vector data sources
-have geometries, such as points, lines, or polygons, and associated records
-with potentially many columns worth of data.
-
-Pyogrio uses a vectorized approach for reading and writing GeoDataFrames to and
-from OGR vector data sources in order to give you faster interoperability. It
-uses pre-compiled bindings for GDAL/OGR so that the performance is primarily
-limited by the underlying I/O speed of data source drivers in GDAL/OGR rather
-than multiple steps of converting to and from Python data types within Python.
+# pyogrio - bulk-oriented spatial vector file I/O using GDAL/OGR
+
+Pyogrio provides fast, bulk-oriented read and write access to 
+[GDAL/OGR](https://gdal.org/en/latest/drivers/vector/index.html) vector data
+sources, such as ESRI Shapefile, GeoPackage, GeoJSON, and several others.
+Vector data sources typically have geometries, such as points, lines, or
+polygons, and associated records with potentially many columns worth of data.
+
+The typical use is to read or write these data sources to/from
+[GeoPandas](https://github.com/geopandas/geopandas) `GeoDataFrames`. Because
+the geometry column is optional, reading or writing only non-spatial data is
+also possible. Hence, GeoPackage attribute tables, DBF files, or CSV files are
+also supported.
+
+Pyogrio is fast because it uses pre-compiled bindings for GDAL/OGR to read and
+write the data records in bulk. This approach avoids multiple steps of
+converting to and from Python data types within Python, so performance becomes
+primarily limited by the underlying I/O speed of data source drivers in
+GDAL/OGR.
 
 We have seen \>5-10x speedups reading files and \>5-20x speedups writing files
-compared to using non-vectorized approaches (Fiona and current I/O support in
-GeoPandas).
-
-You can read these data sources into
-`GeoDataFrames`, read just the non-geometry columns into Pandas `DataFrames`,
-or even read non-spatial data sources that exist alongside vector data sources,
-such as tables in a ESRI File Geodatabase, or antiquated DBF files.
-
-Pyogrio also enables you to write `GeoDataFrames` to at least a few different
-OGR vector data source formats.
+compared to using row-per-row approaches (e.g. Fiona).
 
 Read the documentation for more information:
 [https://pyogrio.readthedocs.io](https://pyogrio.readthedocs.io/en/latest/).
 
-WARNING: Pyogrio is still at an early version and the API is subject to
-substantial change. Please see [CHANGES](CHANGES.md).
-
 ## Requirements
 
-Supports Python 3.8 - 3.11 and GDAL 3.4.x - 3.8.x.
+Supports Python 3.9 - 3.13 and GDAL 3.4.x - 3.9.x.
 
 Reading to GeoDataFrames requires `geopandas>=0.12` with `shapely>=2`.
 
@@ -52,9 +46,9 @@ for more information.
 
 ## Supported vector formats
 
-Pyogrio supports some of the most common vector data source formats (provided
-they are also supported by GDAL/OGR), including ESRI Shapefile, GeoPackage,
-GeoJSON, and FlatGeobuf.
+Pyogrio supports most common vector data source formats (provided they are also
+supported by GDAL/OGR), including ESRI Shapefile, GeoPackage, GeoJSON, and
+FlatGeobuf.
 
 Please see the [list of supported formats](https://pyogrio.readthedocs.io/en/latest/supported_formats.html)
 for more information.
@@ -64,7 +58,7 @@ for more information.
 Please read the [introduction](https://pyogrio.readthedocs.io/en/latest/supported_formats.html)
 for more information and examples to get started using Pyogrio.
 
-You can also check out the the [API documentation](https://pyogrio.readthedocs.io/en/latest/api.html)
+You can also check out the [API documentation](https://pyogrio.readthedocs.io/en/latest/api.html)
 for full details on using the API.
 
 ## Credits

diff --git a/ci/envs/latest.yml b/ci/envs/latest.yml
@@ -7,4 +7,4 @@ dependencies:
   - pytest
   - shapely>=2
   - geopandas-base
-  - pyarrow
+  - pyarrow-core
diff --git a/ci/envs/libgdal3.5.1.yml b/ci/envs/libgdal3.5.1.yml
@@ -5,8 +5,3 @@ dependencies:
   - numpy
   - libgdal==3.5.1
   - pytest
-  - geopandas-base
-  - pip
-  - pip:
-      # install Shapely >= 2.0 using pip because it is not available on conda-forge for above libgdal
-      - shapely>=2
diff --git a/ci/manylinux2014_x86_64-vcpkg-gdal.Dockerfile b/ci/manylinux2014_x86_64-vcpkg-gdal.Dockerfile
@@ -7,7 +7,7 @@ RUN yum install -y curl unzip zip tar perl-IPC-Cmd
 RUN ln -s /opt/python/cp38-cp38/bin/python3 /usr/bin/python3
 
 RUN git clone https://github.com/Microsoft/vcpkg.git /opt/vcpkg && \
-    git -C /opt/vcpkg checkout 73794ce5f63fd138fab999a22959ca7c6305d93c
+    git -C /opt/vcpkg checkout 0857a4b08c14030bbe41e80accb2b1fddb047a74
 
 ENV VCPKG_INSTALLATION_ROOT="/opt/vcpkg"
 ENV PATH="${PATH}:/opt/vcpkg"

diff --git a/ci/manylinux_2_28_aarch64-vcpkg-gdal.Dockerfile b/ci/manylinux_2_28_aarch64-vcpkg-gdal.Dockerfile
@@ -4,7 +4,7 @@ FROM quay.io/pypa/manylinux_2_28_aarch64:2024-08-12-7fde9b1
 RUN dnf -y install curl zip unzip tar ninja-build perl-IPC-Cmd
 
 RUN git clone https://github.com/Microsoft/vcpkg.git /opt/vcpkg && \
-    git -C /opt/vcpkg checkout 73794ce5f63fd138fab999a22959ca7c6305d93c
+    git -C /opt/vcpkg checkout 0857a4b08c14030bbe41e80accb2b1fddb047a74
 
 ENV VCPKG_INSTALLATION_ROOT="/opt/vcpkg"
 ENV PATH="${PATH}:/opt/vcpkg"
@@ -26,10 +26,6 @@ RUN bootstrap-vcpkg.sh && \
 COPY ci/custom-triplets/arm64-linux-dynamic-release.cmake opt/vcpkg/custom-triplets/arm64-linux-dynamic-release.cmake
 COPY ci/vcpkg.json opt/vcpkg/
 
-# temporary workaround for https://github.com/microsoft/vcpkg/issues/36094
-COPY ci/vcpkg_linux_crosscompiling.patch opt/vcpkg/
-RUN git -C /opt/vcpkg apply vcpkg_linux_crosscompiling.patch
-
 ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/vcpkg/installed/arm64-linux-dynamic-release/lib"
 RUN vcpkg install --overlay-triplets=opt/vcpkg/custom-triplets \
     --feature-flags="versions,manifests" \

diff --git a/ci/manylinux_2_28_x86_64-vcpkg-gdal.Dockerfile b/ci/manylinux_2_28_x86_64-vcpkg-gdal.Dockerfile
@@ -4,7 +4,7 @@ FROM quay.io/pypa/manylinux_2_28_x86_64:2024-08-12-7fde9b1
 RUN dnf -y install curl zip unzip tar ninja-build perl-IPC-Cmd
 
 RUN git clone https://github.com/Microsoft/vcpkg.git /opt/vcpkg && \
-    git -C /opt/vcpkg checkout 73794ce5f63fd138fab999a22959ca7c6305d93c
+    git -C /opt/vcpkg checkout 0857a4b08c14030bbe41e80accb2b1fddb047a74
 
 ENV VCPKG_INSTALLATION_ROOT="/opt/vcpkg"
 ENV PATH="${PATH}:/opt/vcpkg"

diff --git a/ci/vcpkg.json b/ci/vcpkg.json
@@ -1,12 +1,12 @@
 {
     "name": "pyogrio",
-    "version": "0.10.0",
+    "version": "0.11.0",
     "dependencies": [
         {
             "name": "gdal",
             "default-features": false,
-            "features": ["recommended-features", "curl", "geos", "iconv"]
+            "features": ["recommended-features", "curl", "geos", "iconv", "openssl"]
         }
     ],
-    "builtin-baseline": "73794ce5f63fd138fab999a22959ca7c6305d93c"
+    "builtin-baseline": "0857a4b08c14030bbe41e80accb2b1fddb047a74"
 }