Merge pull request #74 from paulsengroup/test/fuzzy

Implement fuzzer

.github/workflows/fuzzy-testing.yml

@@ -0,0 +1,279 @@
+# Copyright (C) 2024 Roberto Rossini ([email protected])
+# SPDX-License-Identifier: MIT
+
+name: Run fuzzy tests
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - ".github/workflows/fuzzy-testing.yml"
+      - "cmake/**"
+      - "src/**"
+      - "test/scripts/fuzzer.py"
+      - "utils/devel/stubgen.py"
+      - "utils/devel/symlink_pyarrow_libs.py"
+      - "CMakeLists.txt"
+      - "conanfile.py"
+      - "pyproject.toml"
+
+  pull_request:
+    paths:
+      - "cmake/**"
+      - "src/**"
+      - "test/scripts/fuzzer.py"
+      - "utils/devel/stubgen.py"
+      - "utils/devel/symlink_pyarrow_libs.py"
+      - "CMakeLists.txt"
+      - "conanfile.py"
+      - "pyproject.toml"
+
+  schedule:
+    # Run weekly
+    - cron: "15 3 * * 0"
+
+  workflow_dispatch:
+    inputs:
+      duration:
+        description: "Test duration in seconds"
+        required: true
+        default: "600"
+        type: string
+
+      resolution:
+        description: "Matrix resolution to use for testing"
+        required: true
+        default: "random"
+        type: string
+
+# https://stackoverflow.com/a/72408109
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  build-project:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - {
+              dataset: "4DNFIYECESRC",
+              format: "cool",
+              normalization: "NONE",
+              bin-type: "fixed",
+            }
+          - {
+              dataset: "4DNFIYECESRC",
+              format: "cool",
+              normalization: "weight",
+              bin-type: "fixed",
+            }
+          - {
+              dataset: "4DNFIYECESRC",
+              format: "cool",
+              normalization: "VC",
+              bin-type: "fixed",
+            }
+          - {
+              dataset: "4DNFIYECESRC",
+              format: "cool",
+              normalization: "NONE",
+              bin-type: "variable",
+            }
+          - {
+              dataset: "4DNFIYECESRC",
+              format: "hic8",
+              normalization: "NONE",
+              bin-type: "fixed",
+            }
+          - {
+              dataset: "4DNFIYECESRC",
+              format: "hic8",
+              normalization: "KR",
+              bin-type: "fixed",
+            }
+          - {
+              dataset: "4DNFIYECESRC",
+              format: "hic9",
+              normalization: "NONE",
+              bin-type: "fixed",
+            }
+          - {
+              dataset: "4DNFIYECESRC",
+              format: "hic9",
+              normalization: "VC",
+              bin-type: "fixed",
+            }
+
+    container:
+      image: ghcr.io/paulsengroup/ci-docker-images/ubuntu-24.04-cxx-clang-19
+      options: "--user=root"
+
+    env:
+      CCACHE_DISABLE: "1"
+      CONAN_HOME: "/opt/conan/"
+      HICTK_CI: "1"
+
+    steps:
+      - name: Clone hictkpy
+        uses: actions/checkout@v4
+
+      - name: Fix permissions
+        run: |
+          chown -R $(id -u):$(id -g) $PWD
+
+      - name: Generate cache key
+        id: cache-key
+        run: |
+          hash="${{ hashFiles('conanfile.py') }}"
+
+          echo "key=fuzzer-$hash" >> $GITHUB_OUTPUT
+
+      - name: Install Python
+        run: |
+          apt-get update
+          apt-get install -y python3.12 python3.12-dev
+
+      - name: Restore Conan cache
+        id: cache-conan
+        uses: actions/cache/restore@v4
+        with:
+          key: conan-${{ steps.cache-key.outputs.key }}
+          path: ${{ env.CONAN_HOME }}/p
+
+      - name: Clean Conan cache (pre-build)
+        if: steps.cache-conan.outputs.cache-hit != 'true'
+        run: |
+          conan cache clean "*" --build
+          conan cache clean "*" --download
+          conan cache clean "*" --source
+          conan remove --confirm "*"
+
+      - name: Build and install
+        run: pip install --verbose .
+
+      - name: Clean Conan cache (post-build)
+        if: steps.cache-conan.outputs.cache-hit != 'true'
+        run: |
+          conan cache clean "*" --build
+          conan cache clean "*" --download
+          conan cache clean "*" --source
+
+      - name: Save Conan cache
+        uses: actions/cache/save@v4
+        if: steps.cache-conan.outputs.cache-hit != 'true'
+        with:
+          key: conan-${{ steps.cache-key.outputs.key }}
+          path: ${{ env.CONAN_HOME }}/p
+
+      - name: Install test dependencies
+        run: |
+          pip install --no-cache-dir 'cooler==0.10.*' 'numpy<2'
+
+      - name: Detect CI type
+        id: ci-type
+        run: |
+          if git log --format=%B -n 1 ${{ github.event.after }} | grep -qF '[ci full]'; then
+            echo "type=full" >> $GITHUB_OUTPUT
+          else
+            echo "type=short" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Prepare for test
+        id: test-params
+        run: |
+          duration=120
+          if [[ ${{ github.event_name }} == 'workflow_dispatch' ]]; then
+            duration='${{ inputs.duration }}'
+          elif [[ '${{ steps.ci-type.outputs.type }}' == 'full' ]]; then
+            duration=3600
+          fi
+
+          resolution=50000
+          if [[ ${{ github.event_name }} == 'workflow_dispatch' ]]; then
+            if [[ '${{ inputs.resolution }}' == 'random' ]]; then
+              resolution="$(
+                python3 -c 'import random; import sys; print(random.choice([int(x) for x in sys.argv[1:]]))' \
+                1000 5000 10000 50000 100000 500000
+              )"
+            else
+              resolution='${{ inputs.resolution }}'
+            fi
+          fi
+
+          if [[ '${{ matrix.bin-type }}' == variable ]]; then
+            resolution=0
+          fi
+
+          2>&1 echo "Duration: ${duration}"
+          2>&1 echo "Resolution: ${resolution}"
+
+          echo "duration=$duration" | tee -a "$GITHUB_OUTPUT"
+          echo "resolution=$resolution" | tee -a "$GITHUB_OUTPUT"
+
+      - name: Clone hictk
+        uses: actions/checkout@v4
+        with:
+          repository: "paulsengroup/hictk"
+          path: hictk
+
+      - name: Download test datasets
+        run: |
+          hictk/test/fuzzer/scripts/download_test_datasets.py \
+            hictk/test/fuzzer/test_files.json \
+            . \
+            --format cool "${{ matrix.format }}" \
+            --resolution "${{ steps.test-params.outputs.resolution }}" \
+            --dataset "${{ matrix.dataset }}" \
+            --nproc 2
+
+      - name: Run test (df)
+        run: |
+          test/scripts/fuzzer.py \
+            --resolution ${{ steps.test-params.outputs.resolution }} \
+            --duration '${{ steps.test-params.outputs.duration }}' \
+            --normalization ${{ matrix.normalization }} \
+            --nproc $(nproc) \
+            --format df \
+            *".${{ matrix.format }}" \
+            *.cool
+
+      - name: Run test (numpy)
+        run: |
+          test/scripts/fuzzer.py \
+            --resolution ${{ steps.test-params.outputs.resolution }} \
+            --duration '${{ steps.test-params.outputs.duration }}' \
+            --normalization ${{ matrix.normalization }} \
+            --nproc $(nproc) \
+            --format numpy \
+            *".${{ matrix.format }}" \
+            *.cool
+
+      - name: Run test (coo)
+        run: |
+          test/scripts/fuzzer.py \
+            --resolution ${{ steps.test-params.outputs.resolution }} \
+            --duration '${{ steps.test-params.outputs.duration }}' \
+            --normalization ${{ matrix.normalization }} \
+            --nproc $(nproc) \
+            --format csr \
+            *".${{ matrix.format }}" \
+            *.cool
+
+  fuzzy-testing-status-check:
+    name: Status Check (fuzzy-testing)
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    needs:
+      - build-project
+
+    steps:
+      - name: Collect job results
+        if: needs.build-project.result != 'success'
+        run: exit 1

pyproject.toml

@@ -74,6 +74,12 @@ filterwarnings = [
   "ignore:(ast.Str|Attribute s|ast.NameConstant|ast.Num) is deprecated:DeprecationWarning:_pytest",
   "ignore:datetime\\.datetime\\.utcfromtimestamp\\(\\)*:DeprecationWarning",  # https://github.com/pytest-dev/pytest/issues/11528
 ]
+python_files = [
+  "test/test*.py"
+]
+testpaths = [
+  "test/"
+]
 
 [tool.cibuildwheel]
 skip = ["cp313*", "*musllinux*", "pp*"]

src/hictkpy_pixel_selector.cpp

@@ -344,10 +344,10 @@ hictk::internal::NumericVariant PixelSelector::parse_count_type(std::string_view
   if (type == "int64") {
     return {std::int64_t{}};
   }
-  if (type == "float32" || type == "float") {
+  if (type == "float32") {
     return {float{}};
   }
-  if (type == "float64" || type == "double") {
+  if (type == "float64" || type == "float" || type == "double") {
     return {double{}};
   }