Merge pull request #1891 from swahtz/feature/fvdb

fVDB improved test setup and PyTorch 2.4 update

fvdb/.clang-format

@@ -0,0 +1,100 @@
+---
+Language: Cpp
+
+# Indentation
+IndentWidth: 4
+UseTab: Never
+
+# Line length
+ColumnLimit: 100
+
+# Line endings
+DeriveLineEnding: false
+LineEnding: LF
+
+# Spacing and padding
+SpaceAfterCStyleCast: false
+SpaceAfterTemplateKeyword: true
+SpaceAfterLogicalNot: false
+SpaceAfterControlStatementKeyword: true
+SpaceBeforeParens: ControlStatements
+SpaceBeforeRangeBasedForLoopColon: false
+SpaceInEmptyParentheses: false
+SpacesInAngles: false
+SpacesInContainerLiterals: false
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesBeforeTrailingComments: 1
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
+SpaceBeforeSquareBrackets: false
+SpacesInSquareBrackets: false
+SpaceBeforeCaseColon: false
+BreakBeforeBinaryOperators: None
+SpaceBeforeAssignmentOperators: true
+
+# Brace placement
+BraceWrapping:
+  AfterClass: false
+  AfterControlStatement: false
+  AfterEnum: false
+  AfterFunction: false
+  AfterNamespace: false
+  AfterStruct: false
+  AfterUnion: false
+  BeforeCatch: false
+  BeforeElse: false
+  IndentBraces: false
+  SplitEmptyFunction: false
+  SplitEmptyRecord: false
+  SplitEmptyNamespace: false
+
+# Function definitions
+# BreakAfterReturnType: AllDefinitions
+AlwaysBreakAfterDefinitionReturnType: All
+
+# Alignment
+AlignConsecutiveAssignments: true
+AlignConsecutiveDeclarations: true
+AlignEscapedNewlines: Left
+AlignOperands: true
+AlignTrailingComments:
+  Kind: Always
+  OverEmptyLines: 2
+
+# Miscellaneous
+AllowShortFunctionsOnASingleLine: Inline
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AllowAllParametersOfDeclarationOnNextLine: true
+BreakBeforeBraces: Attach
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakInheritanceList: BeforeColon
+BreakStringLiterals: false
+Cpp11BracedListStyle: false
+IncludeBlocks: Preserve
+IncludeIsMainRegex: "$"
+IncludeCategories:
+  - Regex: '^<.*\.h>'
+    Priority: 1
+  - Regex: ".*"
+    Priority: 2
+IncludeIsMainSourceRegex: "$"
+IndentPPDirectives: None
+IndentWrappedFunctionNames: false
+KeepEmptyLinesAtTheStartOfBlocks: false
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+PenaltyBreakAssignment: 2
+PenaltyBreakBeforeFirstCallParameter: 19
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 60
+SortIncludes: true
+SortUsingDeclarations: true
+TabWidth: 4
+Standard: c++20

fvdb/.github/workflows/tests.yml

@@ -3,37 +3,162 @@ name: Running Tests
 on:
   pull_request:
     branches: [main]
+    paths-ignore:
+      - 'debug/**'
+      - 'docs/**'
+      - 'examples/**'
+      - 'notebooks/**'
+      - 'scripts/**'
+
+permissions:
+  contents: write
+  deployments: write
 
 jobs:
 
-  tests:
-    name: Unit Tests
+  fvdb-build:
+    name: fVDB Build
     runs-on:
       - self-hosted
-
+    container:
+      image: aswf/ci-openvdb:2023-clang15.0
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+
+      - name: Set up fvdb_build Conda env
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniconda-version: "latest"
+          channels: conda-forge
+          channel-priority: strict
+          activate-environment: fvdb_build
+          environment-file: env/build_environment.yml
 
+
+      - name: Buid fvdb
+        run: |
+          TORCH_CUDA_ARCH_LIST="7.0;7.5;8.0;8.6+PTX" MAX_JOBS=$(($(nproc) < $(free -g | awk '/^Mem:/{jobs=int($7/2.5); if(jobs<1) jobs=1; print jobs}') ? $(nproc) : $(free -g | awk '/^Mem:/{jobs=int($7/2.5); if(jobs<1) jobs=1; print jobs}'))) python setup.py bdist_wheel --dist-dir=dist
+        shell:
+          bash -el {0}
+
+      - name: Upload package
+        uses: actions/upload-artifact@v4
+        with:
+            name: fvdb-test-package
+            path: dist/*.whl
+
+      - name: Clean Conda
+        run: |
+          conda clean -pty
+        shell:
+          bash -el {0}
+
+  fvdb-unit-test:
+    needs: [fvdb-build]
+    name: fVDB Unit Tests
+    runs-on:
+      - self-hosted
+    container:
+      image: aswf/ci-openvdb:2023-clang15.0
+    steps:
+      - uses: actions/checkout@v4
       - name: Set up fvdb_test Conda env
         uses: conda-incubator/setup-miniconda@v3
         with:
           miniconda-version: "latest"
-          mamba-version: "*"
-          channels: conda-forge,defaults
-          channel-priority: true
+          channels: conda-forge
+          channel-priority: strict
           activate-environment: fvdb_test
           environment-file: env/test_environment.yml
 
+      - name: Download package
+        uses: actions/download-artifact@v4
+        with:
+            name: fvdb-test-package
+            path: ./dist
+
+      - name: Install package
+        run: |
+            conda activate fvdb_test
+            pip install ./dist/*.whl
+        shell:
+              bash -el {0}
 
       - name: Run tests
         run: |
-          conda activate fvdb_test
-          MAX_JOBS=$(($(nproc) < $(free -g | awk '/^Mem:/{jobs=int($4/2.5); if(jobs<1) jobs=1; print jobs}') ? $(nproc) : $(free -g | awk '/^Mem:/{jobs=int($4/2.5); if(jobs<1) jobs=1; print jobs}'))) python setup.py test
+            cd tests;
+            pytest -v unit
+        shell:
+              bash -el {0}
+
+      - name: Clean Conda
+        run: |
+            conda clean -pty
+        shell:
+            bash -el {0}
+
+  fvdb-benchmarks:
+    needs: [fvdb-build]
+    name: fVDB Continuous Benchmarking
+    runs-on:
+      - self-hosted
+    container:
+      image: aswf/ci-openvdb:2023-clang15.0
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up fvdb_test Conda env
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          miniconda-version: "latest"
+          channels: conda-forge
+          channel-priority: strict
+          activate-environment: fvdb_test
+          environment-file: env/test_environment.yml
+
+      - name: Download package
+        uses: actions/download-artifact@v4
+        with:
+            name: fvdb-test-package
+            path: ./dist
+
+      - name: Install package
+        run: |
+            conda activate fvdb_test
+            pip install ./dist/*.whl
+        shell:
+          bash -el {0}
+
+      - name: Disable git ownership verification
+        run: |
+          git config --global --add safe.directory "$(pwd)"
         shell:
           bash -el {0}
 
+      - name: Run benchmarks
+        run: |
+          cd tests;
+          pytest benchmark --benchmark-json benchmark/output.json
+        shell:
+          bash -el {0}
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+            name: Python Benchmark with pytest-benchmark
+            tool: 'pytest'
+            output-file-path: tests/benchmark/output.json
+            # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t/github-action-not-triggering-gh-pages-upon-push/16096
+            github-token: ${{ secrets.GITHUB_TOKEN }}
+            auto-push: true
+            # Show alert with commit comment on detecting possible performance regression
+            alert-threshold: '200%'
+            comment-on-alert: true
+            fail-on-alert: true
+            alert-comment-cc-users: '@swahtz'
+
       - name: Clean Conda
         run: |
-          conda clean -pty
+            conda clean -pty
         shell:
-          bash -el {0}
+            bash -el {0}

fvdb/.gitignore

@@ -49,4 +49,6 @@ tests/benchmark/data
 releases/
 data/
 !tests/data
-.vscode/launch.json
+.vscode/launch.json
+lightning_logs
+tests/data/

fvdb/.gitlab-ci.yml

@@ -19,7 +19,7 @@ pages:
   stage: deploy
   script:
     # build and publish automated documentation with Sphinx:
-    - export MAX_JOBS=$(free -g | awk '/^Mem:/{jobs=int($4/2.5); if(jobs<1) jobs=1; print jobs}')
+    - export MAX_JOBS=$(free -g | awk '/^Mem:/{jobs=int($4/3); if(jobs<1) jobs=1; print jobs}')
     - python setup.py build_ext --inplace
     - sphinx-build -E -a docs/ public/
 

fvdb/.vscode/c_cpp_properties.json

@@ -8,6 +8,7 @@
                 "${workspaceFolder}/src",
                 "${workspaceFolder}/external/openvdb/nanovdb",
                 "${workspaceFolder}/external/cudnn_fe/include",
+                "${workspaceFolder}/external/cutlass/include",
                 "${env:CONDA_PREFIX}/envs/fvdb/include/python3.10",
                 "${env:CONDA_PREFIX}/envs/fvdb/lib/python3.10/site-packages/torch/include",
                 "${env:CONDA_PREFIX}/envs/fvdb/lib/python3.10/site-packages/torch/include/torch/csrc/api/include",