Merge pull request #39 from Kaszanas/dev

[PR] Preparing 2.0 Release

.env.template

@@ -0,0 +1,2 @@
+# To have imports resolve correctly this should be the path to the root of the project:
+TEST_WORKSPACE=

.github/PULL_REQUEST_TEMPLATE.MD

@@ -2,7 +2,7 @@
 ## Description
 <!--- Describe your changes in detail -->
 
-## Related IssueS
+## Related Issues
 <!--- This project only accepts pull requests related to open issues -->
 <!--- If suggesting a new feature or change, please discuss it in an issue first -->
 <!--- If fixing a bug, there should be an issue describing it with steps to reproduce -->

.github/workflows/ci.yml

@@ -1,26 +1,29 @@
 name: continuous integration (ci)
 
-on: [pull_request, workflow_dispatch]
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+      - dev
+  workflow_dispatch:
 
 # To successfully find the files that are required for testing:
 env:
   TEST_WORKSPACE: ${{ github.workspace }}
 
 jobs:
-
   pre_commit:
     # Set up operating system
     runs-on: ubuntu-latest
-
     # Define job steps
     steps:
-
       - name: Check-out repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
 
       - name: Build Dev Docker Image
         run: |
-          make docker_build_dev
+          make docker_build_devcontainer
 
       - name: Docker Run pre-commit on all files.
         run: |
@@ -37,11 +40,11 @@ jobs:
     # Define job steps
     steps:
       - name: Check-out repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
 
       - name: Build Dev Docker Image
         run: |
-          make docker_build_dev PYTHON_VERSION=${{ matrix.python-version }}
+          make docker_build_devcontainer PYTHON_VERSION=${{ matrix.python-version }}
 
       - name: Build Docker Image With Python ${{ matrix.python-version }}
         run: |

.github/workflows/cla.yml

@@ -19,14 +19,14 @@ jobs:
       - name: "CLA Assistant"
         if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
         # Beta Release
-        uses: contributor-assistant/github-action@v2.3.0
+        uses: contributor-assistant/github-action@ca4a40a7d1004f18d9960b404b97e5f30a505a08
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           # the below token should have repo scope and must be manually added by you in the repository's secret
           PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
         with:
           path-to-signatures: 'signatures/version1/cla.json'
-          path-to-document: 'https://github.com/Kaszanas/SC2DatasetPreparator/blob/main/CLA.md' # e.g. a CLA or a DCO document
+          path-to-document: 'https://github.com/Kaszanas/DatasetPreparator/blob/main/CLA.md' # e.g. a CLA or a DCO document
           # branch should not be protected
           branch: 'main'
           allowlist: Kaszanas,bot*

.github/workflows/docker_images.yml

@@ -0,0 +1,47 @@
+name: Publish Docker Images
+
+# This should run only after the tests from the CI pipeline have passed.
+# On a rare ocassion contributors can trigger this manually, and it should also
+# run after a release has been published.
+on:
+  workflow_run:
+    workflows: ["continuous integration (ci)"]
+    types:
+      - completed
+  push:
+    branches:
+      - main
+      - dev
+  workflow_dispatch:
+  release:
+    types: [published]
+
+jobs:
+  push_to_registries:
+    name: Push Docker Image to Docker Hub
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - name: Check out Code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+      - name: Log in to Docker Hub
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+      - name: Extract Metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804
+        with:
+          images: |
+            kaszanas/datasetpreparator
+      - name: Build and Push Docker images
+        uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4
+        with:
+          context: .
+          file: ./docker/Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.gitignore

@@ -1,7 +1,9 @@
 /.vscode
 /venv*
 
-/processing
+processing/
+maps/
+profiler/
 
 *.SC2Replay
 *.SC2Map
@@ -34,3 +36,5 @@ ruff_cache/
 
 # PyCharm
 /.idea
+
+.env

CHANGELOG.md

@@ -0,0 +1,108 @@
+## 2.0.0 (2025-03-08)
+
+### Feat
+
+- **sc2_map_downloader**: using SC2InfoExtractor go, adjusted documentation
+- **sc2egset_pipeline**: tested full processing pipeline manually
+- **directory_packager**: multithreading in directory packager
+- **directory_flattener**: multithreading file copying
+- added new replaypacks to available_replaypacks
+- added sc2egset_pipeline to dockerfiles
+- separate sc2egset_pipeline and replaypack_processor
+- ignoring maps directory
+- (directory_packager.py) added tqdm progres bar
+- drafted full SC2ReSet/SC2EGSet pipeline
+- **json_merger.py**: added user prompting, and CLI flag
+- added force_overwrite flag to CLI
+- **directory_packager.py**: added user prompting
+- **directory_flattener.py**: added user_prompt feature
+- drafted utils/user_prompt
+- draft functionality of sc2egset_replaypack... full pipeline
+- added processed_mapping_copier target to makefile
+- test workspace in .env
+- sc2infoextractorgo executable path in settings
+- added default flag values for golang
+- added directory checks before file_renamer
+- retry functionality for download_replaypack
+- split download_file from download_replaypack
+- md5 checksum verification for downloaded replaypacks
+- added md5 checksums to available replaypacks
+
+### Fix
+
+- **docker**: fixing volume mount for CI
+- **directory_flattener**: returning list of processed directories
+- fixing file_renamer using directory name
+- fixing replaypack_processor, no exceptions
+- using Path as the inferred path type
+- fixing non-existent argument
+- **sc2_replaypack_processor**: fixing maps directory as arg
+- continue instead of break after download
+- **directory_flattener.py**: manually tested flattening directories
+- manually tested directory_packager, working version
+- fixing pre-commit in dev docker
+- fixing return value, removed range loop
+- fixing opening and writing to file
+- fixing imports in sc2reset
+- mounting curdir as a dot
+- fixing paths in Dockerfile
+- converting paths with click, changed target name
+- fixing glob issues, testing directory flattener
+- fixed log level, fixing path initialization
+- pointing CLA to main
+- fixed import issues in replaypack downloader
+- getting list of files instead of generator
+- fixed missing argument in json_merger
+- lowercase makefile name
+- added logs directory
+- different retry logic, using path to merge suffix
+- changed commitizen pre-commit config
+- removed commitizen-branch hook
+- **deps**: pre-commit autoupdate
+- **deps**: added commitizen to pre-commit
+
+### Refactor
+
+- **directory_flattener**: renamed n_processes -> n_threads
+- removing unused code, replaypack processor
+- maps directory is created, skipping check
+- removed unused code
+- force_overwrite continues directory flattning
+- command saved to a variable
+- removed old directory structure from processing
+- added logging statements
+- applied user prompting for every script
+- **processed_mapping_copier.py**: using pathlib, refactored functionality with iterdir
+- renamed force to force_overwrite
+- using glob instead of os.walk
+- **user_prompt.py**: added logging
+- applied user prompting in sc2egset_replaypack_processor
+- renamed user prompting function
+- drafting refactor of sc2egset_replaypack_processor
+- using dev dockerfile in sc2reset_sc2egset process
+- adjusted make targets for sc2egset, removed unused param
+- changed the processing dir structure
+- refreshed ci installing poetry
+- renamed dir_packager to directory_packager
+- capitalized "AS" in docker
+- renamed sc2_replaypack_processor -> sc2egset_replaypack
+- multiprocessing off in sc2_replaypack_processor
+- no random uuid, using file hash in flattener
+- using new argument classes, refactor
+- added utils directory, drafted README
+- fixed end of file in cla.json
+- sync commit
+- changed dir_flattener logic, using a list of files
+- deleted legacy setup.py
+- ran pre-commit on all files
+
+### Perf
+
+- directory_flattener, hash from filepath, added tqdm
+- downloading maps as a pre-process step
+
+## 1.2.0 (2022-06-08)
+
+## 1.1.0 (2022-03-17)
+
+## 1.0.0 (2021-08-27)

CONTRIBUTING.md

@@ -56,24 +56,36 @@ docker run -it -v .:/app datasetpreparator:devcontainer
 
 ### Local Development
 
-Ready to contribute? Here's how to set up `datasetpreparator` for local development.
+Ready to contribute? Here's how to set up `datasetpreparator` for local development. The code style standards that we use are defined in the `.pre-commit-config.yaml` file.
 
 1. Download a copy of `datasetpreparator` locally.
 2. Install `datasetpreparator` using `poetry`:
 
-    ```console
-      poetry install
-    ```
+```console
+  poetry install
+```
+
+3. Install the pre-commit hooks:
+
+```console
+  poetry run pre-commit install
+```
 
-3. Use `git` (or similar) to create a branch for local development and make your changes:
+4. Use `git` (or similar) to create a branch for local development and make your changes:
 
-    ```console
-      git checkout -b name-of-your-bugfix-or-feature
-    ```
+```console
+  git checkout -b name-of-your-bugfix-or-feature
+```
+
+5. When you're done making changes, check that your changes conform to any code formatting requirements and pass any tests.
 
-4. When you're done making changes, check that your changes conform to any code formatting requirements and pass any tests.
+6. Format your commit with `commitizen`:
+
+```console
+  poetry run cz commit
+```
 
-5. Commit your changes and open a pull request.
+7. Commit your changes (we are using commitizen to check commit messages) and open a pull request.
 
 ## Pull Request Guidelines