Merge pull request #11 from MichaelHaussmann/dev

Dev to main

.github/workflows/publish_pypi.yml

@@ -0,0 +1,43 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Publish To Test PyPI
+
+on:
+  workflow_dispatch:
+#   release:
+#     types: [published]
+
+permissions:
+  contents: read
+
+jobs:
+  deploy:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.7'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build
+    - name: Build package
+      run: python -m build
+    - name: Publish package to TestPyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+#       if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+#       uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_API_TOKEN }}
+#        repository-url: https://test.pypi.org/legacy/

.gitignore

@@ -13,8 +13,8 @@ venv27
 /.limbo/
 /docs-wip
 /docs/_build
-/hamlet_conf/data/testing/SPIL_PROJECTS
-/hamlet_conf/data/caches/*
+/spil_hamlet_conf/data/testing/SPIL_PROJECTS
+/spil_hamlet_conf/data/caches/*
 /memo.md
 *wip
 *_secret.py

README.md

@@ -319,10 +319,11 @@ More about installation, configuration and testing: [spil.readthedocs.io](https:
 Spil thrives to be used interactively. 
 It's performance depends on the data sources that are used.
 
-- Spil ships with a configurable FindInCache to handle data that changes rarely (projects, sequences, asset types). 
-- String / Sid Resolves are internally stored in a lru_cache
-- searches use generators
-
+- Spil uses a `FindInConstants` class to handle configurable data that mostly doesn't change (types, asset types)
+- Spil ships with a configurable `FindInCache` class to handle data that changes rarely (projects, sequences, assets, etc.).
+  (not production ready in current release)
+- String / Sid Resolves are internally stored in a custom lru_cache
+- `Finders` use generators
 
 ## Concepts  
 

conftest.py

@@ -5,11 +5,10 @@
 """
 from contextlib import suppress
 
-import spil  # default config path bootstrap
 from spil import SpilException
 import spil_data_conf
-from scripts.example_sids import sids
-from spil_tests.prep.make_mock_fs import generate_mock_fs
+from hamlet_scripts.example_sids import sids
+from spil.tests.prep.make_mock_fs import generate_mock_fs
 
 
 def pytest_sessionstart(session):

docs/background.md

@@ -97,8 +97,9 @@ Spil aims to be : flexible, pragmatic, simple - and reliable.
 Spil thrives to be used interactively. 
 It's performance depends on the data sources that are used.
 
-- Spil ships with a configurable `FindInCache` class to handle data that changes rarely (projects, sequences, assets, etc.). 
-- Spil uses a `FindInConstant` class to handle configurable data that mostly doesn't change (types, asset types)
+- Spil uses a `FindInConstants` class to handle configurable data that mostly doesn't change (types, asset types)
+- Spil ships with a configurable `FindInCache` class to handle data that changes rarely (projects, sequences, assets, etc.).
+  (not production ready in current release)
 - String / Sid Resolves are internally stored in a custom lru_cache
 - `Finders` use generators
 

docs/conf.py

@@ -16,7 +16,7 @@
 here = Path(__file__)
 root = here.parent.parent
 venv = root / 'venv' / 'lib' / 'python3.7' / 'site-packages'
-spil_conf = root / 'hamlet_conf'
+spil_conf = root / 'spil_hamlet_conf'
 print(root)
 sys.path.append(str(root))
 sys.path.append(str(venv))

docs/configuration.md

@@ -23,15 +23,15 @@ Best way to get started:
 ## Included Demo configuration
 
 To be able to use Spil, the configuration must be added to the python path.  
-Spil ships with an example configuration folder named `hamlet_conf`, for a hypothetical *"hamlet"* project.
+Spil ships with an example configuration folder named `spil_hamlet_conf`, for a hypothetical *"hamlet"* project.
 
-If no configuration is found, the `hamlet_conf` folder is added to the python path during spil import.
+If no configuration is found, the `spil_hamlet_conf` folder is added to the python path during spil import.
 
 ### Adapting the demo config 
 
 The easiest way to start, is to adapt the existing config to your needs.
 
-- Copy or move the `hamlet_conf` folder to a location of your choice, and add it to your python path.
+- Copy or move the `spil_hamlet_conf` folder to a location of your choice, and add it to your python path.
 - You can rename it. 
 - The configuration files inside the folder, `spil_*_conf.py`,  must keep the same naming.
 
@@ -72,7 +72,7 @@ The easiest way to start, is to adapt the existing config to your needs.
 
 ### Included test files
 
-Under `hamlet_conf/tests` are tests scripts, that test the functions of the Sid and the configuration.
+Under `spil_hamlet_conf/tests` are tests scripts, that test the functions of the Sid and the configuration.
 While adapting the configuration, running the tests help detect problems.
 
 See below, [Testing the configuration](configuration.md).
@@ -153,8 +153,8 @@ It configures Finders (FindInCaches, FindInAll), Getters, and Writers.
 
 ## Testing the configuration
 
-- Recreate or adapt the `hamlet_conf/scripts/example_sids.py` script, which generates correctly formatted test Sids.
-- Run `hamlet_conf/scripts/save_examples_to_mock_fs.py`: this will create dummy project files and folders on disk
+- Recreate or adapt the `spil_hamlet_conf/scripts/example_sids.py` script, which generates correctly formatted test Sids.
+- Run `spil_hamlet_conf/scripts/save_examples_to_mock_fs.py`: this will create dummy project files and folders on disk
 
 ### Checking the config
 
@@ -185,7 +185,7 @@ Found: hamlet/a/char/ghost
 Found: hamlet/a/char/gertrude
 ```
 
-Complete tests are found in `hamlet_conf/tests`.
+Complete tests are found in `spil_hamlet_conf/tests`.
 
 Read more here: [testing](testing.md).
 

docs/configuration_advanced.md

@@ -82,7 +82,7 @@ So, you are about to create a new pipeline ? :)
 
 ### Testing the configuration
 
-- Create or adapt the **example_sids.py** script (as found in `hamlet_conf/scripts`) which generates correctly formatted test Sids.
+- Create or adapt the **example_sids.py** script (as found in `spil_hamlet_conf/scripts`) which generates correctly formatted test Sids.
 - Run **save_examples_to_mock_fs.py**: this will create dummy project files and folders on disk
 
 

docs/faq.md

@@ -40,6 +40,14 @@ Once you add your configuration folder (or the demo configuration folder) to the
 
 Please check the documentation for "Adapting the demo config" [configuration](configuration.md).
 
+## The project name is hard coded in the configuration file ?
+
+In the shipped default demo configuration, the project is hardcoded in the spil_sid_conf file.
+This can be changed, to retrieve projects names by any other possible way.
+
+The main idea is: the preciser the information in the configuration, the better the pattern matching works.    
+Data that changes rarely (asset types, task types, and potentially project names) should find their way into the configuration.  
+How this is done can vary, and getting the configuration from a database is absolutely possible.
 
 ## Use the name of an item as it's unique Id - really ? 
 

docs/installation.md

@@ -12,13 +12,13 @@ $ pip install spil
 # Or install latest version
 $ pip install git+https://github.com/MichaelHaussmann/spil.git
 ```
-This installs `spil`, `spil_tests`, `spil_plugins`, and `hamlet_conf`, the demo configuration.
+This installs `spil`, `spil_tests`, `spil_plugins`, and `spil_hamlet_conf`, the demo configuration.
 
 ### Default configuration
 
 To be able to use Spil, the configuration must be added to the python path.  
-Spil ships with an example configuration folder named `hamlet_conf`.
-If no configuration is found, the `hamlet_conf` folder is added to the python path during spil import.
+Spil ships with an example configuration folder named `spil_hamlet_conf`.
+If no configuration is found, the `spil_hamlet_conf` folder is added to the python path during spil import.
 
 Learn more at the [configuration documentation](configuration.md).
 
@@ -38,7 +38,7 @@ The config package could be versioned by current year, for example 2023.0.1
 
 ## Testing the install
 
-Spil works out of the box with the shipped demo config `hamlet_conf`.
+Spil works out of the box with the shipped demo config `spil_hamlet_conf`.
 
 In python, type:
 ```python
@@ -55,14 +55,14 @@ print(sid.as_query())
 
 To play with the Finder and Sid paths, files need to exist on the file system.
 
-To generate test files, run `hamlet_conf/scripts/save_examples_to_mock_fs.py`.
-This will create dummy project files and folders on disk, inside the `hamlet_conf/data` folder.
+To generate test files, run `spil_hamlet_conf/scripts/save_examples_to_mock_fs.py`.
+This will create dummy project files and folders on disk, inside the `spil_hamlet_conf/data` folder.
 
 From the installation folder type:
 ```shell
-$ python hamlet_conf/scripts/save_examples_to_mock_fs.py 
+$ python spil_hamlet_conf/scripts/save_examples_to_mock_fs.py 
 ```
-*If you didn't configure Spil, it prints the location of the `hamlet_conf` path when you use it. For example when you `import spil`.*
+*If you didn't configure Spil, it prints the location of the `spil_hamlet_conf` path when you use it. For example when you `import spil`.*
 *Make sure `spil` is in your python path, which should be the case after installation.*
 
 

docs/testing.md

@@ -7,7 +7,7 @@ Spil uses pytest.
 The *Auto Test* (auto_test.yml) github action runs all tests, including doc-tests, on windows and ubuntu.
 
 Pytest is configured in pyproject.toml and a conftest.py file.
-On session start, Pytest runs test file creations (inside the `hamlet_conf/data/testing` folder).
+On session start, Pytest runs test file creations (inside the `spil_hamlet_conf/data/testing` folder).
 
 The Action is triggered by push or pull-request on main branch.
 
@@ -22,9 +22,9 @@ These modules can be run directly from a code editor.
 ## Integration Tests - Data tests
 
 Most tests work in conjunction with the configuration.
-Per default, they use the shipped demo configuration `hamlet_conf`.
+Per default, they use the shipped demo configuration `spil_hamlet_conf`.
 
-On session start, Pytest runs file creations (inside the `hamlet_conf/data/testing` folder).
+On session start, Pytest runs file creations (inside the `spil_hamlet_conf/data/testing` folder).
 These files are used by the tests (including doctests).
 
 When creating a new configuration for a pipeline, it is recommended to implement related tests.
@@ -42,22 +42,22 @@ When creating a new config, these basic config conformity checks are useful:
 
 For in-depth tests and greater test coverage, it is useful to prepare static data for repeatable tests.
 
-- Recreate or adapt the `hamlet_conf/scripts/generate_example_sids.py` script, which generates correctly formatted test Sids.
-- Save the example sids into a file using `hamlet_conf/scripts/save_examples_to_file.py`
-- Run `hamlet_conf/scripts/save_examples_to_mock_fs.py`: this will create dummy project files and folders on disk (using the test Sids).
+- Recreate or adapt the `spil_hamlet_conf/scripts/generate_example_sids.py` script, which generates correctly formatted test Sids.
+- Save the example sids into a file using `spil_hamlet_conf/scripts/save_examples_to_file.py`
+- Run `spil_hamlet_conf/scripts/save_examples_to_mock_fs.py`: this will create dummy project files and folders on disk (using the test Sids).
 
 ### Testing the Sids and Finders
 
 If the basic config checks pass, and there is testable data, we can continue to usage tests.  
-Complete tests are found in `hamlet_conf/tests`.
+Complete tests are found in `spil_hamlet_conf/tests`.
 
-- `hamlet_conf/tests/core_test.py`: tests given Sids for **core attributes** (types, fields, parent(), etc.)
-- `hamlet_conf/tests/path_test.py`: tests given Sids **path resolving and path related attributes** (path())
-- `hamlet_conf/tests/data_test.py`: tests given Sids **data access attributes** (exists(), children(), get_last(), etc.)
-- `hamlet_conf/tests/finder_test.py`: uses given Sids to build random search Sids, and **tests Finders**.  
+- `spil_hamlet_conf/tests/core_test.py`: tests given Sids for **core attributes** (types, fields, parent(), etc.)
+- `spil_hamlet_conf/tests/path_test.py`: tests given Sids **path resolving and path related attributes** (path())
+- `spil_hamlet_conf/tests/data_test.py`: tests given Sids **data access attributes** (exists(), children(), get_last(), etc.)
+- `spil_hamlet_conf/tests/finder_test.py`: uses given Sids to build random search Sids, and **tests Finders**.  
   Do not hesitate to adapt this test file, depending on the finders you will use.  
   *(Note that you could also manually create a python script with example / test searches)*.
-- `hamlet_conf/tests/quicktest.py`: sandbox-like file to quickly test various things.
+- `spil_hamlet_conf/tests/quicktest.py`: sandbox-like file to quickly test various things.
 
 *(more tests work in progress)*
 

pyproject.toml

@@ -5,15 +5,13 @@ build-backend = "hatchling.build"
 [tool.hatch.build]
 include = [
   "spil",
-  "spil_plugins",
-  "spil_tests",
-  "hamlet_conf"
+  "spil_hamlet_conf"
 ]
 exclude = [".gitignore"]
 
 [project]
 name = "spil"
-version = "0.1.0"
+version = "0.1.0rc2"
 description = "The Simple Pipeline Lib."
 readme = "README.md"
 authors = [{ name = "Michael Haussmann", email = "[email protected]" }]
@@ -37,14 +35,15 @@ dependencies = [
     "Fileseq",
     "future",
     "logzero",
-    "typing-extensions"
+    "typing-extensions",
+    "codetiming",
 ]
 requires-python = ">=3.7"
 
 [project.optional-dependencies]
-dev = ["codetiming", "pytest"]  # "Faker"
-qc = ["mypy", "black", "flake8", "isort", "refurb"]
-sg = ["shotgun-api3", "Unidecode"]
+dev = ["pytest"]  # "Faker"
+qc = ["mypy", "black", "flake8", "isort", "refurb"]  # Code Quality
+sg = ["shotgun-api3", "Unidecode"]  # SG plugin
 
 [project.urls]
 "Homepage" = "https://github.com/MichaelHaussmann/spil"

spil/conf/global_conf.py

@@ -23,8 +23,8 @@
 )
 
 application_path = Path(__file__).parent.parent
-default_sid_conf_path = str(application_path.parent / "hamlet_conf")
-default_sid_conf_data_path = str(application_path.parent / "hamlet_conf" / "data")
+default_sid_conf_path = str(application_path.parent / "spil_hamlet_conf")
+default_sid_conf_data_path = str(application_path.parent / "spil_hamlet_conf" / "data")
 
 # automatic replacement to INFO and WARN in BETA and PROD
 loglevel = 80
@@ -62,7 +62,7 @@
 
     The configuration module "{module}" was not found.
 
-    Ensure to either include the demo "hamlet_conf" in your python path, 
+    Ensure to either include the demo "spil_hamlet_conf" in your python path, 
     or create your own "{module}" and add its folder to the python path.    
 
     (If you are running a py.test edit the SPIL_CONF_PATH variable in tests/test_00_init.py to match a python path.)

spil/sid/core/query_helper.py

@@ -139,7 +139,7 @@ def apply_query(string, query=None, type=None, fields=None):
     This is a problem in case of a read Sid, which could be poly-typed.
 
     Examples.
-    (Note that the examples depend on "hamlet_conf" example configuration. Failure may be due to non matching config)
+    (Note that the examples depend on "spil_hamlet_conf" example configuration. Failure may be due to non matching config)
 
     Query updates the sequence:
     >>> apply_query('hamlet/s/sq010', query='sequence=sq030', type='shot__sequence', fields={'project':'hamlet','type':'s','sequence':'sq010'})