diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml
new file mode 100644
index 0000000..bfa7b0d
--- /dev/null
+++ b/.github/workflows/publish_pypi.yml
@@ -0,0 +1,43 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Publish To Test PyPI
+
+on:
+ workflow_dispatch:
+# release:
+# types: [published]
+
+permissions:
+ contents: read
+
+jobs:
+ deploy:
+
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v3
+ - name: Set up Python
+ uses: actions/setup-python@v3
+ with:
+ python-version: '3.7'
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install build
+ - name: Build package
+ run: python -m build
+ - name: Publish package to TestPyPI
+ uses: pypa/gh-action-pypi-publish@release/v1
+# if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+# uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+ with:
+ user: __token__
+ password: ${{ secrets.PYPI_API_TOKEN }}
+# repository-url: https://test.pypi.org/legacy/
diff --git a/.gitignore b/.gitignore
index 710e2d0..dfa1e0a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,8 +13,8 @@ venv27
/.limbo/
/docs-wip
/docs/_build
-/hamlet_conf/data/testing/SPIL_PROJECTS
-/hamlet_conf/data/caches/*
+/spil_hamlet_conf/data/testing/SPIL_PROJECTS
+/spil_hamlet_conf/data/caches/*
/memo.md
*wip
*_secret.py
diff --git a/README.md b/README.md
index c6e8474..40aee98 100644
--- a/README.md
+++ b/README.md
@@ -319,10 +319,11 @@ More about installation, configuration and testing: [spil.readthedocs.io](https:
Spil thrives to be used interactively.
It's performance depends on the data sources that are used.
-- Spil ships with a configurable FindInCache to handle data that changes rarely (projects, sequences, asset types).
-- String / Sid Resolves are internally stored in a lru_cache
-- searches use generators
-
+- Spil uses a `FindInConstants` class to handle configurable data that mostly doesn't change (types, asset types)
+- Spil ships with a configurable `FindInCache` class to handle data that changes rarely (projects, sequences, assets, etc.).
+ (not production ready in current release)
+- String / Sid Resolves are internally stored in a custom lru_cache
+- `Finders` use generators
## Concepts
diff --git a/conftest.py b/conftest.py
index a984706..afb01d9 100644
--- a/conftest.py
+++ b/conftest.py
@@ -5,11 +5,10 @@
"""
from contextlib import suppress
-import spil # default config path bootstrap
from spil import SpilException
import spil_data_conf
-from scripts.example_sids import sids
-from spil_tests.prep.make_mock_fs import generate_mock_fs
+from hamlet_scripts.example_sids import sids
+from spil.tests.prep.make_mock_fs import generate_mock_fs
def pytest_sessionstart(session):
diff --git a/docs/background.md b/docs/background.md
index ee26dc9..9551ffb 100644
--- a/docs/background.md
+++ b/docs/background.md
@@ -97,8 +97,9 @@ Spil aims to be : flexible, pragmatic, simple - and reliable.
Spil thrives to be used interactively.
It's performance depends on the data sources that are used.
-- Spil ships with a configurable `FindInCache` class to handle data that changes rarely (projects, sequences, assets, etc.).
-- Spil uses a `FindInConstant` class to handle configurable data that mostly doesn't change (types, asset types)
+- Spil uses a `FindInConstants` class to handle configurable data that mostly doesn't change (types, asset types)
+- Spil ships with a configurable `FindInCache` class to handle data that changes rarely (projects, sequences, assets, etc.).
+ (not production ready in current release)
- String / Sid Resolves are internally stored in a custom lru_cache
- `Finders` use generators
diff --git a/docs/conf.py b/docs/conf.py
index c70d819..68a7a7c 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -16,7 +16,7 @@
here = Path(__file__)
root = here.parent.parent
venv = root / 'venv' / 'lib' / 'python3.7' / 'site-packages'
-spil_conf = root / 'hamlet_conf'
+spil_conf = root / 'spil_hamlet_conf'
print(root)
sys.path.append(str(root))
sys.path.append(str(venv))
diff --git a/docs/configuration.md b/docs/configuration.md
index 4e1db47..855b398 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -23,15 +23,15 @@ Best way to get started:
## Included Demo configuration
To be able to use Spil, the configuration must be added to the python path.
-Spil ships with an example configuration folder named `hamlet_conf`, for a hypothetical *"hamlet"* project.
+Spil ships with an example configuration folder named `spil_hamlet_conf`, for a hypothetical *"hamlet"* project.
-If no configuration is found, the `hamlet_conf` folder is added to the python path during spil import.
+If no configuration is found, the `spil_hamlet_conf` folder is added to the python path during spil import.
### Adapting the demo config
The easiest way to start, is to adapt the existing config to your needs.
-- Copy or move the `hamlet_conf` folder to a location of your choice, and add it to your python path.
+- Copy or move the `spil_hamlet_conf` folder to a location of your choice, and add it to your python path.
- You can rename it.
- The configuration files inside the folder, `spil_*_conf.py`, must keep the same naming.
@@ -72,7 +72,7 @@ The easiest way to start, is to adapt the existing config to your needs.
### Included test files
-Under `hamlet_conf/tests` are tests scripts, that test the functions of the Sid and the configuration.
+Under `spil_hamlet_conf/tests` are tests scripts, that test the functions of the Sid and the configuration.
While adapting the configuration, running the tests help detect problems.
See below, [Testing the configuration](configuration.md).
@@ -153,8 +153,8 @@ It configures Finders (FindInCaches, FindInAll), Getters, and Writers.
## Testing the configuration
-- Recreate or adapt the `hamlet_conf/scripts/example_sids.py` script, which generates correctly formatted test Sids.
-- Run `hamlet_conf/scripts/save_examples_to_mock_fs.py`: this will create dummy project files and folders on disk
+- Recreate or adapt the `spil_hamlet_conf/scripts/example_sids.py` script, which generates correctly formatted test Sids.
+- Run `spil_hamlet_conf/scripts/save_examples_to_mock_fs.py`: this will create dummy project files and folders on disk
### Checking the config
@@ -185,7 +185,7 @@ Found: hamlet/a/char/ghost
Found: hamlet/a/char/gertrude
```
-Complete tests are found in `hamlet_conf/tests`.
+Complete tests are found in `spil_hamlet_conf/tests`.
Read more here: [testing](testing.md).
diff --git a/docs/configuration_advanced.md b/docs/configuration_advanced.md
index 04f891b..b62a128 100644
--- a/docs/configuration_advanced.md
+++ b/docs/configuration_advanced.md
@@ -82,7 +82,7 @@ So, you are about to create a new pipeline ? :)
### Testing the configuration
-- Create or adapt the **example_sids.py** script (as found in `hamlet_conf/scripts`) which generates correctly formatted test Sids.
+- Create or adapt the **example_sids.py** script (as found in `spil_hamlet_conf/scripts`) which generates correctly formatted test Sids.
- Run **save_examples_to_mock_fs.py**: this will create dummy project files and folders on disk
diff --git a/docs/faq.md b/docs/faq.md
index 79ad38a..edc0b24 100644
--- a/docs/faq.md
+++ b/docs/faq.md
@@ -40,6 +40,14 @@ Once you add your configuration folder (or the demo configuration folder) to the
Please check the documentation for "Adapting the demo config" [configuration](configuration.md).
+## The project name is hard coded in the configuration file ?
+
+In the shipped default demo configuration, the project is hardcoded in the spil_sid_conf file.
+This can be changed, to retrieve projects names by any other possible way.
+
+The main idea is: the preciser the information in the configuration, the better the pattern matching works.
+Data that changes rarely (asset types, task types, and potentially project names) should find their way into the configuration.
+How this is done can vary, and getting the configuration from a database is absolutely possible.
## Use the name of an item as it's unique Id - really ?
diff --git a/docs/installation.md b/docs/installation.md
index 00b287f..55978ad 100644
--- a/docs/installation.md
+++ b/docs/installation.md
@@ -12,13 +12,13 @@ $ pip install spil
# Or install latest version
$ pip install git+https://github.com/MichaelHaussmann/spil.git
```
-This installs `spil`, `spil_tests`, `spil_plugins`, and `hamlet_conf`, the demo configuration.
+This installs `spil`, `spil_tests`, `spil_plugins`, and `spil_hamlet_conf`, the demo configuration.
### Default configuration
To be able to use Spil, the configuration must be added to the python path.
-Spil ships with an example configuration folder named `hamlet_conf`.
-If no configuration is found, the `hamlet_conf` folder is added to the python path during spil import.
+Spil ships with an example configuration folder named `spil_hamlet_conf`.
+If no configuration is found, the `spil_hamlet_conf` folder is added to the python path during spil import.
Learn more at the [configuration documentation](configuration.md).
@@ -38,7 +38,7 @@ The config package could be versioned by current year, for example 2023.0.1
## Testing the install
-Spil works out of the box with the shipped demo config `hamlet_conf`.
+Spil works out of the box with the shipped demo config `spil_hamlet_conf`.
In python, type:
```python
@@ -55,14 +55,14 @@ print(sid.as_query())
To play with the Finder and Sid paths, files need to exist on the file system.
-To generate test files, run `hamlet_conf/scripts/save_examples_to_mock_fs.py`.
-This will create dummy project files and folders on disk, inside the `hamlet_conf/data` folder.
+To generate test files, run `spil_hamlet_conf/scripts/save_examples_to_mock_fs.py`.
+This will create dummy project files and folders on disk, inside the `spil_hamlet_conf/data` folder.
From the installation folder type:
```shell
-$ python hamlet_conf/scripts/save_examples_to_mock_fs.py
+$ python spil_hamlet_conf/scripts/save_examples_to_mock_fs.py
```
-*If you didn't configure Spil, it prints the location of the `hamlet_conf` path when you use it. For example when you `import spil`.*
+*If you didn't configure Spil, it prints the location of the `spil_hamlet_conf` path when you use it. For example when you `import spil`.*
*Make sure `spil` is in your python path, which should be the case after installation.*
diff --git a/docs/testing.md b/docs/testing.md
index 267c462..dd59731 100644
--- a/docs/testing.md
+++ b/docs/testing.md
@@ -7,7 +7,7 @@ Spil uses pytest.
The *Auto Test* (auto_test.yml) github action runs all tests, including doc-tests, on windows and ubuntu.
Pytest is configured in pyproject.toml and a conftest.py file.
-On session start, Pytest runs test file creations (inside the `hamlet_conf/data/testing` folder).
+On session start, Pytest runs test file creations (inside the `spil_hamlet_conf/data/testing` folder).
The Action is triggered by push or pull-request on main branch.
@@ -22,9 +22,9 @@ These modules can be run directly from a code editor.
## Integration Tests - Data tests
Most tests work in conjunction with the configuration.
-Per default, they use the shipped demo configuration `hamlet_conf`.
+Per default, they use the shipped demo configuration `spil_hamlet_conf`.
-On session start, Pytest runs file creations (inside the `hamlet_conf/data/testing` folder).
+On session start, Pytest runs file creations (inside the `spil_hamlet_conf/data/testing` folder).
These files are used by the tests (including doctests).
When creating a new configuration for a pipeline, it is recommended to implement related tests.
@@ -42,22 +42,22 @@ When creating a new config, these basic config conformity checks are useful:
For in-depth tests and greater test coverage, it is useful to prepare static data for repeatable tests.
-- Recreate or adapt the `hamlet_conf/scripts/generate_example_sids.py` script, which generates correctly formatted test Sids.
-- Save the example sids into a file using `hamlet_conf/scripts/save_examples_to_file.py`
-- Run `hamlet_conf/scripts/save_examples_to_mock_fs.py`: this will create dummy project files and folders on disk (using the test Sids).
+- Recreate or adapt the `spil_hamlet_conf/scripts/generate_example_sids.py` script, which generates correctly formatted test Sids.
+- Save the example sids into a file using `spil_hamlet_conf/scripts/save_examples_to_file.py`
+- Run `spil_hamlet_conf/scripts/save_examples_to_mock_fs.py`: this will create dummy project files and folders on disk (using the test Sids).
### Testing the Sids and Finders
If the basic config checks pass, and there is testable data, we can continue to usage tests.
-Complete tests are found in `hamlet_conf/tests`.
+Complete tests are found in `spil_hamlet_conf/tests`.
-- `hamlet_conf/tests/core_test.py`: tests given Sids for **core attributes** (types, fields, parent(), etc.)
-- `hamlet_conf/tests/path_test.py`: tests given Sids **path resolving and path related attributes** (path())
-- `hamlet_conf/tests/data_test.py`: tests given Sids **data access attributes** (exists(), children(), get_last(), etc.)
-- `hamlet_conf/tests/finder_test.py`: uses given Sids to build random search Sids, and **tests Finders**.
+- `spil_hamlet_conf/tests/core_test.py`: tests given Sids for **core attributes** (types, fields, parent(), etc.)
+- `spil_hamlet_conf/tests/path_test.py`: tests given Sids **path resolving and path related attributes** (path())
+- `spil_hamlet_conf/tests/data_test.py`: tests given Sids **data access attributes** (exists(), children(), get_last(), etc.)
+- `spil_hamlet_conf/tests/finder_test.py`: uses given Sids to build random search Sids, and **tests Finders**.
Do not hesitate to adapt this test file, depending on the finders you will use.
*(Note that you could also manually create a python script with example / test searches)*.
-- `hamlet_conf/tests/quicktest.py`: sandbox-like file to quickly test various things.
+- `spil_hamlet_conf/tests/quicktest.py`: sandbox-like file to quickly test various things.
*(more tests work in progress)*
diff --git a/docs/wait.md b/docs/wait.md
deleted file mode 100644
index c70c2cd..0000000
--- a/docs/wait.md
+++ /dev/null
@@ -1,6 +0,0 @@
-
-
-
-This documentation is work in progress.
-
-The new version of spil has not been released yet.
\ No newline at end of file
diff --git a/pyproject.toml b/pyproject.toml
index 9578d6b..f31dd73 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -5,15 +5,13 @@ build-backend = "hatchling.build"
[tool.hatch.build]
include = [
"spil",
- "spil_plugins",
- "spil_tests",
- "hamlet_conf"
+ "spil_hamlet_conf"
]
exclude = [".gitignore"]
[project]
name = "spil"
-version = "0.1.0"
+version = "0.1.0rc2"
description = "The Simple Pipeline Lib."
readme = "README.md"
authors = [{ name = "Michael Haussmann", email = "spil@xeo.info" }]
@@ -37,14 +35,15 @@ dependencies = [
"Fileseq",
"future",
"logzero",
- "typing-extensions"
+ "typing-extensions",
+ "codetiming",
]
requires-python = ">=3.7"
[project.optional-dependencies]
-dev = ["codetiming", "pytest"] # "Faker"
-qc = ["mypy", "black", "flake8", "isort", "refurb"]
-sg = ["shotgun-api3", "Unidecode"]
+dev = ["pytest"] # "Faker"
+qc = ["mypy", "black", "flake8", "isort", "refurb"] # Code Quality
+sg = ["shotgun-api3", "Unidecode"] # SG plugin
[project.urls]
"Homepage" = "https://github.com/MichaelHaussmann/spil"
diff --git a/spil/conf/global_conf.py b/spil/conf/global_conf.py
index fea5a8a..070f7d3 100644
--- a/spil/conf/global_conf.py
+++ b/spil/conf/global_conf.py
@@ -23,8 +23,8 @@
)
application_path = Path(__file__).parent.parent
-default_sid_conf_path = str(application_path.parent / "hamlet_conf")
-default_sid_conf_data_path = str(application_path.parent / "hamlet_conf" / "data")
+default_sid_conf_path = str(application_path.parent / "spil_hamlet_conf")
+default_sid_conf_data_path = str(application_path.parent / "spil_hamlet_conf" / "data")
# automatic replacement to INFO and WARN in BETA and PROD
loglevel = 80
@@ -62,7 +62,7 @@
The configuration module "{module}" was not found.
- Ensure to either include the demo "hamlet_conf" in your python path,
+ Ensure to either include the demo "spil_hamlet_conf" in your python path,
or create your own "{module}" and add its folder to the python path.
(If you are running a py.test edit the SPIL_CONF_PATH variable in tests/test_00_init.py to match a python path.)
diff --git a/spil/sid/core/query_helper.py b/spil/sid/core/query_helper.py
index dbd6ff1..96cca99 100644
--- a/spil/sid/core/query_helper.py
+++ b/spil/sid/core/query_helper.py
@@ -139,7 +139,7 @@ def apply_query(string, query=None, type=None, fields=None):
This is a problem in case of a read Sid, which could be poly-typed.
Examples.
- (Note that the examples depend on "hamlet_conf" example configuration. Failure may be due to non matching config)
+ (Note that the examples depend on "spil_hamlet_conf" example configuration. Failure may be due to non matching config)
Query updates the sequence:
>>> apply_query('hamlet/s/sq010', query='sequence=sq030', type='shot__sequence', fields={'project':'hamlet','type':'s','sequence':'sq010'})
diff --git a/spil/sid/read/finders/find_cache.py b/spil/sid/read/finders/find_cache.py
index 9485f8a..e4b5d76 100644
--- a/spil/sid/read/finders/find_cache.py
+++ b/spil/sid/read/finders/find_cache.py
@@ -1,6 +1,26 @@
"""
+This file is part of SPIL, The Simple Pipeline Lib.
+(C) copyright 2019-2023 Michael Haussmann, spil@xeo.info
+
+SPIL is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+
+SPIL is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License along with SPIL.
+If not, see .
+"""
+"""
+
+
+
+
+
THIS IS WORK IN PROGRESS.
+
+
+
+
"""
from __future__ import annotations
@@ -41,11 +61,7 @@ def get_sidcache(data_search, data_source, cache_file=None, name=None):
Each sid_cache_file is handled by its own, single, SidCache instance.
A SidCache object handles only one sid_cache_file.
For example:
- a = SidCache(cache_file='/shots.txt')
- b = SidCache(cache_file='/shots.txt')
- c = SidCache(cache_file='/assets.txt')
- a and b will be the same object instance, and access the same data.
- c will be another object.
+
Note that the file can be accessed concurrently from different python instances, from different machines.
SidCache still needs to ensure threadsafety.
@@ -106,8 +122,8 @@ class FindInCache(Finder):
For 'hamlet/s/*/*/*' (sequences, shots and tasks for hamlet project)
Originally loaded from FS (File Search).
```
- >>> sidcache = get_sidcache(sid_cache_file='/temp/hamlet_shots.txt', data_search='hamlet/s/*/*', data_source=FS())
- >>> sidcache.find('hamlet/s/*')
+ ... sidcache = get_sidcache(sid_cache_file='/temp/hamlet_shots.txt', data_search='hamlet/s/*/*', data_source=FS())
+ ... sidcache.find('hamlet/s/*')
```
TODO: optional multiple data sources
TODO: ttl for warmup
diff --git a/spil/sid/sid.py b/spil/sid/sid.py
index 87bfb80..c8782aa 100644
--- a/spil/sid/sid.py
+++ b/spil/sid/sid.py
@@ -657,8 +657,8 @@ def path(self, config: Optional[str] = None) -> Path | None:
True
>>> path = Sid('hamlet/a/char/ophelia/model/v001/w/ma').path()
- >>> path.relative_to(conf.default_sid_conf_path).as_posix() # to be location and os independent
- 'data/testing/SPIL_PROJECTS/LOCAL/PROJECTS/HAMLET/PROD/ASSETS/char/ophelia/model/v001/char_ophelia_model_WORK_v001.ma'
+ >>> path.relative_to(conf.default_sid_conf_data_path).as_posix() # to be location and os independent
+ 'testing/SPIL_PROJECTS/LOCAL/PROJECTS/HAMLET/PROD/ASSETS/char/ophelia/model/v001/char_ophelia_model_WORK_v001.ma'
>>> Sid('bla/bla').path()
@@ -948,7 +948,7 @@ class Sid(DataSid):
>>> Sid(fields={'project': 'hamlet', 'sequence': 'sq010', 'type': 's'}) # fields dict
Sid('shot__sequence:hamlet/s/sq010')
- >>> path = Path(conf.default_sid_conf_path) / "data/testing/SPIL_PROJECTS/LOCAL/PROJECTS/HAMLET/PROD/ASSETS/char/ophelia/model/v001/char_ophelia_model_WORK_v001.ma"
+ >>> path = Path(conf.default_sid_conf_data_path) / "testing/SPIL_PROJECTS/LOCAL/PROJECTS/HAMLET/PROD/ASSETS/char/ophelia/model/v001/char_ophelia_model_WORK_v001.ma"
>>> Sid(path=path) # path (default config) # TODO: any config
Sid('asset__file:hamlet/a/char/ophelia/model/v001/w/ma')
@@ -967,11 +967,8 @@ class Sid(DataSid):
if __name__ == "__main__":
- from spil_tests import stop
- from pprint import pprint
- from spil.util.log import debug, setLevel, INFO, DEBUG, info
-
- setLevel(INFO)
+ # from spil.util.log import setLevel, INFO
+ # setLevel(INFO)
s = Sid("hamlet/s/sq010/sh0010/anim")
print(s.exists())
diff --git a/spil_tests/__init__.py b/spil/tests/__init__.py
similarity index 100%
rename from spil_tests/__init__.py
rename to spil/tests/__init__.py
diff --git a/spil_plugins/crud/__init__.py b/spil/tests/config_checks/__init__.py
similarity index 100%
rename from spil_plugins/crud/__init__.py
rename to spil/tests/config_checks/__init__.py
diff --git a/spil_tests/config_checks/check_01_sid_config.py b/spil/tests/config_checks/check_01_sid_config.py
similarity index 98%
rename from spil_tests/config_checks/check_01_sid_config.py
rename to spil/tests/config_checks/check_01_sid_config.py
index aeda9c5..760e212 100644
--- a/spil_tests/config_checks/check_01_sid_config.py
+++ b/spil/tests/config_checks/check_01_sid_config.py
@@ -24,7 +24,7 @@
from spil.conf import sid_templates # type: ignore
from spil.util.log import DEBUG, ERROR, get_logger
-log = get_logger('spil_tests')
+log = get_logger('tests')
log.setLevel(DEBUG)
diff --git a/spil_tests/config_checks/check_02_path_config.py b/spil/tests/config_checks/check_02_path_config.py
similarity index 99%
rename from spil_tests/config_checks/check_02_path_config.py
rename to spil/tests/config_checks/check_02_path_config.py
index caecff3..3d618f9 100644
--- a/spil_tests/config_checks/check_02_path_config.py
+++ b/spil/tests/config_checks/check_02_path_config.py
@@ -27,7 +27,7 @@
path_templates = path_config.path_templates
from spil.util.log import DEBUG, ERROR, get_logger
-log = get_logger('spil_tests')
+log = get_logger('tests')
log.setLevel(DEBUG)
diff --git a/spil_tests/config_checks/check_03_config_coverage.py b/spil/tests/config_checks/check_03_config_coverage.py
similarity index 97%
rename from spil_tests/config_checks/check_03_config_coverage.py
rename to spil/tests/config_checks/check_03_config_coverage.py
index 33da6fc..40ac4c9 100644
--- a/spil_tests/config_checks/check_03_config_coverage.py
+++ b/spil/tests/config_checks/check_03_config_coverage.py
@@ -11,7 +11,7 @@
from spil.conf import sid_templates # type: ignore
from spil.util.log import DEBUG, ERROR, get_logger
-log = get_logger('spil_tests')
+log = get_logger('tests')
log.setLevel(DEBUG)
diff --git a/spil_tests/config_checks/__init__.py b/spil/tests/feature_tests/__init__.py
similarity index 100%
rename from spil_tests/config_checks/__init__.py
rename to spil/tests/feature_tests/__init__.py
diff --git a/spil_tests/feature_tests/test_lru.py b/spil/tests/feature_tests/test_lru.py
similarity index 100%
rename from spil_tests/feature_tests/test_lru.py
rename to spil/tests/feature_tests/test_lru.py
diff --git a/spil_tests/feature_tests/__init__.py b/spil/tests/prep/__init__.py
similarity index 100%
rename from spil_tests/feature_tests/__init__.py
rename to spil/tests/prep/__init__.py
diff --git a/spil_tests/prep/build_searches.py b/spil/tests/prep/build_searches.py
similarity index 100%
rename from spil_tests/prep/build_searches.py
rename to spil/tests/prep/build_searches.py
diff --git a/spil_tests/prep/make_mock_fs.py b/spil/tests/prep/make_mock_fs.py
similarity index 100%
rename from spil_tests/prep/make_mock_fs.py
rename to spil/tests/prep/make_mock_fs.py
diff --git a/spil_tests/prep/parse_sids_from_fs.py b/spil/tests/prep/parse_sids_from_fs.py
similarity index 97%
rename from spil_tests/prep/parse_sids_from_fs.py
rename to spil/tests/prep/parse_sids_from_fs.py
index 320591b..b9ae5b2 100644
--- a/spil_tests/prep/parse_sids_from_fs.py
+++ b/spil/tests/prep/parse_sids_from_fs.py
@@ -13,7 +13,7 @@
"""
-# from spil_tests.utils import init # needs to be before spil.conf import
+# from tests.utils import init # needs to be before spil.conf import
from spil.util.log import debug, info
from spil import Sid
diff --git a/spil_tests/prep/save_sid_list_to_file.py b/spil/tests/prep/save_sid_list_to_file.py
similarity index 100%
rename from spil_tests/prep/save_sid_list_to_file.py
rename to spil/tests/prep/save_sid_list_to_file.py
diff --git a/spil_tests/utils/__init__.py b/spil/tests/utils/__init__.py
similarity index 100%
rename from spil_tests/utils/__init__.py
rename to spil/tests/utils/__init__.py
diff --git a/spil_tests/utils/dualsearch_ab_tester.py b/spil/tests/utils/dualsearch_ab_tester.py
similarity index 93%
rename from spil_tests/utils/dualsearch_ab_tester.py
rename to spil/tests/utils/dualsearch_ab_tester.py
index 396a022..2f31d83 100644
--- a/spil_tests/utils/dualsearch_ab_tester.py
+++ b/spil/tests/utils/dualsearch_ab_tester.py
@@ -11,8 +11,8 @@
If not, see .
"""
from spil.util.exception import raiser
-from spil.util.log import DEBUG, ERROR, get_logger
-from spil_tests import Timer
+from spil.util.log import get_logger
+from spil.tests import Timer
log = get_logger("spil_tests")
@@ -65,8 +65,8 @@ def check_search_ab(searches, finderA, finderB, as_sid=False, raise_problems=Fal
if __name__ == "__main__":
- from spil import FindInList, FindInPaths, FindInAll
- from spil.util.log import setLevel, ERROR, DEBUG
+ from spil import FindInPaths, FindInAll
+ from spil.util.log import setLevel, ERROR
setLevel(ERROR)
diff --git a/spil_tests/utils/search_tester.py b/spil/tests/utils/search_tester.py
similarity index 93%
rename from spil_tests/utils/search_tester.py
rename to spil/tests/utils/search_tester.py
index 493fca7..c978673 100644
--- a/spil_tests/utils/search_tester.py
+++ b/spil/tests/utils/search_tester.py
@@ -11,9 +11,9 @@
If not, see .
"""
from spil import Sid, FindInPaths, SpilException
-from spil_tests.utils.sid_full_tester import test_full_sid
-from spil.util.log import DEBUG, ERROR, get_logger
-from spil_tests import Timer
+from spil.tests.utils.sid_full_tester import test_full_sid
+from spil.util.log import get_logger
+from spil.tests import Timer
log = get_logger("spil_tests")
@@ -76,7 +76,7 @@ def check_searches_in_finder(searches, finder=None, as_sid=True, do_log=True, do
if __name__ == "__main__":
- from spil.util.log import setLevel, ERROR, DEBUG
+ from spil.util.log import setLevel, ERROR
setLevel(ERROR)
diff --git a/spil_tests/utils/sid_core_tester.py b/spil/tests/utils/sid_core_tester.py
similarity index 98%
rename from spil_tests/utils/sid_core_tester.py
rename to spil/tests/utils/sid_core_tester.py
index b8e802d..5a0f35f 100644
--- a/spil_tests/utils/sid_core_tester.py
+++ b/spil/tests/utils/sid_core_tester.py
@@ -133,6 +133,7 @@ def check_typed_sids(sids, reraise=True, replace=None):
setLevel(INFO)
- from scripts.example_sids import sids
+ # from hamlet_scripts.example_sids import sids
+ sids = []
check_typed_sids(sids)
diff --git a/spil_tests/utils/sid_data_tester.py b/spil/tests/utils/sid_data_tester.py
similarity index 100%
rename from spil_tests/utils/sid_data_tester.py
rename to spil/tests/utils/sid_data_tester.py
diff --git a/spil_tests/utils/sid_full_tester.py b/spil/tests/utils/sid_full_tester.py
similarity index 100%
rename from spil_tests/utils/sid_full_tester.py
rename to spil/tests/utils/sid_full_tester.py
diff --git a/spil_tests/utils/sid_path_tester.py b/spil/tests/utils/sid_path_tester.py
similarity index 91%
rename from spil_tests/utils/sid_path_tester.py
rename to spil/tests/utils/sid_path_tester.py
index fc7eea8..a65f8ab 100644
--- a/spil_tests/utils/sid_path_tester.py
+++ b/spil/tests/utils/sid_path_tester.py
@@ -9,7 +9,6 @@
You should have received a copy of the GNU Lesser General Public License along with SPIL.
If not, see .
-
"""
from spil.util.log import DEBUG, get_logger
from spil import Sid, SpilException
@@ -45,7 +44,9 @@ def check_path_sid(s, configs=[], reraise=True, replace=None):
except AssertionError:
msg = "Sid path is ambiguous. {}\nsid: {}\nsid.path(): {}\nSid(path=sid.path()): {}\n".format(
"(Sid is a search, this may be normal)." if sid.is_search() else "",
- sid, sid.path(), Sid(path=sid.path())
+ sid,
+ sid.path(),
+ Sid(path=sid.path()),
)
log.warning(msg)
try:
@@ -54,7 +55,10 @@ def check_path_sid(s, configs=[], reraise=True, replace=None):
except AssertionError:
msg = "Sid path is ambiguous. {}\nsid: {}\nsid.path(): {}\nSid(path=sid.path()): {}\nSid(path=sid.path()).path(): {}".format(
"(Sid is a search, this may be normal)." if sid.is_search() else "",
- sid, sid.path(), Sid(path=sid.path()), Sid(path=sid.path()).path()
+ sid,
+ sid.path(),
+ Sid(path=sid.path()),
+ Sid(path=sid.path()).path(),
)
log.warning(msg)
else:
@@ -92,7 +96,7 @@ def check_path_sids(sids, configs=[], reraise=True, replace=None):
return
for i, s in enumerate(sids):
- log.info('*' * 75)
+ log.info("*" * 75)
log.info("----------------------------------------- {}".format(i))
check_path_sid(s, configs=configs, reraise=reraise, replace=replace)
@@ -103,6 +107,7 @@ def check_path_sids(sids, configs=[], reraise=True, replace=None):
setLevel(INFO)
- from scripts.example_sids import sids
+ # from hamlet_scripts.example_sids import sids
+ sids = []
check_path_sids(sids)
diff --git a/hamlet_conf/data/templates/empty.ma b/spil_hamlet_conf/data/templates/empty.ma
similarity index 100%
rename from hamlet_conf/data/templates/empty.ma
rename to spil_hamlet_conf/data/templates/empty.ma
diff --git a/hamlet_conf/data/templates/empty.mb b/spil_hamlet_conf/data/templates/empty.mb
similarity index 100%
rename from hamlet_conf/data/templates/empty.mb
rename to spil_hamlet_conf/data/templates/empty.mb
diff --git a/hamlet_conf/data/testing/hamlet.sids.txt b/spil_hamlet_conf/data/testing/hamlet.sids.txt
similarity index 100%
rename from hamlet_conf/data/testing/hamlet.sids.txt
rename to spil_hamlet_conf/data/testing/hamlet.sids.txt
diff --git a/spil_tests/prep/__init__.py b/spil_hamlet_conf/hamlet_plugins/__init__.py
similarity index 100%
rename from spil_tests/prep/__init__.py
rename to spil_hamlet_conf/hamlet_plugins/__init__.py
diff --git a/spil_plugins/crud/next_get.py b/spil_hamlet_conf/hamlet_plugins/next_get.py
similarity index 100%
rename from spil_plugins/crud/next_get.py
rename to spil_hamlet_conf/hamlet_plugins/next_get.py
diff --git a/hamlet_conf/scripts/example_sids.py b/spil_hamlet_conf/hamlet_scripts/example_sids.py
similarity index 85%
rename from hamlet_conf/scripts/example_sids.py
rename to spil_hamlet_conf/hamlet_scripts/example_sids.py
index 79bc54c..26c437f 100644
--- a/hamlet_conf/scripts/example_sids.py
+++ b/spil_hamlet_conf/hamlet_scripts/example_sids.py
@@ -7,8 +7,7 @@
This shows all the types covered by the test sids,
and which types are not.
"""
-from scripts.save_examples_to_file import sid_file
-from spil_tests.config_checks.check_03_config_coverage import test_config_coverage
+from hamlet_scripts.save_examples_to_file import sid_file
sids = []
with sid_file.open() as f:
@@ -17,15 +16,14 @@
if __name__ == "__main__":
import sys
- from spil_tests import Timer
+ from spil.tests import Timer
- from spil.util.log import DEBUG, ERROR, get_logger
+ from spil.util.log import DEBUG, get_logger
log = get_logger("spil_tests")
log.setLevel(DEBUG)
from spil import Sid
- from pprint import pprint
print("Start")
# pprint(sids)
diff --git a/hamlet_conf/scripts/generate_example_sids.py b/spil_hamlet_conf/hamlet_scripts/generate_example_sids.py
similarity index 96%
rename from hamlet_conf/scripts/generate_example_sids.py
rename to spil_hamlet_conf/hamlet_scripts/generate_example_sids.py
index 8017ece..dbbfbee 100644
--- a/hamlet_conf/scripts/generate_example_sids.py
+++ b/spil_hamlet_conf/hamlet_scripts/generate_example_sids.py
@@ -19,12 +19,11 @@
from math import ceil
from time import sleep
-import spil # config path bootstrap
from spil_sid_conf import projects, asset_types, asset_tasks
from spil_sid_conf import extensions_scene, extensions_movie, shot_tasks, extensions_cache
from logging import debug
-from spil_tests.config_checks.check_03_config_coverage import test_config_coverage
+from spil.tests.config_checks.check_03_config_coverage import test_config_coverage
do_intermediates = True # If this is False, only leave paths will be generated. See: LS extrapolate.
repeat_times = 1 # 3
@@ -147,15 +146,15 @@ def append(sids, data):
if __name__ == '__main__':
import sys
- # from spil_tests import Timer
+ # from tests import Timer
- from spil.util.log import DEBUG, ERROR, get_logger
+ from spil.util.log import DEBUG, get_logger
log = get_logger("spil_tests")
log.setLevel(DEBUG)
from spil import Sid
- from pprint import pprint
+
print('Start')
#pprint(sids)
diff --git a/hamlet_conf/scripts/save_examples_to_file.py b/spil_hamlet_conf/hamlet_scripts/save_examples_to_file.py
similarity index 62%
rename from hamlet_conf/scripts/save_examples_to_file.py
rename to spil_hamlet_conf/hamlet_scripts/save_examples_to_file.py
index ec10a18..a795e8c 100644
--- a/hamlet_conf/scripts/save_examples_to_file.py
+++ b/spil_hamlet_conf/hamlet_scripts/save_examples_to_file.py
@@ -6,12 +6,11 @@
"""
from pathlib import Path
+from spil import conf
+from hamlet_scripts.generate_example_sids import sids # type: ignore
+from spil.tests.prep.save_sid_list_to_file import write_sids_to_file
-import spil # default config path bootstrap
-from scripts.generate_example_sids import sids # type: ignore
-from spil_tests.prep.save_sid_list_to_file import write_sids_to_file
-
-sid_file = Path(__file__).parent.parent / "data" / "testing" / "hamlet.sids.txt"
+sid_file = Path(conf.default_sid_conf_data_path) / "testing" / "hamlet.sids.txt"
if not sid_file.exists():
print(f"Creating Sid test file: {sid_file}")
diff --git a/hamlet_conf/scripts/save_examples_to_mock_fs.py b/spil_hamlet_conf/hamlet_scripts/save_examples_to_mock_fs.py
similarity index 68%
rename from hamlet_conf/scripts/save_examples_to_mock_fs.py
rename to spil_hamlet_conf/hamlet_scripts/save_examples_to_mock_fs.py
index a9f188b..1d033bc 100644
--- a/hamlet_conf/scripts/save_examples_to_mock_fs.py
+++ b/spil_hamlet_conf/hamlet_scripts/save_examples_to_mock_fs.py
@@ -5,14 +5,13 @@
according to the path configs.
"""
-import spil # default config path bootstrap
import spil_data_conf
-from scripts.example_sids import sids
-from spil_tests.prep.make_mock_fs import generate_mock_fs
+from hamlet_scripts.example_sids import sids
+from spil.tests.prep.make_mock_fs import generate_mock_fs
if __name__ == "__main__":
- from spil.util.log import setLevel, INFO, DEBUG, info
+ from spil.util.log import setLevel, INFO
print("")
print("Tests start")
diff --git a/hamlet_conf/tests/core_test.py b/spil_hamlet_conf/hamlet_tests/core_test.py
similarity index 68%
rename from hamlet_conf/tests/core_test.py
rename to spil_hamlet_conf/hamlet_tests/core_test.py
index e466d51..191226d 100644
--- a/hamlet_conf/tests/core_test.py
+++ b/spil_hamlet_conf/hamlet_tests/core_test.py
@@ -5,9 +5,8 @@
"""
import random
-import spil # config path bootstrap
-from scripts.example_sids import sids # type: ignore
-from spil_tests.utils.sid_core_tester import check_typed_sids
+from hamlet_scripts.example_sids import sids # type: ignore
+from spil.tests.utils.sid_core_tester import check_typed_sids
from spil.util.log import setLevel, INFO
def test_typed_sids():
diff --git a/hamlet_conf/tests/data_test.py b/spil_hamlet_conf/hamlet_tests/data_test.py
similarity index 81%
rename from hamlet_conf/tests/data_test.py
rename to spil_hamlet_conf/hamlet_tests/data_test.py
index f1d1284..c1f4772 100644
--- a/hamlet_conf/tests/data_test.py
+++ b/spil_hamlet_conf/hamlet_tests/data_test.py
@@ -9,8 +9,8 @@
import random
# import spil # config path bootstrap
-from scripts.example_sids import sids # type: ignore
-from spil_tests.utils.sid_data_tester import check_data_sids
+from hamlet_scripts.example_sids import sids # type: ignore
+from spil.tests.utils.sid_data_tester import check_data_sids
def test_data_sids():
diff --git a/hamlet_conf/tests/finder_test.py b/spil_hamlet_conf/hamlet_tests/finder_test.py
similarity index 82%
rename from hamlet_conf/tests/finder_test.py
rename to spil_hamlet_conf/hamlet_tests/finder_test.py
index 889dd83..6257f14 100644
--- a/hamlet_conf/tests/finder_test.py
+++ b/spil_hamlet_conf/hamlet_tests/finder_test.py
@@ -5,16 +5,15 @@
"""
import random
-import spil # default config path bootstrap
-from scripts.example_sids import sids # type: ignore
+from hamlet_scripts.example_sids import sids # type: ignore
# from spil_plugins.sg.find_sg import FindInSG
from spil import FindInList, FindInPaths, FindInAll
-from spil_tests.prep.build_searches import from_sid_build_searches
-from spil_tests.utils.search_tester import check_searches_in_finder
+from spil.tests.prep.build_searches import from_sid_build_searches
+from spil.tests.utils.search_tester import check_searches_in_finder
from spil.util.log import get_logger
-from spil_tests.utils.dualsearch_ab_tester import check_search_ab
+from spil.tests.utils.dualsearch_ab_tester import check_search_ab
log = get_logger("spil_tests", color=False)
@@ -57,6 +56,6 @@ def test_finders():
if __name__ == "__main__":
- from spil.util.log import setLevel, INFO
+ from spil.util.log import INFO
log.setLevel(INFO)
test_finders()
diff --git a/hamlet_conf/tests/path_test.py b/spil_hamlet_conf/hamlet_tests/path_test.py
similarity index 73%
rename from hamlet_conf/tests/path_test.py
rename to spil_hamlet_conf/hamlet_tests/path_test.py
index 3534b43..0e8b563 100644
--- a/hamlet_conf/tests/path_test.py
+++ b/spil_hamlet_conf/hamlet_tests/path_test.py
@@ -6,10 +6,9 @@
"""
import random
-import spil # config path bootstrap
import spil_data_conf # type: ignore
-from scripts.example_sids import sids # type: ignore
-from spil_tests.utils.sid_path_tester import check_path_sids
+from hamlet_scripts.example_sids import sids # type: ignore
+from spil.tests.utils.sid_path_tester import check_path_sids
def test_path_sids():
diff --git a/hamlet_conf/tests/quicktest.py b/spil_hamlet_conf/hamlet_tests/quicktest.py
similarity index 92%
rename from hamlet_conf/tests/quicktest.py
rename to spil_hamlet_conf/hamlet_tests/quicktest.py
index 2e517c8..006cb06 100644
--- a/hamlet_conf/tests/quicktest.py
+++ b/spil_hamlet_conf/hamlet_tests/quicktest.py
@@ -6,9 +6,9 @@
if __name__ == "__main__":
- from spil_tests import stop
+ from spil.tests import stop
from pprint import pprint
- from spil.util.log import debug, setLevel, INFO, DEBUG, info
+ from spil.util.log import setLevel, INFO
setLevel(INFO)
diff --git a/hamlet_conf/spil_data_conf.py b/spil_hamlet_conf/spil_data_conf.py
similarity index 72%
rename from hamlet_conf/spil_data_conf.py
rename to spil_hamlet_conf/spil_data_conf.py
index b762a7c..3bbcf6a 100644
--- a/hamlet_conf/spil_data_conf.py
+++ b/spil_hamlet_conf/spil_data_conf.py
@@ -5,7 +5,7 @@
It is ingested by spil.conf.data_conf_load, which reads it into spil.conf.
-*This is work in progress*
+*This is beta / work in progress*
The data config specification is subject to change.
(without affecting spil's API).
@@ -13,33 +13,18 @@
"""
from __future__ import annotations
from pathlib import Path
-# attribute getters
-# cachable_attributes (by getter / by type / with TTL - for example publish file size, date, owner
-# sid_cache_path = '/home/mh/PycharmProjects/spil2/hamlet_conf/data/caches'
-# sid_cache_folder = sid_cache_path
-
-# ---------------------------------------------
+#########################################################
# Path config
path_configs = {'local': 'spil_fs_conf',
'server': 'spil_fs_server_conf'
}
default_path_config = 'local'
-
-# WriteToPaths & GetFromPaths
-path_data_suffix = '.data.json'
-# WriteToPath: create. If a template exists for a suffix (extension), we copy it.
-create_file_using_template = { # type: ignore
- # 'ma': '.../empty.ma',
- # 'mb': '.../empty.mb'
-}
-# WriteToPath: create. If no template exists, we create an empty file with path.touch(), if create_using_touch is True
-create_file_using_touch = True
-# If nothing of these is set, we do not create a file.
-# ---------------------------------------------
-# End Path config
+#########################################################
+#########################################################
+# Config for FindInAll
def get_finder_for(search_sid, config=None): # get finder by Sid and optional config
"""
Configuration used by FindInAll, to define which Finder is used for a given Search Sid.
@@ -47,7 +32,7 @@ def get_finder_for(search_sid, config=None): # get finder by Sid and optional c
For a given Search Sid, and optional config,
looks up the Search Sid's type and the matching Finder, as defined in an ad-hoc dict.
- The config allows to create multiple FindInAll instances / behaviours.
+ The "config" allows to create multiple FindInAll instances / behaviours.
(similar to the FindInPaths where we Find in different file systems)
Args:
@@ -55,7 +40,7 @@ def get_finder_for(search_sid, config=None): # get finder by Sid and optional c
config: optional config to be able to define multiple FindInAll instances.
Returns:
- A Finder object for this search.
+ A Finder instance for this search.
"""
# type: ignore
from spil_sid_conf import projects, asset_types # type: ignore
@@ -84,20 +69,32 @@ def get_finder_for(search_sid, config=None): # get finder by Sid and optional c
return None
+#########################################################
+# Config for GetFromAll
def get_getter_for(sid, attribute=None, config=None):
"""
+ Configuration used by GetFromAll, to define which Getter is used for a given Sid or Search Sid.
- *This is work in progress*
+ For a given Sid, and optional attribute and config,
+ looks up the Sid's type and the matching Getter, as defined in an ad-hoc dict.
- For a given attribute, looks up the matching attribute_getters, as defined in a dict.
- Returned value is a getter function.
+ The "config" allows to create multiple GetFromAll instances / behaviours.
+ (similar to the FindInPaths where we Get from different file systems)
+
+ The function may return None, in which case the getter will not return anything.
+
+ Args:
+ sid: sid that is queried by GetFromAll
+ attribute: optional attribute name
+ config: optional config to be able to define multiple GetFromAll instances.
- Currently, the sid argument is not used.
+ Returns:
+ A Getter instance for this search, or None is none is defined.
"""
# from spil_action.libs.files import get_comment, get_size, get_time
from spil import Getter, GetFromPaths
# from spil_plugins.sg.get_sg import GetFromSG
- from spil_plugins.crud.next_get import NextGetter
+ from hamlet_plugins.next_get import NextGetter
attribute_getters = {
"next.version": NextGetter()
@@ -135,11 +132,28 @@ def get_getter_for(sid, attribute=None, config=None):
def get_writer_for(sid):
"""
+ Configuration used by WriteToAll, to define which Writer is used for a given Sid.
+
*This is work in progress*
"""
raise NotImplementedError("get_writer_for is not implemented")
+#########################################################
+# Config for WriteToPaths & GetFromPaths
+#
+path_data_suffix = '.data.json'
+# WriteToPath: create. If a template exists for a suffix (extension), we copy it.
+create_file_using_template = { # type: ignore
+ # 'ma': '.../empty.ma',
+ # 'mb': '.../empty.mb'
+}
+# WriteToPath: create. If no template exists, we create an empty file with path.touch(), if create_using_touch is True
+create_file_using_touch = True
+# If nothing of these is set, we do not create a file.
+
+
+# Used by WriteToPaths and GetFromPaths
def get_data_json_path(sid_path: Path) -> Path:
"""
For a given Sid path, returns the path of a hidden data "sidecar" json file.
@@ -156,3 +170,7 @@ def get_data_json_path(sid_path: Path) -> Path:
# TODO: add file rotation
data_path = sid_path.with_name('.' + sid_path.name).with_suffix(path_data_suffix)
return data_path
+
+# End of Config for WriteToPaths & GetFromPaths
+#########################################################
+
diff --git a/hamlet_conf/spil_fs_conf.py b/spil_hamlet_conf/spil_fs_conf.py
similarity index 100%
rename from hamlet_conf/spil_fs_conf.py
rename to spil_hamlet_conf/spil_fs_conf.py
diff --git a/hamlet_conf/spil_fs_server_conf.py b/spil_hamlet_conf/spil_fs_server_conf.py
similarity index 100%
rename from hamlet_conf/spil_fs_server_conf.py
rename to spil_hamlet_conf/spil_fs_server_conf.py
diff --git a/hamlet_conf/spil_sg_conf.py b/spil_hamlet_conf/spil_sg_conf.py
similarity index 100%
rename from hamlet_conf/spil_sg_conf.py
rename to spil_hamlet_conf/spil_sg_conf.py
diff --git a/hamlet_conf/spil_sid_conf.py b/spil_hamlet_conf/spil_sid_conf.py
similarity index 100%
rename from hamlet_conf/spil_sid_conf.py
rename to spil_hamlet_conf/spil_sid_conf.py