diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml new file mode 100644 index 0000000..bfa7b0d --- /dev/null +++ b/.github/workflows/publish_pypi.yml @@ -0,0 +1,43 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Publish To Test PyPI + +on: + workflow_dispatch: +# release: +# types: [published] + +permissions: + contents: read + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.7' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + - name: Build package + run: python -m build + - name: Publish package to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 +# if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') +# uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} +# repository-url: https://test.pypi.org/legacy/ diff --git a/.gitignore b/.gitignore index 710e2d0..dfa1e0a 100644 --- a/.gitignore +++ b/.gitignore @@ -13,8 +13,8 @@ venv27 /.limbo/ /docs-wip /docs/_build -/hamlet_conf/data/testing/SPIL_PROJECTS -/hamlet_conf/data/caches/* +/spil_hamlet_conf/data/testing/SPIL_PROJECTS +/spil_hamlet_conf/data/caches/* /memo.md *wip *_secret.py diff --git a/README.md b/README.md index c6e8474..40aee98 100644 --- a/README.md +++ b/README.md @@ -319,10 +319,11 @@ More about installation, configuration and testing: [spil.readthedocs.io](https: Spil thrives to be used interactively. It's performance depends on the data sources that are used. -- Spil ships with a configurable FindInCache to handle data that changes rarely (projects, sequences, asset types). -- String / Sid Resolves are internally stored in a lru_cache -- searches use generators - +- Spil uses a `FindInConstants` class to handle configurable data that mostly doesn't change (types, asset types) +- Spil ships with a configurable `FindInCache` class to handle data that changes rarely (projects, sequences, assets, etc.). + (not production ready in current release) +- String / Sid Resolves are internally stored in a custom lru_cache +- `Finders` use generators ## Concepts diff --git a/conftest.py b/conftest.py index a984706..afb01d9 100644 --- a/conftest.py +++ b/conftest.py @@ -5,11 +5,10 @@ """ from contextlib import suppress -import spil # default config path bootstrap from spil import SpilException import spil_data_conf -from scripts.example_sids import sids -from spil_tests.prep.make_mock_fs import generate_mock_fs +from hamlet_scripts.example_sids import sids +from spil.tests.prep.make_mock_fs import generate_mock_fs def pytest_sessionstart(session): diff --git a/docs/background.md b/docs/background.md index ee26dc9..9551ffb 100644 --- a/docs/background.md +++ b/docs/background.md @@ -97,8 +97,9 @@ Spil aims to be : flexible, pragmatic, simple - and reliable. Spil thrives to be used interactively. It's performance depends on the data sources that are used. -- Spil ships with a configurable `FindInCache` class to handle data that changes rarely (projects, sequences, assets, etc.). -- Spil uses a `FindInConstant` class to handle configurable data that mostly doesn't change (types, asset types) +- Spil uses a `FindInConstants` class to handle configurable data that mostly doesn't change (types, asset types) +- Spil ships with a configurable `FindInCache` class to handle data that changes rarely (projects, sequences, assets, etc.). + (not production ready in current release) - String / Sid Resolves are internally stored in a custom lru_cache - `Finders` use generators diff --git a/docs/conf.py b/docs/conf.py index c70d819..68a7a7c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,7 +16,7 @@ here = Path(__file__) root = here.parent.parent venv = root / 'venv' / 'lib' / 'python3.7' / 'site-packages' -spil_conf = root / 'hamlet_conf' +spil_conf = root / 'spil_hamlet_conf' print(root) sys.path.append(str(root)) sys.path.append(str(venv)) diff --git a/docs/configuration.md b/docs/configuration.md index 4e1db47..855b398 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -23,15 +23,15 @@ Best way to get started: ## Included Demo configuration To be able to use Spil, the configuration must be added to the python path. -Spil ships with an example configuration folder named `hamlet_conf`, for a hypothetical *"hamlet"* project. +Spil ships with an example configuration folder named `spil_hamlet_conf`, for a hypothetical *"hamlet"* project. -If no configuration is found, the `hamlet_conf` folder is added to the python path during spil import. +If no configuration is found, the `spil_hamlet_conf` folder is added to the python path during spil import. ### Adapting the demo config The easiest way to start, is to adapt the existing config to your needs. -- Copy or move the `hamlet_conf` folder to a location of your choice, and add it to your python path. +- Copy or move the `spil_hamlet_conf` folder to a location of your choice, and add it to your python path. - You can rename it. - The configuration files inside the folder, `spil_*_conf.py`, must keep the same naming. @@ -72,7 +72,7 @@ The easiest way to start, is to adapt the existing config to your needs. ### Included test files -Under `hamlet_conf/tests` are tests scripts, that test the functions of the Sid and the configuration. +Under `spil_hamlet_conf/tests` are tests scripts, that test the functions of the Sid and the configuration. While adapting the configuration, running the tests help detect problems. See below, [Testing the configuration](configuration.md). @@ -153,8 +153,8 @@ It configures Finders (FindInCaches, FindInAll), Getters, and Writers. ## Testing the configuration -- Recreate or adapt the `hamlet_conf/scripts/example_sids.py` script, which generates correctly formatted test Sids. -- Run `hamlet_conf/scripts/save_examples_to_mock_fs.py`: this will create dummy project files and folders on disk +- Recreate or adapt the `spil_hamlet_conf/scripts/example_sids.py` script, which generates correctly formatted test Sids. +- Run `spil_hamlet_conf/scripts/save_examples_to_mock_fs.py`: this will create dummy project files and folders on disk ### Checking the config @@ -185,7 +185,7 @@ Found: hamlet/a/char/ghost Found: hamlet/a/char/gertrude ``` -Complete tests are found in `hamlet_conf/tests`. +Complete tests are found in `spil_hamlet_conf/tests`. Read more here: [testing](testing.md). diff --git a/docs/configuration_advanced.md b/docs/configuration_advanced.md index 04f891b..b62a128 100644 --- a/docs/configuration_advanced.md +++ b/docs/configuration_advanced.md @@ -82,7 +82,7 @@ So, you are about to create a new pipeline ? :) ### Testing the configuration -- Create or adapt the **example_sids.py** script (as found in `hamlet_conf/scripts`) which generates correctly formatted test Sids. +- Create or adapt the **example_sids.py** script (as found in `spil_hamlet_conf/scripts`) which generates correctly formatted test Sids. - Run **save_examples_to_mock_fs.py**: this will create dummy project files and folders on disk diff --git a/docs/faq.md b/docs/faq.md index 79ad38a..edc0b24 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -40,6 +40,14 @@ Once you add your configuration folder (or the demo configuration folder) to the Please check the documentation for "Adapting the demo config" [configuration](configuration.md). +## The project name is hard coded in the configuration file ? + +In the shipped default demo configuration, the project is hardcoded in the spil_sid_conf file. +This can be changed, to retrieve projects names by any other possible way. + +The main idea is: the preciser the information in the configuration, the better the pattern matching works. +Data that changes rarely (asset types, task types, and potentially project names) should find their way into the configuration. +How this is done can vary, and getting the configuration from a database is absolutely possible. ## Use the name of an item as it's unique Id - really ? diff --git a/docs/installation.md b/docs/installation.md index 00b287f..55978ad 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -12,13 +12,13 @@ $ pip install spil # Or install latest version $ pip install git+https://github.com/MichaelHaussmann/spil.git ``` -This installs `spil`, `spil_tests`, `spil_plugins`, and `hamlet_conf`, the demo configuration. +This installs `spil`, `spil_tests`, `spil_plugins`, and `spil_hamlet_conf`, the demo configuration. ### Default configuration To be able to use Spil, the configuration must be added to the python path. -Spil ships with an example configuration folder named `hamlet_conf`. -If no configuration is found, the `hamlet_conf` folder is added to the python path during spil import. +Spil ships with an example configuration folder named `spil_hamlet_conf`. +If no configuration is found, the `spil_hamlet_conf` folder is added to the python path during spil import. Learn more at the [configuration documentation](configuration.md). @@ -38,7 +38,7 @@ The config package could be versioned by current year, for example 2023.0.1 ## Testing the install -Spil works out of the box with the shipped demo config `hamlet_conf`. +Spil works out of the box with the shipped demo config `spil_hamlet_conf`. In python, type: ```python @@ -55,14 +55,14 @@ print(sid.as_query()) To play with the Finder and Sid paths, files need to exist on the file system. -To generate test files, run `hamlet_conf/scripts/save_examples_to_mock_fs.py`. -This will create dummy project files and folders on disk, inside the `hamlet_conf/data` folder. +To generate test files, run `spil_hamlet_conf/scripts/save_examples_to_mock_fs.py`. +This will create dummy project files and folders on disk, inside the `spil_hamlet_conf/data` folder. From the installation folder type: ```shell -$ python hamlet_conf/scripts/save_examples_to_mock_fs.py +$ python spil_hamlet_conf/scripts/save_examples_to_mock_fs.py ``` -*If you didn't configure Spil, it prints the location of the `hamlet_conf` path when you use it. For example when you `import spil`.* +*If you didn't configure Spil, it prints the location of the `spil_hamlet_conf` path when you use it. For example when you `import spil`.* *Make sure `spil` is in your python path, which should be the case after installation.* diff --git a/docs/testing.md b/docs/testing.md index 267c462..dd59731 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -7,7 +7,7 @@ Spil uses pytest. The *Auto Test* (auto_test.yml) github action runs all tests, including doc-tests, on windows and ubuntu. Pytest is configured in pyproject.toml and a conftest.py file. -On session start, Pytest runs test file creations (inside the `hamlet_conf/data/testing` folder). +On session start, Pytest runs test file creations (inside the `spil_hamlet_conf/data/testing` folder). The Action is triggered by push or pull-request on main branch. @@ -22,9 +22,9 @@ These modules can be run directly from a code editor. ## Integration Tests - Data tests Most tests work in conjunction with the configuration. -Per default, they use the shipped demo configuration `hamlet_conf`. +Per default, they use the shipped demo configuration `spil_hamlet_conf`. -On session start, Pytest runs file creations (inside the `hamlet_conf/data/testing` folder). +On session start, Pytest runs file creations (inside the `spil_hamlet_conf/data/testing` folder). These files are used by the tests (including doctests). When creating a new configuration for a pipeline, it is recommended to implement related tests. @@ -42,22 +42,22 @@ When creating a new config, these basic config conformity checks are useful: For in-depth tests and greater test coverage, it is useful to prepare static data for repeatable tests. -- Recreate or adapt the `hamlet_conf/scripts/generate_example_sids.py` script, which generates correctly formatted test Sids. -- Save the example sids into a file using `hamlet_conf/scripts/save_examples_to_file.py` -- Run `hamlet_conf/scripts/save_examples_to_mock_fs.py`: this will create dummy project files and folders on disk (using the test Sids). +- Recreate or adapt the `spil_hamlet_conf/scripts/generate_example_sids.py` script, which generates correctly formatted test Sids. +- Save the example sids into a file using `spil_hamlet_conf/scripts/save_examples_to_file.py` +- Run `spil_hamlet_conf/scripts/save_examples_to_mock_fs.py`: this will create dummy project files and folders on disk (using the test Sids). ### Testing the Sids and Finders If the basic config checks pass, and there is testable data, we can continue to usage tests. -Complete tests are found in `hamlet_conf/tests`. +Complete tests are found in `spil_hamlet_conf/tests`. -- `hamlet_conf/tests/core_test.py`: tests given Sids for **core attributes** (types, fields, parent(), etc.) -- `hamlet_conf/tests/path_test.py`: tests given Sids **path resolving and path related attributes** (path()) -- `hamlet_conf/tests/data_test.py`: tests given Sids **data access attributes** (exists(), children(), get_last(), etc.) -- `hamlet_conf/tests/finder_test.py`: uses given Sids to build random search Sids, and **tests Finders**. +- `spil_hamlet_conf/tests/core_test.py`: tests given Sids for **core attributes** (types, fields, parent(), etc.) +- `spil_hamlet_conf/tests/path_test.py`: tests given Sids **path resolving and path related attributes** (path()) +- `spil_hamlet_conf/tests/data_test.py`: tests given Sids **data access attributes** (exists(), children(), get_last(), etc.) +- `spil_hamlet_conf/tests/finder_test.py`: uses given Sids to build random search Sids, and **tests Finders**. Do not hesitate to adapt this test file, depending on the finders you will use. *(Note that you could also manually create a python script with example / test searches)*. -- `hamlet_conf/tests/quicktest.py`: sandbox-like file to quickly test various things. +- `spil_hamlet_conf/tests/quicktest.py`: sandbox-like file to quickly test various things. *(more tests work in progress)* diff --git a/docs/wait.md b/docs/wait.md deleted file mode 100644 index c70c2cd..0000000 --- a/docs/wait.md +++ /dev/null @@ -1,6 +0,0 @@ - - - -This documentation is work in progress. - -The new version of spil has not been released yet. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 9578d6b..f31dd73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,15 +5,13 @@ build-backend = "hatchling.build" [tool.hatch.build] include = [ "spil", - "spil_plugins", - "spil_tests", - "hamlet_conf" + "spil_hamlet_conf" ] exclude = [".gitignore"] [project] name = "spil" -version = "0.1.0" +version = "0.1.0rc2" description = "The Simple Pipeline Lib." readme = "README.md" authors = [{ name = "Michael Haussmann", email = "spil@xeo.info" }] @@ -37,14 +35,15 @@ dependencies = [ "Fileseq", "future", "logzero", - "typing-extensions" + "typing-extensions", + "codetiming", ] requires-python = ">=3.7" [project.optional-dependencies] -dev = ["codetiming", "pytest"] # "Faker" -qc = ["mypy", "black", "flake8", "isort", "refurb"] -sg = ["shotgun-api3", "Unidecode"] +dev = ["pytest"] # "Faker" +qc = ["mypy", "black", "flake8", "isort", "refurb"] # Code Quality +sg = ["shotgun-api3", "Unidecode"] # SG plugin [project.urls] "Homepage" = "https://github.com/MichaelHaussmann/spil" diff --git a/spil/conf/global_conf.py b/spil/conf/global_conf.py index fea5a8a..070f7d3 100644 --- a/spil/conf/global_conf.py +++ b/spil/conf/global_conf.py @@ -23,8 +23,8 @@ ) application_path = Path(__file__).parent.parent -default_sid_conf_path = str(application_path.parent / "hamlet_conf") -default_sid_conf_data_path = str(application_path.parent / "hamlet_conf" / "data") +default_sid_conf_path = str(application_path.parent / "spil_hamlet_conf") +default_sid_conf_data_path = str(application_path.parent / "spil_hamlet_conf" / "data") # automatic replacement to INFO and WARN in BETA and PROD loglevel = 80 @@ -62,7 +62,7 @@ The configuration module "{module}" was not found. - Ensure to either include the demo "hamlet_conf" in your python path, + Ensure to either include the demo "spil_hamlet_conf" in your python path, or create your own "{module}" and add its folder to the python path. (If you are running a py.test edit the SPIL_CONF_PATH variable in tests/test_00_init.py to match a python path.) diff --git a/spil/sid/core/query_helper.py b/spil/sid/core/query_helper.py index dbd6ff1..96cca99 100644 --- a/spil/sid/core/query_helper.py +++ b/spil/sid/core/query_helper.py @@ -139,7 +139,7 @@ def apply_query(string, query=None, type=None, fields=None): This is a problem in case of a read Sid, which could be poly-typed. Examples. - (Note that the examples depend on "hamlet_conf" example configuration. Failure may be due to non matching config) + (Note that the examples depend on "spil_hamlet_conf" example configuration. Failure may be due to non matching config) Query updates the sequence: >>> apply_query('hamlet/s/sq010', query='sequence=sq030', type='shot__sequence', fields={'project':'hamlet','type':'s','sequence':'sq010'}) diff --git a/spil/sid/read/finders/find_cache.py b/spil/sid/read/finders/find_cache.py index 9485f8a..e4b5d76 100644 --- a/spil/sid/read/finders/find_cache.py +++ b/spil/sid/read/finders/find_cache.py @@ -1,6 +1,26 @@ """ +This file is part of SPIL, The Simple Pipeline Lib. +(C) copyright 2019-2023 Michael Haussmann, spil@xeo.info + +SPIL is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. + +SPIL is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public License along with SPIL. +If not, see . +""" +""" + + + + + THIS IS WORK IN PROGRESS. + + + + """ from __future__ import annotations @@ -41,11 +61,7 @@ def get_sidcache(data_search, data_source, cache_file=None, name=None): Each sid_cache_file is handled by its own, single, SidCache instance. A SidCache object handles only one sid_cache_file. For example: - a = SidCache(cache_file='/shots.txt') - b = SidCache(cache_file='/shots.txt') - c = SidCache(cache_file='/assets.txt') - a and b will be the same object instance, and access the same data. - c will be another object. + Note that the file can be accessed concurrently from different python instances, from different machines. SidCache still needs to ensure threadsafety. @@ -106,8 +122,8 @@ class FindInCache(Finder): For 'hamlet/s/*/*/*' (sequences, shots and tasks for hamlet project) Originally loaded from FS (File Search). ``` - >>> sidcache = get_sidcache(sid_cache_file='/temp/hamlet_shots.txt', data_search='hamlet/s/*/*', data_source=FS()) - >>> sidcache.find('hamlet/s/*') + ... sidcache = get_sidcache(sid_cache_file='/temp/hamlet_shots.txt', data_search='hamlet/s/*/*', data_source=FS()) + ... sidcache.find('hamlet/s/*') ``` TODO: optional multiple data sources TODO: ttl for warmup diff --git a/spil/sid/sid.py b/spil/sid/sid.py index 87bfb80..c8782aa 100644 --- a/spil/sid/sid.py +++ b/spil/sid/sid.py @@ -657,8 +657,8 @@ def path(self, config: Optional[str] = None) -> Path | None: True >>> path = Sid('hamlet/a/char/ophelia/model/v001/w/ma').path() - >>> path.relative_to(conf.default_sid_conf_path).as_posix() # to be location and os independent - 'data/testing/SPIL_PROJECTS/LOCAL/PROJECTS/HAMLET/PROD/ASSETS/char/ophelia/model/v001/char_ophelia_model_WORK_v001.ma' + >>> path.relative_to(conf.default_sid_conf_data_path).as_posix() # to be location and os independent + 'testing/SPIL_PROJECTS/LOCAL/PROJECTS/HAMLET/PROD/ASSETS/char/ophelia/model/v001/char_ophelia_model_WORK_v001.ma' >>> Sid('bla/bla').path() @@ -948,7 +948,7 @@ class Sid(DataSid): >>> Sid(fields={'project': 'hamlet', 'sequence': 'sq010', 'type': 's'}) # fields dict Sid('shot__sequence:hamlet/s/sq010') - >>> path = Path(conf.default_sid_conf_path) / "data/testing/SPIL_PROJECTS/LOCAL/PROJECTS/HAMLET/PROD/ASSETS/char/ophelia/model/v001/char_ophelia_model_WORK_v001.ma" + >>> path = Path(conf.default_sid_conf_data_path) / "testing/SPIL_PROJECTS/LOCAL/PROJECTS/HAMLET/PROD/ASSETS/char/ophelia/model/v001/char_ophelia_model_WORK_v001.ma" >>> Sid(path=path) # path (default config) # TODO: any config Sid('asset__file:hamlet/a/char/ophelia/model/v001/w/ma') @@ -967,11 +967,8 @@ class Sid(DataSid): if __name__ == "__main__": - from spil_tests import stop - from pprint import pprint - from spil.util.log import debug, setLevel, INFO, DEBUG, info - - setLevel(INFO) + # from spil.util.log import setLevel, INFO + # setLevel(INFO) s = Sid("hamlet/s/sq010/sh0010/anim") print(s.exists()) diff --git a/spil_tests/__init__.py b/spil/tests/__init__.py similarity index 100% rename from spil_tests/__init__.py rename to spil/tests/__init__.py diff --git a/spil_plugins/crud/__init__.py b/spil/tests/config_checks/__init__.py similarity index 100% rename from spil_plugins/crud/__init__.py rename to spil/tests/config_checks/__init__.py diff --git a/spil_tests/config_checks/check_01_sid_config.py b/spil/tests/config_checks/check_01_sid_config.py similarity index 98% rename from spil_tests/config_checks/check_01_sid_config.py rename to spil/tests/config_checks/check_01_sid_config.py index aeda9c5..760e212 100644 --- a/spil_tests/config_checks/check_01_sid_config.py +++ b/spil/tests/config_checks/check_01_sid_config.py @@ -24,7 +24,7 @@ from spil.conf import sid_templates # type: ignore from spil.util.log import DEBUG, ERROR, get_logger -log = get_logger('spil_tests') +log = get_logger('tests') log.setLevel(DEBUG) diff --git a/spil_tests/config_checks/check_02_path_config.py b/spil/tests/config_checks/check_02_path_config.py similarity index 99% rename from spil_tests/config_checks/check_02_path_config.py rename to spil/tests/config_checks/check_02_path_config.py index caecff3..3d618f9 100644 --- a/spil_tests/config_checks/check_02_path_config.py +++ b/spil/tests/config_checks/check_02_path_config.py @@ -27,7 +27,7 @@ path_templates = path_config.path_templates from spil.util.log import DEBUG, ERROR, get_logger -log = get_logger('spil_tests') +log = get_logger('tests') log.setLevel(DEBUG) diff --git a/spil_tests/config_checks/check_03_config_coverage.py b/spil/tests/config_checks/check_03_config_coverage.py similarity index 97% rename from spil_tests/config_checks/check_03_config_coverage.py rename to spil/tests/config_checks/check_03_config_coverage.py index 33da6fc..40ac4c9 100644 --- a/spil_tests/config_checks/check_03_config_coverage.py +++ b/spil/tests/config_checks/check_03_config_coverage.py @@ -11,7 +11,7 @@ from spil.conf import sid_templates # type: ignore from spil.util.log import DEBUG, ERROR, get_logger -log = get_logger('spil_tests') +log = get_logger('tests') log.setLevel(DEBUG) diff --git a/spil_tests/config_checks/__init__.py b/spil/tests/feature_tests/__init__.py similarity index 100% rename from spil_tests/config_checks/__init__.py rename to spil/tests/feature_tests/__init__.py diff --git a/spil_tests/feature_tests/test_lru.py b/spil/tests/feature_tests/test_lru.py similarity index 100% rename from spil_tests/feature_tests/test_lru.py rename to spil/tests/feature_tests/test_lru.py diff --git a/spil_tests/feature_tests/__init__.py b/spil/tests/prep/__init__.py similarity index 100% rename from spil_tests/feature_tests/__init__.py rename to spil/tests/prep/__init__.py diff --git a/spil_tests/prep/build_searches.py b/spil/tests/prep/build_searches.py similarity index 100% rename from spil_tests/prep/build_searches.py rename to spil/tests/prep/build_searches.py diff --git a/spil_tests/prep/make_mock_fs.py b/spil/tests/prep/make_mock_fs.py similarity index 100% rename from spil_tests/prep/make_mock_fs.py rename to spil/tests/prep/make_mock_fs.py diff --git a/spil_tests/prep/parse_sids_from_fs.py b/spil/tests/prep/parse_sids_from_fs.py similarity index 97% rename from spil_tests/prep/parse_sids_from_fs.py rename to spil/tests/prep/parse_sids_from_fs.py index 320591b..b9ae5b2 100644 --- a/spil_tests/prep/parse_sids_from_fs.py +++ b/spil/tests/prep/parse_sids_from_fs.py @@ -13,7 +13,7 @@ """ -# from spil_tests.utils import init # needs to be before spil.conf import +# from tests.utils import init # needs to be before spil.conf import from spil.util.log import debug, info from spil import Sid diff --git a/spil_tests/prep/save_sid_list_to_file.py b/spil/tests/prep/save_sid_list_to_file.py similarity index 100% rename from spil_tests/prep/save_sid_list_to_file.py rename to spil/tests/prep/save_sid_list_to_file.py diff --git a/spil_tests/utils/__init__.py b/spil/tests/utils/__init__.py similarity index 100% rename from spil_tests/utils/__init__.py rename to spil/tests/utils/__init__.py diff --git a/spil_tests/utils/dualsearch_ab_tester.py b/spil/tests/utils/dualsearch_ab_tester.py similarity index 93% rename from spil_tests/utils/dualsearch_ab_tester.py rename to spil/tests/utils/dualsearch_ab_tester.py index 396a022..2f31d83 100644 --- a/spil_tests/utils/dualsearch_ab_tester.py +++ b/spil/tests/utils/dualsearch_ab_tester.py @@ -11,8 +11,8 @@ If not, see . """ from spil.util.exception import raiser -from spil.util.log import DEBUG, ERROR, get_logger -from spil_tests import Timer +from spil.util.log import get_logger +from spil.tests import Timer log = get_logger("spil_tests") @@ -65,8 +65,8 @@ def check_search_ab(searches, finderA, finderB, as_sid=False, raise_problems=Fal if __name__ == "__main__": - from spil import FindInList, FindInPaths, FindInAll - from spil.util.log import setLevel, ERROR, DEBUG + from spil import FindInPaths, FindInAll + from spil.util.log import setLevel, ERROR setLevel(ERROR) diff --git a/spil_tests/utils/search_tester.py b/spil/tests/utils/search_tester.py similarity index 93% rename from spil_tests/utils/search_tester.py rename to spil/tests/utils/search_tester.py index 493fca7..c978673 100644 --- a/spil_tests/utils/search_tester.py +++ b/spil/tests/utils/search_tester.py @@ -11,9 +11,9 @@ If not, see . """ from spil import Sid, FindInPaths, SpilException -from spil_tests.utils.sid_full_tester import test_full_sid -from spil.util.log import DEBUG, ERROR, get_logger -from spil_tests import Timer +from spil.tests.utils.sid_full_tester import test_full_sid +from spil.util.log import get_logger +from spil.tests import Timer log = get_logger("spil_tests") @@ -76,7 +76,7 @@ def check_searches_in_finder(searches, finder=None, as_sid=True, do_log=True, do if __name__ == "__main__": - from spil.util.log import setLevel, ERROR, DEBUG + from spil.util.log import setLevel, ERROR setLevel(ERROR) diff --git a/spil_tests/utils/sid_core_tester.py b/spil/tests/utils/sid_core_tester.py similarity index 98% rename from spil_tests/utils/sid_core_tester.py rename to spil/tests/utils/sid_core_tester.py index b8e802d..5a0f35f 100644 --- a/spil_tests/utils/sid_core_tester.py +++ b/spil/tests/utils/sid_core_tester.py @@ -133,6 +133,7 @@ def check_typed_sids(sids, reraise=True, replace=None): setLevel(INFO) - from scripts.example_sids import sids + # from hamlet_scripts.example_sids import sids + sids = [] check_typed_sids(sids) diff --git a/spil_tests/utils/sid_data_tester.py b/spil/tests/utils/sid_data_tester.py similarity index 100% rename from spil_tests/utils/sid_data_tester.py rename to spil/tests/utils/sid_data_tester.py diff --git a/spil_tests/utils/sid_full_tester.py b/spil/tests/utils/sid_full_tester.py similarity index 100% rename from spil_tests/utils/sid_full_tester.py rename to spil/tests/utils/sid_full_tester.py diff --git a/spil_tests/utils/sid_path_tester.py b/spil/tests/utils/sid_path_tester.py similarity index 91% rename from spil_tests/utils/sid_path_tester.py rename to spil/tests/utils/sid_path_tester.py index fc7eea8..a65f8ab 100644 --- a/spil_tests/utils/sid_path_tester.py +++ b/spil/tests/utils/sid_path_tester.py @@ -9,7 +9,6 @@ You should have received a copy of the GNU Lesser General Public License along with SPIL. If not, see . - """ from spil.util.log import DEBUG, get_logger from spil import Sid, SpilException @@ -45,7 +44,9 @@ def check_path_sid(s, configs=[], reraise=True, replace=None): except AssertionError: msg = "Sid path is ambiguous. {}\nsid: {}\nsid.path(): {}\nSid(path=sid.path()): {}\n".format( "(Sid is a search, this may be normal)." if sid.is_search() else "", - sid, sid.path(), Sid(path=sid.path()) + sid, + sid.path(), + Sid(path=sid.path()), ) log.warning(msg) try: @@ -54,7 +55,10 @@ def check_path_sid(s, configs=[], reraise=True, replace=None): except AssertionError: msg = "Sid path is ambiguous. {}\nsid: {}\nsid.path(): {}\nSid(path=sid.path()): {}\nSid(path=sid.path()).path(): {}".format( "(Sid is a search, this may be normal)." if sid.is_search() else "", - sid, sid.path(), Sid(path=sid.path()), Sid(path=sid.path()).path() + sid, + sid.path(), + Sid(path=sid.path()), + Sid(path=sid.path()).path(), ) log.warning(msg) else: @@ -92,7 +96,7 @@ def check_path_sids(sids, configs=[], reraise=True, replace=None): return for i, s in enumerate(sids): - log.info('*' * 75) + log.info("*" * 75) log.info("----------------------------------------- {}".format(i)) check_path_sid(s, configs=configs, reraise=reraise, replace=replace) @@ -103,6 +107,7 @@ def check_path_sids(sids, configs=[], reraise=True, replace=None): setLevel(INFO) - from scripts.example_sids import sids + # from hamlet_scripts.example_sids import sids + sids = [] check_path_sids(sids) diff --git a/hamlet_conf/data/templates/empty.ma b/spil_hamlet_conf/data/templates/empty.ma similarity index 100% rename from hamlet_conf/data/templates/empty.ma rename to spil_hamlet_conf/data/templates/empty.ma diff --git a/hamlet_conf/data/templates/empty.mb b/spil_hamlet_conf/data/templates/empty.mb similarity index 100% rename from hamlet_conf/data/templates/empty.mb rename to spil_hamlet_conf/data/templates/empty.mb diff --git a/hamlet_conf/data/testing/hamlet.sids.txt b/spil_hamlet_conf/data/testing/hamlet.sids.txt similarity index 100% rename from hamlet_conf/data/testing/hamlet.sids.txt rename to spil_hamlet_conf/data/testing/hamlet.sids.txt diff --git a/spil_tests/prep/__init__.py b/spil_hamlet_conf/hamlet_plugins/__init__.py similarity index 100% rename from spil_tests/prep/__init__.py rename to spil_hamlet_conf/hamlet_plugins/__init__.py diff --git a/spil_plugins/crud/next_get.py b/spil_hamlet_conf/hamlet_plugins/next_get.py similarity index 100% rename from spil_plugins/crud/next_get.py rename to spil_hamlet_conf/hamlet_plugins/next_get.py diff --git a/hamlet_conf/scripts/example_sids.py b/spil_hamlet_conf/hamlet_scripts/example_sids.py similarity index 85% rename from hamlet_conf/scripts/example_sids.py rename to spil_hamlet_conf/hamlet_scripts/example_sids.py index 79bc54c..26c437f 100644 --- a/hamlet_conf/scripts/example_sids.py +++ b/spil_hamlet_conf/hamlet_scripts/example_sids.py @@ -7,8 +7,7 @@ This shows all the types covered by the test sids, and which types are not. """ -from scripts.save_examples_to_file import sid_file -from spil_tests.config_checks.check_03_config_coverage import test_config_coverage +from hamlet_scripts.save_examples_to_file import sid_file sids = [] with sid_file.open() as f: @@ -17,15 +16,14 @@ if __name__ == "__main__": import sys - from spil_tests import Timer + from spil.tests import Timer - from spil.util.log import DEBUG, ERROR, get_logger + from spil.util.log import DEBUG, get_logger log = get_logger("spil_tests") log.setLevel(DEBUG) from spil import Sid - from pprint import pprint print("Start") # pprint(sids) diff --git a/hamlet_conf/scripts/generate_example_sids.py b/spil_hamlet_conf/hamlet_scripts/generate_example_sids.py similarity index 96% rename from hamlet_conf/scripts/generate_example_sids.py rename to spil_hamlet_conf/hamlet_scripts/generate_example_sids.py index 8017ece..dbbfbee 100644 --- a/hamlet_conf/scripts/generate_example_sids.py +++ b/spil_hamlet_conf/hamlet_scripts/generate_example_sids.py @@ -19,12 +19,11 @@ from math import ceil from time import sleep -import spil # config path bootstrap from spil_sid_conf import projects, asset_types, asset_tasks from spil_sid_conf import extensions_scene, extensions_movie, shot_tasks, extensions_cache from logging import debug -from spil_tests.config_checks.check_03_config_coverage import test_config_coverage +from spil.tests.config_checks.check_03_config_coverage import test_config_coverage do_intermediates = True # If this is False, only leave paths will be generated. See: LS extrapolate. repeat_times = 1 # 3 @@ -147,15 +146,15 @@ def append(sids, data): if __name__ == '__main__': import sys - # from spil_tests import Timer + # from tests import Timer - from spil.util.log import DEBUG, ERROR, get_logger + from spil.util.log import DEBUG, get_logger log = get_logger("spil_tests") log.setLevel(DEBUG) from spil import Sid - from pprint import pprint + print('Start') #pprint(sids) diff --git a/hamlet_conf/scripts/save_examples_to_file.py b/spil_hamlet_conf/hamlet_scripts/save_examples_to_file.py similarity index 62% rename from hamlet_conf/scripts/save_examples_to_file.py rename to spil_hamlet_conf/hamlet_scripts/save_examples_to_file.py index ec10a18..a795e8c 100644 --- a/hamlet_conf/scripts/save_examples_to_file.py +++ b/spil_hamlet_conf/hamlet_scripts/save_examples_to_file.py @@ -6,12 +6,11 @@ """ from pathlib import Path +from spil import conf +from hamlet_scripts.generate_example_sids import sids # type: ignore +from spil.tests.prep.save_sid_list_to_file import write_sids_to_file -import spil # default config path bootstrap -from scripts.generate_example_sids import sids # type: ignore -from spil_tests.prep.save_sid_list_to_file import write_sids_to_file - -sid_file = Path(__file__).parent.parent / "data" / "testing" / "hamlet.sids.txt" +sid_file = Path(conf.default_sid_conf_data_path) / "testing" / "hamlet.sids.txt" if not sid_file.exists(): print(f"Creating Sid test file: {sid_file}") diff --git a/hamlet_conf/scripts/save_examples_to_mock_fs.py b/spil_hamlet_conf/hamlet_scripts/save_examples_to_mock_fs.py similarity index 68% rename from hamlet_conf/scripts/save_examples_to_mock_fs.py rename to spil_hamlet_conf/hamlet_scripts/save_examples_to_mock_fs.py index a9f188b..1d033bc 100644 --- a/hamlet_conf/scripts/save_examples_to_mock_fs.py +++ b/spil_hamlet_conf/hamlet_scripts/save_examples_to_mock_fs.py @@ -5,14 +5,13 @@ according to the path configs. """ -import spil # default config path bootstrap import spil_data_conf -from scripts.example_sids import sids -from spil_tests.prep.make_mock_fs import generate_mock_fs +from hamlet_scripts.example_sids import sids +from spil.tests.prep.make_mock_fs import generate_mock_fs if __name__ == "__main__": - from spil.util.log import setLevel, INFO, DEBUG, info + from spil.util.log import setLevel, INFO print("") print("Tests start") diff --git a/hamlet_conf/tests/core_test.py b/spil_hamlet_conf/hamlet_tests/core_test.py similarity index 68% rename from hamlet_conf/tests/core_test.py rename to spil_hamlet_conf/hamlet_tests/core_test.py index e466d51..191226d 100644 --- a/hamlet_conf/tests/core_test.py +++ b/spil_hamlet_conf/hamlet_tests/core_test.py @@ -5,9 +5,8 @@ """ import random -import spil # config path bootstrap -from scripts.example_sids import sids # type: ignore -from spil_tests.utils.sid_core_tester import check_typed_sids +from hamlet_scripts.example_sids import sids # type: ignore +from spil.tests.utils.sid_core_tester import check_typed_sids from spil.util.log import setLevel, INFO def test_typed_sids(): diff --git a/hamlet_conf/tests/data_test.py b/spil_hamlet_conf/hamlet_tests/data_test.py similarity index 81% rename from hamlet_conf/tests/data_test.py rename to spil_hamlet_conf/hamlet_tests/data_test.py index f1d1284..c1f4772 100644 --- a/hamlet_conf/tests/data_test.py +++ b/spil_hamlet_conf/hamlet_tests/data_test.py @@ -9,8 +9,8 @@ import random # import spil # config path bootstrap -from scripts.example_sids import sids # type: ignore -from spil_tests.utils.sid_data_tester import check_data_sids +from hamlet_scripts.example_sids import sids # type: ignore +from spil.tests.utils.sid_data_tester import check_data_sids def test_data_sids(): diff --git a/hamlet_conf/tests/finder_test.py b/spil_hamlet_conf/hamlet_tests/finder_test.py similarity index 82% rename from hamlet_conf/tests/finder_test.py rename to spil_hamlet_conf/hamlet_tests/finder_test.py index 889dd83..6257f14 100644 --- a/hamlet_conf/tests/finder_test.py +++ b/spil_hamlet_conf/hamlet_tests/finder_test.py @@ -5,16 +5,15 @@ """ import random -import spil # default config path bootstrap -from scripts.example_sids import sids # type: ignore +from hamlet_scripts.example_sids import sids # type: ignore # from spil_plugins.sg.find_sg import FindInSG from spil import FindInList, FindInPaths, FindInAll -from spil_tests.prep.build_searches import from_sid_build_searches -from spil_tests.utils.search_tester import check_searches_in_finder +from spil.tests.prep.build_searches import from_sid_build_searches +from spil.tests.utils.search_tester import check_searches_in_finder from spil.util.log import get_logger -from spil_tests.utils.dualsearch_ab_tester import check_search_ab +from spil.tests.utils.dualsearch_ab_tester import check_search_ab log = get_logger("spil_tests", color=False) @@ -57,6 +56,6 @@ def test_finders(): if __name__ == "__main__": - from spil.util.log import setLevel, INFO + from spil.util.log import INFO log.setLevel(INFO) test_finders() diff --git a/hamlet_conf/tests/path_test.py b/spil_hamlet_conf/hamlet_tests/path_test.py similarity index 73% rename from hamlet_conf/tests/path_test.py rename to spil_hamlet_conf/hamlet_tests/path_test.py index 3534b43..0e8b563 100644 --- a/hamlet_conf/tests/path_test.py +++ b/spil_hamlet_conf/hamlet_tests/path_test.py @@ -6,10 +6,9 @@ """ import random -import spil # config path bootstrap import spil_data_conf # type: ignore -from scripts.example_sids import sids # type: ignore -from spil_tests.utils.sid_path_tester import check_path_sids +from hamlet_scripts.example_sids import sids # type: ignore +from spil.tests.utils.sid_path_tester import check_path_sids def test_path_sids(): diff --git a/hamlet_conf/tests/quicktest.py b/spil_hamlet_conf/hamlet_tests/quicktest.py similarity index 92% rename from hamlet_conf/tests/quicktest.py rename to spil_hamlet_conf/hamlet_tests/quicktest.py index 2e517c8..006cb06 100644 --- a/hamlet_conf/tests/quicktest.py +++ b/spil_hamlet_conf/hamlet_tests/quicktest.py @@ -6,9 +6,9 @@ if __name__ == "__main__": - from spil_tests import stop + from spil.tests import stop from pprint import pprint - from spil.util.log import debug, setLevel, INFO, DEBUG, info + from spil.util.log import setLevel, INFO setLevel(INFO) diff --git a/hamlet_conf/spil_data_conf.py b/spil_hamlet_conf/spil_data_conf.py similarity index 72% rename from hamlet_conf/spil_data_conf.py rename to spil_hamlet_conf/spil_data_conf.py index b762a7c..3bbcf6a 100644 --- a/hamlet_conf/spil_data_conf.py +++ b/spil_hamlet_conf/spil_data_conf.py @@ -5,7 +5,7 @@ It is ingested by spil.conf.data_conf_load, which reads it into spil.conf. -*This is work in progress* +*This is beta / work in progress* The data config specification is subject to change. (without affecting spil's API). @@ -13,33 +13,18 @@ """ from __future__ import annotations from pathlib import Path -# attribute getters -# cachable_attributes (by getter / by type / with TTL - for example publish file size, date, owner -# sid_cache_path = '/home/mh/PycharmProjects/spil2/hamlet_conf/data/caches' -# sid_cache_folder = sid_cache_path - -# --------------------------------------------- +######################################################### # Path config path_configs = {'local': 'spil_fs_conf', 'server': 'spil_fs_server_conf' } default_path_config = 'local' - -# WriteToPaths & GetFromPaths -path_data_suffix = '.data.json' -# WriteToPath: create. If a template exists for a suffix (extension), we copy it. -create_file_using_template = { # type: ignore - # 'ma': '.../empty.ma', - # 'mb': '.../empty.mb' -} -# WriteToPath: create. If no template exists, we create an empty file with path.touch(), if create_using_touch is True -create_file_using_touch = True -# If nothing of these is set, we do not create a file. -# --------------------------------------------- -# End Path config +######################################################### +######################################################### +# Config for FindInAll def get_finder_for(search_sid, config=None): # get finder by Sid and optional config """ Configuration used by FindInAll, to define which Finder is used for a given Search Sid. @@ -47,7 +32,7 @@ def get_finder_for(search_sid, config=None): # get finder by Sid and optional c For a given Search Sid, and optional config, looks up the Search Sid's type and the matching Finder, as defined in an ad-hoc dict. - The config allows to create multiple FindInAll instances / behaviours. + The "config" allows to create multiple FindInAll instances / behaviours. (similar to the FindInPaths where we Find in different file systems) Args: @@ -55,7 +40,7 @@ def get_finder_for(search_sid, config=None): # get finder by Sid and optional c config: optional config to be able to define multiple FindInAll instances. Returns: - A Finder object for this search. + A Finder instance for this search. """ # type: ignore from spil_sid_conf import projects, asset_types # type: ignore @@ -84,20 +69,32 @@ def get_finder_for(search_sid, config=None): # get finder by Sid and optional c return None +######################################################### +# Config for GetFromAll def get_getter_for(sid, attribute=None, config=None): """ + Configuration used by GetFromAll, to define which Getter is used for a given Sid or Search Sid. - *This is work in progress* + For a given Sid, and optional attribute and config, + looks up the Sid's type and the matching Getter, as defined in an ad-hoc dict. - For a given attribute, looks up the matching attribute_getters, as defined in a dict. - Returned value is a getter function. + The "config" allows to create multiple GetFromAll instances / behaviours. + (similar to the FindInPaths where we Get from different file systems) + + The function may return None, in which case the getter will not return anything. + + Args: + sid: sid that is queried by GetFromAll + attribute: optional attribute name + config: optional config to be able to define multiple GetFromAll instances. - Currently, the sid argument is not used. + Returns: + A Getter instance for this search, or None is none is defined. """ # from spil_action.libs.files import get_comment, get_size, get_time from spil import Getter, GetFromPaths # from spil_plugins.sg.get_sg import GetFromSG - from spil_plugins.crud.next_get import NextGetter + from hamlet_plugins.next_get import NextGetter attribute_getters = { "next.version": NextGetter() @@ -135,11 +132,28 @@ def get_getter_for(sid, attribute=None, config=None): def get_writer_for(sid): """ + Configuration used by WriteToAll, to define which Writer is used for a given Sid. + *This is work in progress* """ raise NotImplementedError("get_writer_for is not implemented") +######################################################### +# Config for WriteToPaths & GetFromPaths +# +path_data_suffix = '.data.json' +# WriteToPath: create. If a template exists for a suffix (extension), we copy it. +create_file_using_template = { # type: ignore + # 'ma': '.../empty.ma', + # 'mb': '.../empty.mb' +} +# WriteToPath: create. If no template exists, we create an empty file with path.touch(), if create_using_touch is True +create_file_using_touch = True +# If nothing of these is set, we do not create a file. + + +# Used by WriteToPaths and GetFromPaths def get_data_json_path(sid_path: Path) -> Path: """ For a given Sid path, returns the path of a hidden data "sidecar" json file. @@ -156,3 +170,7 @@ def get_data_json_path(sid_path: Path) -> Path: # TODO: add file rotation data_path = sid_path.with_name('.' + sid_path.name).with_suffix(path_data_suffix) return data_path + +# End of Config for WriteToPaths & GetFromPaths +######################################################### + diff --git a/hamlet_conf/spil_fs_conf.py b/spil_hamlet_conf/spil_fs_conf.py similarity index 100% rename from hamlet_conf/spil_fs_conf.py rename to spil_hamlet_conf/spil_fs_conf.py diff --git a/hamlet_conf/spil_fs_server_conf.py b/spil_hamlet_conf/spil_fs_server_conf.py similarity index 100% rename from hamlet_conf/spil_fs_server_conf.py rename to spil_hamlet_conf/spil_fs_server_conf.py diff --git a/hamlet_conf/spil_sg_conf.py b/spil_hamlet_conf/spil_sg_conf.py similarity index 100% rename from hamlet_conf/spil_sg_conf.py rename to spil_hamlet_conf/spil_sg_conf.py diff --git a/hamlet_conf/spil_sid_conf.py b/spil_hamlet_conf/spil_sid_conf.py similarity index 100% rename from hamlet_conf/spil_sid_conf.py rename to spil_hamlet_conf/spil_sid_conf.py