From 2aab3e82d1053c623ce85dac72659b7e5c164f81 Mon Sep 17 00:00:00 2001 From: Ethan Ho <53266718+ethho@users.noreply.github.com> Date: Sun, 22 Sep 2024 12:16:25 -0500 Subject: [PATCH 1/6] Disable hidden attributes (#1091) by default --- datajoint/declare.py | 22 +++++++++-------- datajoint/settings.py | 1 + tests/test_declare.py | 57 ++++++++++++++++++++++++++++++++++++++----- 3 files changed, 64 insertions(+), 16 deletions(-) diff --git a/datajoint/declare.py b/datajoint/declare.py index 1c02564a..d813374a 100644 --- a/datajoint/declare.py +++ b/datajoint/declare.py @@ -10,6 +10,7 @@ from .errors import DataJointError, _support_filepath_types, FILEPATH_FEATURE_SWITCH from .attribute_adapter import get_adapter from .condition import translate_attribute +from .settings import config UUID_DATA_TYPE = "binary(16)" MAX_TABLE_NAME_LENGTH = 64 @@ -311,17 +312,18 @@ def declare(full_table_name, definition, context): external_stores, ) = prepare_declare(definition, context) - metadata_attr_sql = [ - "`_{full_table_name}_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP" - ] - attribute_sql.extend( - attr.format( - full_table_name=sha1( - full_table_name.replace("`", "").encode("utf-8") - ).hexdigest() + if config.get("enable_hidden_attributes", False): + metadata_attr_sql = [ + "`_{full_table_name}_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP" + ] + attribute_sql.extend( + attr.format( + full_table_name=sha1( + full_table_name.replace("`", "").encode("utf-8") + ).hexdigest() + ) + for attr in metadata_attr_sql ) - for attr in metadata_attr_sql - ) if not primary_key: raise DataJointError("Table must have a primary key") diff --git a/datajoint/settings.py b/datajoint/settings.py index 4fc7f301..e1779859 100644 --- a/datajoint/settings.py +++ b/datajoint/settings.py @@ -47,6 +47,7 @@ "display.show_tuple_count": True, "database.use_tls": None, "enable_python_native_blobs": True, # python-native/dj0 encoding support + "enable_hidden_attributes": False, "filepath_checksum_size_limit": None, # file size limit for when to disable checksums } ) diff --git a/tests/test_declare.py b/tests/test_declare.py index f848d31c..0b808e56 100644 --- a/tests/test_declare.py +++ b/tests/test_declare.py @@ -3,6 +3,25 @@ import datajoint as dj import inspect from datajoint.declare import declare +from datajoint.settings import config + + +@pytest.fixture(scope="function") +def enable_hidden_attributes(): + orig_config_val = config.get("enable_hidden_attributes") + config["enable_hidden_attributes"] = True + yield + if orig_config_val is not None: + config["enable_hidden_attributes"] = orig_config_val + + +@pytest.fixture(scope="function") +def disable_hidden_attributes(): + orig_config_val = config.get("enable_hidden_attributes") + config["enable_hidden_attributes"] = False + yield + if orig_config_val is not None: + config["enable_hidden_attributes"] = orig_config_val def test_schema_decorator(schema_any): @@ -373,9 +392,35 @@ class Table_With_Underscores(dj.Manual): schema_any(Table_With_Underscores) -def test_hidden_attributes(schema_any): - assert ( - list(Experiment().heading._attributes.keys())[-1].split("_")[2] == "timestamp" - ) - assert any(a.is_hidden for a in Experiment().heading._attributes.values()) - assert not any(a.is_hidden for a in Experiment().heading.attributes.values()) +def test_hidden_attributes_default_value(): + config_val = config.get("enable_hidden_attributes") + assert config_val is not None and not config_val, \ + "Default value for enable_hidden_attributes is not False" + + +def test_hidden_attributes_enabled(enable_hidden_attributes, schema_any): + orig_config_val = config.get("enable_hidden_attributes") + config["enable_hidden_attributes"] = True + + msg = f"{Experiment().heading._attributes=}" + assert any(a.name.endswith("_timestamp") for a in Experiment().heading._attributes.values()), msg + assert any(a.name.startswith("_") for a in Experiment().heading._attributes.values()), msg + assert any(a.is_hidden for a in Experiment().heading._attributes.values()), msg + assert not any(a.is_hidden for a in Experiment().heading.attributes.values()), msg + + if orig_config_val is not None: + config["enable_hidden_attributes"] = orig_config_val + + +def test_hidden_attributes_disabled(disable_hidden_attributes, schema_any): + orig_config_val = config.get("enable_hidden_attributes") + config["enable_hidden_attributes"] = False + + msg = f"{Experiment().heading._attributes=}" + assert not any(a.name.endswith("_timestamp") for a in Experiment().heading._attributes.values()), msg + assert not any(a.name.startswith("_") for a in Experiment().heading._attributes.values()), msg + assert not any(a.is_hidden for a in Experiment().heading._attributes.values()), msg + assert not any(a.is_hidden for a in Experiment().heading.attributes.values()), msg + + if orig_config_val is not None: + config["enable_hidden_attributes"] = orig_config_val From 668b9bdc77cf26d503859a56f1dcfa199d543908 Mon Sep 17 00:00:00 2001 From: Ethan Ho <53266718+ethho@users.noreply.github.com> Date: Sun, 22 Sep 2024 12:18:53 -0500 Subject: [PATCH 2/6] Format with black==24.4.2 --- tests/test_declare.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/tests/test_declare.py b/tests/test_declare.py index 0b808e56..dc60deb9 100644 --- a/tests/test_declare.py +++ b/tests/test_declare.py @@ -394,8 +394,9 @@ class Table_With_Underscores(dj.Manual): def test_hidden_attributes_default_value(): config_val = config.get("enable_hidden_attributes") - assert config_val is not None and not config_val, \ - "Default value for enable_hidden_attributes is not False" + assert ( + config_val is not None and not config_val + ), "Default value for enable_hidden_attributes is not False" def test_hidden_attributes_enabled(enable_hidden_attributes, schema_any): @@ -403,8 +404,12 @@ def test_hidden_attributes_enabled(enable_hidden_attributes, schema_any): config["enable_hidden_attributes"] = True msg = f"{Experiment().heading._attributes=}" - assert any(a.name.endswith("_timestamp") for a in Experiment().heading._attributes.values()), msg - assert any(a.name.startswith("_") for a in Experiment().heading._attributes.values()), msg + assert any( + a.name.endswith("_timestamp") for a in Experiment().heading._attributes.values() + ), msg + assert any( + a.name.startswith("_") for a in Experiment().heading._attributes.values() + ), msg assert any(a.is_hidden for a in Experiment().heading._attributes.values()), msg assert not any(a.is_hidden for a in Experiment().heading.attributes.values()), msg @@ -417,8 +422,12 @@ def test_hidden_attributes_disabled(disable_hidden_attributes, schema_any): config["enable_hidden_attributes"] = False msg = f"{Experiment().heading._attributes=}" - assert not any(a.name.endswith("_timestamp") for a in Experiment().heading._attributes.values()), msg - assert not any(a.name.startswith("_") for a in Experiment().heading._attributes.values()), msg + assert not any( + a.name.endswith("_timestamp") for a in Experiment().heading._attributes.values() + ), msg + assert not any( + a.name.startswith("_") for a in Experiment().heading._attributes.values() + ), msg assert not any(a.is_hidden for a in Experiment().heading._attributes.values()), msg assert not any(a.is_hidden for a in Experiment().heading.attributes.values()), msg From b7e99bc9815fc5b52586f64f568d4993660e6db4 Mon Sep 17 00:00:00 2001 From: Ethan Ho <53266718+ethho@users.noreply.github.com> Date: Sun, 22 Sep 2024 14:02:17 -0500 Subject: [PATCH 3/6] Rename enable_hidden_attributes to add_hidden_timestamp sed -i 's/enable_hidden_attributes/add_hidden_timestamp/g' ./**/*.py --- datajoint/declare.py | 2 +- datajoint/settings.py | 2 +- tests/test_declare.py | 40 +++++++++++++++------------------------- 3 files changed, 17 insertions(+), 27 deletions(-) diff --git a/datajoint/declare.py b/datajoint/declare.py index d813374a..b1194880 100644 --- a/datajoint/declare.py +++ b/datajoint/declare.py @@ -312,7 +312,7 @@ def declare(full_table_name, definition, context): external_stores, ) = prepare_declare(definition, context) - if config.get("enable_hidden_attributes", False): + if config.get("add_hidden_timestamp", False): metadata_attr_sql = [ "`_{full_table_name}_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP" ] diff --git a/datajoint/settings.py b/datajoint/settings.py index e1779859..cdf27891 100644 --- a/datajoint/settings.py +++ b/datajoint/settings.py @@ -47,7 +47,7 @@ "display.show_tuple_count": True, "database.use_tls": None, "enable_python_native_blobs": True, # python-native/dj0 encoding support - "enable_hidden_attributes": False, + "add_hidden_timestamp": False, "filepath_checksum_size_limit": None, # file size limit for when to disable checksums } ) diff --git a/tests/test_declare.py b/tests/test_declare.py index dc60deb9..1b186778 100644 --- a/tests/test_declare.py +++ b/tests/test_declare.py @@ -7,21 +7,21 @@ @pytest.fixture(scope="function") -def enable_hidden_attributes(): - orig_config_val = config.get("enable_hidden_attributes") - config["enable_hidden_attributes"] = True +def enable_add_hidden_timestamp(): + orig_config_val = config.get("add_hidden_timestamp") + config["add_hidden_timestamp"] = True yield if orig_config_val is not None: - config["enable_hidden_attributes"] = orig_config_val + config["add_hidden_timestamp"] = orig_config_val @pytest.fixture(scope="function") -def disable_hidden_attributes(): - orig_config_val = config.get("enable_hidden_attributes") - config["enable_hidden_attributes"] = False +def disable_add_hidden_timestamp(): + orig_config_val = config.get("add_hidden_timestamp") + config["add_hidden_timestamp"] = False yield if orig_config_val is not None: - config["enable_hidden_attributes"] = orig_config_val + config["add_hidden_timestamp"] = orig_config_val def test_schema_decorator(schema_any): @@ -392,17 +392,15 @@ class Table_With_Underscores(dj.Manual): schema_any(Table_With_Underscores) -def test_hidden_attributes_default_value(): - config_val = config.get("enable_hidden_attributes") +def test_add_hidden_timestamp_default_value(): + config_val = config.get("add_hidden_timestamp") assert ( config_val is not None and not config_val - ), "Default value for enable_hidden_attributes is not False" + ), "Default value for add_hidden_timestamp is not False" -def test_hidden_attributes_enabled(enable_hidden_attributes, schema_any): - orig_config_val = config.get("enable_hidden_attributes") - config["enable_hidden_attributes"] = True - +def test_add_hidden_timestamp_enabled(enable_add_hidden_timestamp, schema_any): + assert config["add_hidden_timestamp"], "add_hidden_timestamp is not enabled" msg = f"{Experiment().heading._attributes=}" assert any( a.name.endswith("_timestamp") for a in Experiment().heading._attributes.values() @@ -413,14 +411,9 @@ def test_hidden_attributes_enabled(enable_hidden_attributes, schema_any): assert any(a.is_hidden for a in Experiment().heading._attributes.values()), msg assert not any(a.is_hidden for a in Experiment().heading.attributes.values()), msg - if orig_config_val is not None: - config["enable_hidden_attributes"] = orig_config_val - - -def test_hidden_attributes_disabled(disable_hidden_attributes, schema_any): - orig_config_val = config.get("enable_hidden_attributes") - config["enable_hidden_attributes"] = False +def test_add_hidden_timestamp_disabled(disable_add_hidden_timestamp, schema_any): + assert not config["add_hidden_timestamp"], "expected add_hidden_timestamp to be False" msg = f"{Experiment().heading._attributes=}" assert not any( a.name.endswith("_timestamp") for a in Experiment().heading._attributes.values() @@ -430,6 +423,3 @@ def test_hidden_attributes_disabled(disable_hidden_attributes, schema_any): ), msg assert not any(a.is_hidden for a in Experiment().heading._attributes.values()), msg assert not any(a.is_hidden for a in Experiment().heading.attributes.values()), msg - - if orig_config_val is not None: - config["enable_hidden_attributes"] = orig_config_val From e300770c165e065d9ed913d4c6f45f0493bf2320 Mon Sep 17 00:00:00 2001 From: Ethan Ho <53266718+ethho@users.noreply.github.com> Date: Sun, 22 Sep 2024 14:04:47 -0500 Subject: [PATCH 4/6] Format with black==24.4.2 --- tests/test_declare.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_declare.py b/tests/test_declare.py index 1b186778..b3c92829 100644 --- a/tests/test_declare.py +++ b/tests/test_declare.py @@ -413,7 +413,9 @@ def test_add_hidden_timestamp_enabled(enable_add_hidden_timestamp, schema_any): def test_add_hidden_timestamp_disabled(disable_add_hidden_timestamp, schema_any): - assert not config["add_hidden_timestamp"], "expected add_hidden_timestamp to be False" + assert not config[ + "add_hidden_timestamp" + ], "expected add_hidden_timestamp to be False" msg = f"{Experiment().heading._attributes=}" assert not any( a.name.endswith("_timestamp") for a in Experiment().heading._attributes.values() From 18983d71a7a4ecbc32c827b627d46b7d890c771a Mon Sep 17 00:00:00 2001 From: Ethan Ho <53266718+ethho@users.noreply.github.com> Date: Sun, 22 Sep 2024 14:17:11 -0500 Subject: [PATCH 5/6] Update CHANGELOG --- CHANGELOG.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a64ca378..e7581fc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,15 @@ ## Release notes ### 0.14.3 -- Sep 20, 2024 -- Added - `dj.Top` restriction ([#1024](https://github.com/datajoint/datajoint-python/issues/1024)) PR [#1084](https://github.com/datajoint/datajoint-python/pull/1084) +- Added - `dj.Top` restriction - PR [#1024](https://github.com/datajoint/datajoint-python/issues/1024)) PR [#1084](https://github.com/datajoint/datajoint-python/pull/1084) - Fixed - Added encapsulating double quotes to comply with [DOT language](https://graphviz.org/doc/info/lang.html) - PR [#1177](https://github.com/datajoint/datajoint-python/pull/1177) -- Added - Datajoint python CLI ([#940](https://github.com/datajoint/datajoint-python/issues/940)) PR [#1095](https://github.com/datajoint/datajoint-python/pull/1095) +- Added - Datajoint python CLI ([#940](https://github.com/datajoint/datajoint-python/issues/940)) - PR [#1095](https://github.com/datajoint/datajoint-python/pull/1095) - Added - Ability to set hidden attributes on a table - PR [#1091](https://github.com/datajoint/datajoint-python/pull/1091) - Added - Ability to specify a list of keys to populate - PR [#989](https://github.com/datajoint/datajoint-python/pull/989) -- Fixed - fixed topological sort [#1057](https://github.com/datajoint/datajoint-python/issues/1057)- PR [#1184](https://github.com/datajoint/datajoint-python/pull/1184) -- Fixed - .parts() not always returning parts [#1103](https://github.com/datajoint/datajoint-python/issues/1103)- PR [#1184](https://github.com/datajoint/datajoint-python/pull/1184) -- Changed - replace `setup.py` with `pyproject.toml` PR [#1183](https://github.com/datajoint/datajoint-python/pull/1183) +- Fixed - fixed topological sort [#1057](https://github.com/datajoint/datajoint-python/issues/1057) - PR [#1184](https://github.com/datajoint/datajoint-python/pull/1184) +- Fixed - .parts() not always returning parts [#1103](https://github.com/datajoint/datajoint-python/issues/1103) - PR [#1184](https://github.com/datajoint/datajoint-python/pull/1184) +- Changed - replace `setup.py` with `pyproject.toml` - PR [#1183](https://github.com/datajoint/datajoint-python/pull/1183) +- Changed - disable `add_hidden_timestamp` configuration option by default - PR [#1188](https://github.com/datajoint/datajoint-python/pull/1188) ### 0.14.2 -- Aug 19, 2024 - Added - Migrate nosetests to pytest - PR [#1142](https://github.com/datajoint/datajoint-python/pull/1142) From 23d95793e47873354fe75b94bed66136f2d5d641 Mon Sep 17 00:00:00 2001 From: Ethan Ho <53266718+ethho@users.noreply.github.com> Date: Sun, 22 Sep 2024 14:27:16 -0500 Subject: [PATCH 6/6] Update release date for 0.14.3 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7581fc0..1a7b8603 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ## Release notes -### 0.14.3 -- Sep 20, 2024 +### 0.14.3 -- Sep 23, 2024 - Added - `dj.Top` restriction - PR [#1024](https://github.com/datajoint/datajoint-python/issues/1024)) PR [#1084](https://github.com/datajoint/datajoint-python/pull/1084) - Fixed - Added encapsulating double quotes to comply with [DOT language](https://graphviz.org/doc/info/lang.html) - PR [#1177](https://github.com/datajoint/datajoint-python/pull/1177) - Added - Datajoint python CLI ([#940](https://github.com/datajoint/datajoint-python/issues/940)) - PR [#1095](https://github.com/datajoint/datajoint-python/pull/1095)