From 0b5cc409f46d56eb8414ea78549c477386b969bd Mon Sep 17 00:00:00 2001 From: Martin Wellman Date: Fri, 24 May 2024 15:03:28 -0700 Subject: [PATCH 1/5] Support for enum mapping for slots with multiple enum ranges --- .../transformer/object_transformer.py | 59 +++++++++++-------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/src/linkml_map/transformer/object_transformer.py b/src/linkml_map/transformer/object_transformer.py index 6126b7f..f606496 100644 --- a/src/linkml_map/transformer/object_transformer.py +++ b/src/linkml_map/transformer/object_transformer.py @@ -181,10 +181,15 @@ def map_object( elif target_type == "curie": return self.compress_uri(source_obj) return source_obj - if source_type in sv.all_enums(): - # TODO: enum derivations - return self.transform_enum(source_obj, source_type, source_obj) - # return str(source_obj) + + # Do enumeration transform if source_type has enumeration name(s) + source_type_enums = yaml.safe_load(source_type) + if not isinstance(source_type_enums, list): + source_type_enums = [source_type_enums] + source_type_enums = [enum for enum in source_type_enums if enum in sv.all_enums()] + if len(source_type_enums) > 0: + return self.transform_enum(source_obj, source_type_enums, source_obj) + source_obj_typed = None if isinstance(source_obj, (BaseModel, YAMLRoot)): # ensure dict @@ -424,24 +429,28 @@ def transform_object( tr_obj_dict = self.map_object(source_obj, source_type_name) return target_class(**tr_obj_dict) - def transform_enum(self, source_value: str, enum_name: str, source_obj: Any) -> Optional[str]: - enum_deriv = self._get_enum_derivation(enum_name) - if enum_deriv.expr: - try: - if enum_deriv.expr: - v = eval_expr(enum_deriv.expr, **source_obj, NULL=None) - except Exception: - aeval = Interpreter(usersyms={"src": source_obj, "target": None}) - aeval(enum_deriv.expr) - v = aeval.symtable["target"] - if v is not None: - return v - for pv_deriv in enum_deriv.permissible_value_derivations.values(): - if source_value == pv_deriv.populated_from: - return pv_deriv.name - if source_value in pv_deriv.sources: - return pv_deriv.name - if enum_deriv.mirror_source: - return str(source_value) - else: - return None + def transform_enum( + self, source_value: str, enum_name: Union[str, List[str]], source_obj: Any + ) -> Optional[str]: + if isinstance(enum_name, str): + enum_name = [enum_name] + for cur_enum in enum_name: + enum_deriv = self._get_enum_derivation(cur_enum) + if enum_deriv.expr: + try: + if enum_deriv.expr: + v = eval_expr(enum_deriv.expr, **source_obj, NULL=None) + except Exception: + aeval = Interpreter(usersyms={"src": source_obj, "target": None}) + aeval(enum_deriv.expr) + v = aeval.symtable["target"] + if v is not None: + return v + for pv_deriv in enum_deriv.permissible_value_derivations.values(): + if source_value == pv_deriv.populated_from: + return pv_deriv.name + if source_value in pv_deriv.sources: + return pv_deriv.name + if enum_deriv.mirror_source: + return str(source_value) + return None From a6ee85dc8373767e97101c962e7648464f09a766 Mon Sep 17 00:00:00 2001 From: Martin Wellman Date: Thu, 20 Jun 2024 13:48:29 -0700 Subject: [PATCH 2/5] Unit test for mapping ranges with multiple enumerations --- .../multi_enum/data/Container-001.yaml | 33 +++++++ .../output/Container-001.transformed.yaml | 33 +++++++ .../examples/multi_enum/source/lights.yaml | 71 ++++++++++++++ .../multi_enum/target/lights_inventory.yaml | 61 ++++++++++++ .../lights_to_lights_simple.transform.yaml | 93 +++++++++++++++++++ .../test_transformer_examples.py | 1 + 6 files changed, 292 insertions(+) create mode 100644 tests/input/examples/multi_enum/data/Container-001.yaml create mode 100644 tests/input/examples/multi_enum/output/Container-001.transformed.yaml create mode 100644 tests/input/examples/multi_enum/source/lights.yaml create mode 100644 tests/input/examples/multi_enum/target/lights_inventory.yaml create mode 100644 tests/input/examples/multi_enum/transform/lights_to_lights_simple.transform.yaml diff --git a/tests/input/examples/multi_enum/data/Container-001.yaml b/tests/input/examples/multi_enum/data/Container-001.yaml new file mode 100644 index 0000000..c23bfa4 --- /dev/null +++ b/tests/input/examples/multi_enum/data/Container-001.yaml @@ -0,0 +1,33 @@ +lights: + - colors: + - light_red + - dark_green + - dark_blue + main_color: dark_green + sizes: + - xsmall + - small + - medium + - large + - xlarge + sku: a0001 + - colors: + - light_green + main_color: light_green + sizes: + - small + - other + sku: a0002 + - colors: + - other + main_color: other + sizes: + - medium + sku: a0003 + - colors: + - light_blue + main_color: light_blue + sizes: + - xsmall + - not_available + sku: a0004 diff --git a/tests/input/examples/multi_enum/output/Container-001.transformed.yaml b/tests/input/examples/multi_enum/output/Container-001.transformed.yaml new file mode 100644 index 0000000..e3bcd1e --- /dev/null +++ b/tests/input/examples/multi_enum/output/Container-001.transformed.yaml @@ -0,0 +1,33 @@ +lights_inventory: +- light_sizes: + - size_0 + - size_0 + - size_1 + - size_2 + - size_2 + light_colors: + - red + - green + - blue + light_main_color: green + SKU: a0001 +- light_sizes: + - size_0 + - oth + light_colors: + - green + light_main_color: green + SKU: a0002 +- light_sizes: + - size_1 + light_colors: + - oth + light_main_color: oth + SKU: a0003 +- light_sizes: + - size_0 + - na + light_colors: + - blue + light_main_color: blue + SKU: a0004 diff --git a/tests/input/examples/multi_enum/source/lights.yaml b/tests/input/examples/multi_enum/source/lights.yaml new file mode 100644 index 0000000..6994e0d --- /dev/null +++ b/tests/input/examples/multi_enum/source/lights.yaml @@ -0,0 +1,71 @@ +name: light_bulbs +id: light_bulbs +description: Light Bulbs +imports: +- linkml:types +default_range: string +enums: + size: + permissible_values: + xsmall: + small: + medium: + large: + xlarge: + primary_colors: + permissible_values: + light_red: + dark_red: + light_green: + dark_green: + light_blue: + dark_blue: + secondary_colors: + permissible_values: + light_cyan: + dark_cyan: + light_magenta: + dark_magenta: + light_yellow: + dark_yellow: + missingness: + permissible_values: + not_available: + other: +classes: + lights: + slots: + - sku + - sizes + - colors + - main_color + Container: + tree_root: True + attributes: + lights: + range: lights + multivalued: True + inlined_as_list: True +slots: + sizes: + range: + - size + - missingness + multivalued: True + inlined_as_list: True + colors: + range: + - primary_colors + - secondary_colors + - missingness + multivalued: True + inlined_as_list: True + main_color: + range: + - primary_colors + - secondary_colors + - missingness + multivalued: False + sku: + identifier: True + range: string diff --git a/tests/input/examples/multi_enum/target/lights_inventory.yaml b/tests/input/examples/multi_enum/target/lights_inventory.yaml new file mode 100644 index 0000000..792e10b --- /dev/null +++ b/tests/input/examples/multi_enum/target/lights_inventory.yaml @@ -0,0 +1,61 @@ +name: light_bulbs_simple +id: light_bulbs_simple +description: Light Bulbs (Simplified) +imports: +- linkml:types +default_range: string +enums: + light_size_enum: + permissible_values: + size_0: + size_1: + size_2: + light_primary_colors_enum: + permissible_values: + red: + green: + blue: + light_secondary_colors_enum: + permissible_values: + cyan: + magenta: + yellow: + missingness_enum: + permissible_values: + na: + oth: +classes: + lights_inventory: + slots: + - SKU + - light_sizes + - light_colors + Container: + tree_root: True + attributes: + all_lights: + range: lights_inventory + multivalued: True + inlined_as_list: True +slots: + SKU: + range: string + light_sizes: + range: + - light_size_enum + - missingness_enum + multivalued: True + inlined_as_list: True + light_colors: + range: + - light_primary_colors_enum + - light_secondary_colors_enum + - missingness_enum + multivalued: True + inlined_as_list: True + light_main_color: + range: + - light_primary_colors_enum + - light_secondary_colors_enum + - missingness_enum + multivalued: False diff --git a/tests/input/examples/multi_enum/transform/lights_to_lights_simple.transform.yaml b/tests/input/examples/multi_enum/transform/lights_to_lights_simple.transform.yaml new file mode 100644 index 0000000..d552b5a --- /dev/null +++ b/tests/input/examples/multi_enum/transform/lights_to_lights_simple.transform.yaml @@ -0,0 +1,93 @@ +class_derivations: + lights_inventory: + name: lights_inventory + populated_from: lights + slot_derivations: + light_sizes: + name: light_sizes + populated_from: sizes + light_colors: + name: light_colors + populated_from: colors + light_main_color: + name: light_main_color + populated_from: main_color + SKU: + name: SKU + populated_from: sku + Container: + name: Container + slot_derivations: + lights_inventory: + populated_from: lights + range: string +enum_derivations: + light_size_enum: + name: light_size_enum + mirror_source: false + populated_from: size + permissible_value_derivations: + size_0: + name: size_0 + sources: + - xsmall + - small + size_1: + name: size_1 + populated_from: medium + size_2: + name: size_2 + sources: + - large + - xlarge + missingness_enum: + name: missingness_enum + mirror_source: false + populated_from: missingness + permissible_value_derivations: + na: + name: na + populated_from: not_available + oth: + name: oth + populated_from: other + light_primary_colors_enum: + name: light_primary_colors_enum + mirror_source: false + populated_from: primary_colors + permissible_value_derivations: + red: + name: red + sources: + - light_red + - dark_red + green: + name: green + sources: + - light_green + - dark_green + blue: + name: blue + sources: + - light_blue + - dark_blue + light_secondary_colors_enum: + name: light_secondary_colors_enum + mirror_source: false + populated_from: secondary_colors + permissible_value_derivations: + cyan: + name: cyan + sources: + - light_cyan + - dark_cyan + magenta: + name: magenta + sources: + - light_magenta + - dark_magenta + yellow: + name: yellow + sources: + - light_yellow + - dark_yellow diff --git a/tests/test_transformer/test_transformer_examples.py b/tests/test_transformer/test_transformer_examples.py index cce4c9d..7aaab2f 100644 --- a/tests/test_transformer/test_transformer_examples.py +++ b/tests/test_transformer/test_transformer_examples.py @@ -9,6 +9,7 @@ "personinfo_basic", "type_coercion", "biolink", + "multi_enum", ] From e9c055fed99f212c02c58ff598c2da31c5de8e19 Mon Sep 17 00:00:00 2001 From: Martin Wellman Date: Tue, 9 Jul 2024 10:14:47 -0700 Subject: [PATCH 3/5] Spelling of socioeconomic to pass Codespell check --- .github/CODE_OF_CONDUCT.md | 2 +- .../examples/multi_enum/data/Container-001.yaml | 4 ++++ .../multi_enum/output/Container-001.transformed.yaml | 4 ++++ tests/input/examples/multi_enum/source/lights.yaml | 12 ++++++++++++ .../examples/multi_enum/target/lights_inventory.yaml | 3 +++ .../transform/lights_to_lights_simple.transform.yaml | 10 ++++++++++ 6 files changed, 34 insertions(+), 1 deletion(-) diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md index 4d1ac9e..841ad1e 100644 --- a/.github/CODE_OF_CONDUCT.md +++ b/.github/CODE_OF_CONDUCT.md @@ -5,7 +5,7 @@ We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender -identity and expression, level of experience, education, socio-economic status, +identity and expression, level of experience, education, socioeconomic status, nationality, personal appearance, race, caste, color, religion, or sexual identity and orientation. diff --git a/tests/input/examples/multi_enum/data/Container-001.yaml b/tests/input/examples/multi_enum/data/Container-001.yaml index c23bfa4..2b3cb6e 100644 --- a/tests/input/examples/multi_enum/data/Container-001.yaml +++ b/tests/input/examples/multi_enum/data/Container-001.yaml @@ -4,6 +4,7 @@ lights: - dark_green - dark_blue main_color: dark_green + mirror_test_slot: light_red sizes: - xsmall - small @@ -14,6 +15,7 @@ lights: - colors: - light_green main_color: light_green + mirror_test_slot: light_magenta sizes: - small - other @@ -21,12 +23,14 @@ lights: - colors: - other main_color: other + mirror_test_slot: not_available sizes: - medium sku: a0003 - colors: - light_blue main_color: light_blue + mirror_test_slot: light_red sizes: - xsmall - not_available diff --git a/tests/input/examples/multi_enum/output/Container-001.transformed.yaml b/tests/input/examples/multi_enum/output/Container-001.transformed.yaml index e3bcd1e..d22ca7c 100644 --- a/tests/input/examples/multi_enum/output/Container-001.transformed.yaml +++ b/tests/input/examples/multi_enum/output/Container-001.transformed.yaml @@ -10,6 +10,7 @@ lights_inventory: - green - blue light_main_color: green + mirror_test_slot: red SKU: a0001 - light_sizes: - size_0 @@ -17,12 +18,14 @@ lights_inventory: light_colors: - green light_main_color: green + mirror_test_slot: magenta SKU: a0002 - light_sizes: - size_1 light_colors: - oth light_main_color: oth + mirror_test_slot: not_available SKU: a0003 - light_sizes: - size_0 @@ -30,4 +33,5 @@ lights_inventory: light_colors: - blue light_main_color: blue + mirror_test_slot: red SKU: a0004 diff --git a/tests/input/examples/multi_enum/source/lights.yaml b/tests/input/examples/multi_enum/source/lights.yaml index 6994e0d..badcd81 100644 --- a/tests/input/examples/multi_enum/source/lights.yaml +++ b/tests/input/examples/multi_enum/source/lights.yaml @@ -28,6 +28,11 @@ enums: dark_magenta: light_yellow: dark_yellow: + mirror_test_colors: + permissible_values: + purple: + turquoise: + fuschia: missingness: permissible_values: not_available: @@ -39,6 +44,7 @@ classes: - sizes - colors - main_color + - mirror_test_slot Container: tree_root: True attributes: @@ -66,6 +72,12 @@ slots: - secondary_colors - missingness multivalued: False + mirror_test_slot: + range: + - primary_colors + - secondary_colors + - mirror_test_colors + - missingness sku: identifier: True range: string diff --git a/tests/input/examples/multi_enum/target/lights_inventory.yaml b/tests/input/examples/multi_enum/target/lights_inventory.yaml index 792e10b..54fe39e 100644 --- a/tests/input/examples/multi_enum/target/lights_inventory.yaml +++ b/tests/input/examples/multi_enum/target/lights_inventory.yaml @@ -30,6 +30,7 @@ classes: - SKU - light_sizes - light_colors + - light_mirror_test_slot Container: tree_root: True attributes: @@ -59,3 +60,5 @@ slots: - light_secondary_colors_enum - missingness_enum multivalued: False + light_mirror_test_slot: + range: string diff --git a/tests/input/examples/multi_enum/transform/lights_to_lights_simple.transform.yaml b/tests/input/examples/multi_enum/transform/lights_to_lights_simple.transform.yaml index d552b5a..ba053ba 100644 --- a/tests/input/examples/multi_enum/transform/lights_to_lights_simple.transform.yaml +++ b/tests/input/examples/multi_enum/transform/lights_to_lights_simple.transform.yaml @@ -12,6 +12,9 @@ class_derivations: light_main_color: name: light_main_color populated_from: main_color + light_mirror_test_slot: + name: mirror_test_slot + populated_from: mirror_test_slot SKU: name: SKU populated_from: sku @@ -91,3 +94,10 @@ enum_derivations: sources: - light_yellow - dark_yellow + light_test_colors_enum: + # This derivation is for testing mirror_source in the line of transformations for ranges with multiple enums. + # When we reach an enum in the source range that has mirror_source=true, the mapper should stop immediately and return + # the mirrored value, ignoring any enum mappings that may occur in a later enum derivation. + name: light_test_colors_enum + mirror_source: true + populated_from: mirror_test_colors \ No newline at end of file From 1e630ff41cf4f99a3dac3c8ae64b6c0a0d4b9e57 Mon Sep 17 00:00:00 2001 From: Martin Wellman Date: Tue, 9 Jul 2024 10:16:48 -0700 Subject: [PATCH 4/5] Rename fuschia to fuchsia for Codespell --- tests/input/examples/multi_enum/source/lights.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/input/examples/multi_enum/source/lights.yaml b/tests/input/examples/multi_enum/source/lights.yaml index badcd81..d075f66 100644 --- a/tests/input/examples/multi_enum/source/lights.yaml +++ b/tests/input/examples/multi_enum/source/lights.yaml @@ -32,7 +32,7 @@ enums: permissible_values: purple: turquoise: - fuschia: + fuchsia: missingness: permissible_values: not_available: From 8bc84f8f57312da900b715cef0c4508d52b26f65 Mon Sep 17 00:00:00 2001 From: Martin Wellman Date: Tue, 9 Jul 2024 10:35:53 -0700 Subject: [PATCH 5/5] Simplified unit test for multi-enums --- .../multi_enum/data/Container-001.yaml | 24 ++--------- .../output/Container-001.transformed.yaml | 32 ++++---------- .../examples/multi_enum/source/lights.yaml | 26 ++--------- .../multi_enum/target/lights_inventory.yaml | 3 -- .../lights_to_lights_simple.transform.yaml | 43 +++++++------------ 5 files changed, 31 insertions(+), 97 deletions(-) diff --git a/tests/input/examples/multi_enum/data/Container-001.yaml b/tests/input/examples/multi_enum/data/Container-001.yaml index 2b3cb6e..91c4001 100644 --- a/tests/input/examples/multi_enum/data/Container-001.yaml +++ b/tests/input/examples/multi_enum/data/Container-001.yaml @@ -3,35 +3,19 @@ lights: - light_red - dark_green - dark_blue + - light_magenta + - not_available main_color: dark_green mirror_test_slot: light_red - sizes: - - xsmall - - small - - medium - - large - - xlarge - sku: a0001 - colors: - light_green main_color: light_green mirror_test_slot: light_magenta - sizes: - - small - - other - sku: a0002 - colors: - other main_color: other - mirror_test_slot: not_available - sizes: - - medium - sku: a0003 + mirror_test_slot: dark_purple - colors: - light_blue main_color: light_blue - mirror_test_slot: light_red - sizes: - - xsmall - - not_available - sku: a0004 + mirror_test_slot: not_available \ No newline at end of file diff --git a/tests/input/examples/multi_enum/output/Container-001.transformed.yaml b/tests/input/examples/multi_enum/output/Container-001.transformed.yaml index d22ca7c..b1bfff1 100644 --- a/tests/input/examples/multi_enum/output/Container-001.transformed.yaml +++ b/tests/input/examples/multi_enum/output/Container-001.transformed.yaml @@ -1,37 +1,21 @@ lights_inventory: -- light_sizes: - - size_0 - - size_0 - - size_1 - - size_2 - - size_2 - light_colors: +- light_colors: - red - green - blue + - magenta + - na light_main_color: green mirror_test_slot: red - SKU: a0001 -- light_sizes: - - size_0 - - oth - light_colors: +- light_colors: - green light_main_color: green mirror_test_slot: magenta - SKU: a0002 -- light_sizes: - - size_1 - light_colors: +- light_colors: - oth light_main_color: oth - mirror_test_slot: not_available - SKU: a0003 -- light_sizes: - - size_0 - - na - light_colors: + mirror_test_slot: purple +- light_colors: - blue light_main_color: blue - mirror_test_slot: red - SKU: a0004 + mirror_test_slot: not_available diff --git a/tests/input/examples/multi_enum/source/lights.yaml b/tests/input/examples/multi_enum/source/lights.yaml index d075f66..68fd29a 100644 --- a/tests/input/examples/multi_enum/source/lights.yaml +++ b/tests/input/examples/multi_enum/source/lights.yaml @@ -5,13 +5,6 @@ imports: - linkml:types default_range: string enums: - size: - permissible_values: - xsmall: - small: - medium: - large: - xlarge: primary_colors: permissible_values: light_red: @@ -30,9 +23,9 @@ enums: dark_yellow: mirror_test_colors: permissible_values: - purple: - turquoise: - fuchsia: + dark_purple: + dark_turquoise: + dark_fuchsia: missingness: permissible_values: not_available: @@ -40,8 +33,6 @@ enums: classes: lights: slots: - - sku - - sizes - colors - main_color - mirror_test_slot @@ -53,12 +44,6 @@ classes: multivalued: True inlined_as_list: True slots: - sizes: - range: - - size - - missingness - multivalued: True - inlined_as_list: True colors: range: - primary_colors @@ -77,7 +62,4 @@ slots: - primary_colors - secondary_colors - mirror_test_colors - - missingness - sku: - identifier: True - range: string + - missingness \ No newline at end of file diff --git a/tests/input/examples/multi_enum/target/lights_inventory.yaml b/tests/input/examples/multi_enum/target/lights_inventory.yaml index 54fe39e..2f7ad8a 100644 --- a/tests/input/examples/multi_enum/target/lights_inventory.yaml +++ b/tests/input/examples/multi_enum/target/lights_inventory.yaml @@ -27,7 +27,6 @@ enums: classes: lights_inventory: slots: - - SKU - light_sizes - light_colors - light_mirror_test_slot @@ -39,8 +38,6 @@ classes: multivalued: True inlined_as_list: True slots: - SKU: - range: string light_sizes: range: - light_size_enum diff --git a/tests/input/examples/multi_enum/transform/lights_to_lights_simple.transform.yaml b/tests/input/examples/multi_enum/transform/lights_to_lights_simple.transform.yaml index ba053ba..7cbf213 100644 --- a/tests/input/examples/multi_enum/transform/lights_to_lights_simple.transform.yaml +++ b/tests/input/examples/multi_enum/transform/lights_to_lights_simple.transform.yaml @@ -3,9 +3,6 @@ class_derivations: name: lights_inventory populated_from: lights slot_derivations: - light_sizes: - name: light_sizes - populated_from: sizes light_colors: name: light_colors populated_from: colors @@ -15,9 +12,6 @@ class_derivations: light_mirror_test_slot: name: mirror_test_slot populated_from: mirror_test_slot - SKU: - name: SKU - populated_from: sku Container: name: Container slot_derivations: @@ -25,24 +19,6 @@ class_derivations: populated_from: lights range: string enum_derivations: - light_size_enum: - name: light_size_enum - mirror_source: false - populated_from: size - permissible_value_derivations: - size_0: - name: size_0 - sources: - - xsmall - - small - size_1: - name: size_1 - populated_from: medium - size_2: - name: size_2 - sources: - - large - - xlarge missingness_enum: name: missingness_enum mirror_source: false @@ -95,9 +71,20 @@ enum_derivations: - light_yellow - dark_yellow light_test_colors_enum: - # This derivation is for testing mirror_source in the line of transformations for ranges with multiple enums. - # When we reach an enum in the source range that has mirror_source=true, the mapper should stop immediately and return - # the mirrored value, ignoring any enum mappings that may occur in a later enum derivation. + # This derivation is for testing mirror_source=True. When transforming a slot with multiple enumerations in the range, + # and we encounter an enum derivation with mirror_source=True, then if no permissible value derivation is matched we return + # the source value unchanged, and ignore any further enum derivations. name: light_test_colors_enum mirror_source: true - populated_from: mirror_test_colors \ No newline at end of file + populated_from: mirror_test_colors + permissible_value_derivations: + purple: + name: purple + populated_from: dark_purple + turquoise: + name: turquoise + populated_from: dark_turquoise + fuchsia: + name: fuchsia + populated_from: dark_fuchsia + \ No newline at end of file