-
Notifications
You must be signed in to change notification settings - Fork 7
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #793 from fractal-analytics-platform/pydantic-v2-s…
…econd-attempt Fully move to Pydantic v2
Showing
68 changed files
with
1,515 additions
and
1,122 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,81 @@ | ||
""" | ||
Custom Pydantic v2 JSON Schema generation tools. | ||
As of Pydantic V2, the JSON Schema representation of model attributes marked | ||
as `Optional` changed, and the new behavior consists in marking the | ||
corresponding properties as an `anyOf` of either a `null` or the actual type. | ||
This is not always the required behavior, see e.g. | ||
* https://github.com/pydantic/pydantic/issues/7161 | ||
* https://github.com/pydantic/pydantic/issues/8394 | ||
Here we list some alternative ways of reverting this change. | ||
""" | ||
import logging | ||
|
||
from pydantic.json_schema import GenerateJsonSchema | ||
from pydantic.json_schema import JsonSchemaValue | ||
from pydantic_core.core_schema import WithDefaultSchema | ||
|
||
logger = logging.getLogger("CustomGenerateJsonSchema") | ||
|
||
|
||
class CustomGenerateJsonSchema(GenerateJsonSchema): | ||
def get_flattened_anyof( | ||
self, schemas: list[JsonSchemaValue] | ||
) -> JsonSchemaValue: | ||
null_schema = {"type": "null"} | ||
if null_schema in schemas: | ||
logger.warning( | ||
"Drop `null_schema` before calling `get_flattened_anyof`" | ||
) | ||
schemas.pop(schemas.index(null_schema)) | ||
return super().get_flattened_anyof(schemas) | ||
|
||
def default_schema(self, schema: WithDefaultSchema) -> JsonSchemaValue: | ||
json_schema = super().default_schema(schema) | ||
if "default" in json_schema.keys() and json_schema["default"] is None: | ||
logger.warning(f"Pop `None` default value from {json_schema=}") | ||
json_schema.pop("default") | ||
return json_schema | ||
|
||
|
||
# class GenerateJsonSchemaA(GenerateJsonSchema): | ||
# def nullable_schema(self, schema): | ||
# null_schema = {"type": "null"} | ||
# inner_json_schema = self.generate_inner(schema["schema"]) | ||
# if inner_json_schema == null_schema: | ||
# return null_schema | ||
# else: | ||
# logging.info("A: Skip calling `get_flattened_anyof` method") | ||
# return inner_json_schema | ||
|
||
|
||
# class GenerateJsonSchemaB(GenerateJsonSchemaA): | ||
# def default_schema(self, schema: WithDefaultSchema) -> JsonSchemaValue: | ||
# original_json_schema = super().default_schema(schema) | ||
# new_json_schema = deepcopy(original_json_schema) | ||
# default = new_json_schema.get("default", None) | ||
# if default is None: | ||
# logging.info("B: Pop None default") | ||
# new_json_schema.pop("default") | ||
# return new_json_schema | ||
|
||
|
||
# class GenerateJsonSchemaC(GenerateJsonSchema): | ||
# def get_flattened_anyof( | ||
# self, schemas: list[JsonSchemaValue] | ||
# ) -> JsonSchemaValue: | ||
|
||
# original_json_schema_value = super().get_flattened_anyof(schemas) | ||
# members = original_json_schema_value.get("anyOf") | ||
# logging.info("C", original_json_schema_value) | ||
# if ( | ||
# members is not None | ||
# and len(members) == 2 | ||
# and {"type": "null"} in members | ||
# ): | ||
# new_json_schema_value = {"type": [t["type"] for t in members]} | ||
# logging.info("C", new_json_schema_value) | ||
# return new_json_schema_value | ||
# else: | ||
# return original_json_schema_value |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
9 changes: 8 additions & 1 deletion
9
tests/data/fake_package_for_schema_experiments/my_package/lib_custom_models.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,5 +1,12 @@ | ||
from pydantic.v1 import BaseModel | ||
from pydantic import BaseModel | ||
|
||
|
||
class CustomModel(BaseModel): | ||
""" | ||
Short description | ||
Attributes: | ||
x: Description of `x` | ||
""" | ||
|
||
x: int |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,5 +1,5 @@ | ||
{ | ||
"path": 0, | ||
"path": "0", | ||
"coordinateTransformations": [ | ||
{ | ||
"type": "scale", | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,5 +1,5 @@ | ||
{ | ||
"path": 0, | ||
"path": "0", | ||
"coordinateTransformations": [ | ||
{ | ||
"type": "translation", | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,5 +1,5 @@ | ||
{ | ||
"path": 0, | ||
"path": "0", | ||
"coordinateTransformations": [ | ||
{ | ||
"type": "scale", | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -24,7 +24,7 @@ | |
], | ||
"datasets": [ | ||
{ | ||
"path": 0, | ||
"path": "0", | ||
"coordinateTransformations": [ | ||
{ | ||
"type": "scale", | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -15,7 +15,7 @@ | |
], | ||
"datasets": [ | ||
{ | ||
"path": 0, | ||
"path": "0", | ||
"coordinateTransformations": [ | ||
{ | ||
"type": "scale", | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,39 @@ | ||
{ | ||
"axes": [ | ||
{ | ||
"name": "c", | ||
"type": "channel" | ||
}, | ||
{ | ||
"name": "c", | ||
"type": "channel" | ||
}, | ||
{ | ||
"name": "y", | ||
"type": "space", | ||
"unit": "micrometer" | ||
}, | ||
{ | ||
"name": "x", | ||
"type": "space", | ||
"unit": "micrometer" | ||
} | ||
], | ||
"datasets": [ | ||
{ | ||
"path": "0", | ||
"coordinateTransformations": [ | ||
{ | ||
"type": "scale", | ||
"scale": [ | ||
1.0, | ||
1.0, | ||
0.1625, | ||
0.1625 | ||
] | ||
} | ||
] | ||
} | ||
], | ||
"version": "0.4" | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,97 @@ | ||
{ | ||
"coordinateTransformations": null, | ||
"axes": [ | ||
{ | ||
"name": "c", | ||
"type": "channel" | ||
}, | ||
{ | ||
"name": "z", | ||
"type": "space", | ||
"unit": "micrometer" | ||
}, | ||
{ | ||
"name": "y", | ||
"type": "space", | ||
"unit": "micrometer" | ||
}, | ||
{ | ||
"name": "x", | ||
"type": "space", | ||
"unit": "micrometer" | ||
} | ||
], | ||
"datasets": [ | ||
{ | ||
"path": "0", | ||
"coordinateTransformations": [ | ||
{ | ||
"type": "scale", | ||
"scale": [ | ||
1.0, | ||
1.0, | ||
0.1625, | ||
0.1625 | ||
] | ||
} | ||
] | ||
}, | ||
{ | ||
"path": "1", | ||
"coordinateTransformations": [ | ||
{ | ||
"type": "scale", | ||
"scale": [ | ||
1.0, | ||
1.0, | ||
0.325, | ||
0.325 | ||
] | ||
} | ||
] | ||
}, | ||
{ | ||
"path": "2", | ||
"coordinateTransformations": [ | ||
{ | ||
"type": "scale", | ||
"scale": [ | ||
1.0, | ||
1.0, | ||
0.65, | ||
0.65 | ||
] | ||
} | ||
] | ||
}, | ||
{ | ||
"path": "3", | ||
"coordinateTransformations": [ | ||
{ | ||
"type": "scale", | ||
"scale": [ | ||
1.0, | ||
1.0, | ||
1.3, | ||
1.3 | ||
] | ||
} | ||
] | ||
}, | ||
{ | ||
"path": "4", | ||
"coordinateTransformations": [ | ||
{ | ||
"type": "scale", | ||
"scale": [ | ||
1.0, | ||
1.0, | ||
2.6, | ||
2.6 | ||
] | ||
} | ||
] | ||
} | ||
], | ||
"version": "0.4" | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
{ | ||
"tables": [ | ||
"FOV_ROI_table" | ||
] | ||
} |
6 changes: 3 additions & 3 deletions
6
tests/data/plate_ones.zarr/B/03/0/tables/FOV_ROI_table/.zattrs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,6 +1,6 @@ | ||
{ | ||
"fractal_table_version": "1", | ||
"type": "roi_table", | ||
"encoding-type": "anndata", | ||
"encoding-version": "0.1.0" | ||
"encoding-version": "0.1.0", | ||
"fractal_table_version": "1", | ||
"type": "roi_table" | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,48 @@ | ||
import json | ||
from typing import Optional | ||
|
||
from pydantic.validate_call_decorator import validate_call | ||
|
||
from fractal_tasks_core.dev.lib_args_schemas import ( | ||
create_schema_for_single_task, | ||
) | ||
|
||
|
||
@validate_call | ||
def task_function( | ||
arg1: str, | ||
arg2: Optional[str] = None, | ||
arg3: Optional[list[str]] = None, | ||
): | ||
""" | ||
Short task description | ||
|
||
Args: | ||
arg1: This is the argument description | ||
arg2: This is the argument description | ||
arg3: This is the argument description | ||
""" | ||
pass | ||
|
||
|
||
def test_optional_argument(): | ||
""" | ||
As a first implementation of the Pydantic V2 schema generation, we are not | ||
supporting the `anyOf` pattern for nullable attributes. This test verifies | ||
that the type of nullable properties is not `anyOf`, and that they are not | ||
required. | ||
|
||
Note: future versions of fractal-tasks-core may change this behavior. | ||
""" | ||
schema = create_schema_for_single_task( | ||
task_function=validate_call(task_function), | ||
executable=__file__, | ||
package=None, | ||
verbose=True, | ||
) | ||
print(json.dumps(schema, indent=2, sort_keys=True)) | ||
print() | ||
assert schema["properties"]["arg2"]["type"] == "string" | ||
assert "arg2" not in schema["required"] | ||
assert schema["properties"]["arg3"]["type"] == "array" | ||
assert "arg3" not in schema["required"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,36 @@ | ||
import json | ||
|
||
from devtools import debug | ||
from pydantic import validate_call | ||
|
||
from fractal_tasks_core.dev.lib_args_schemas import ( | ||
create_schema_for_single_task, | ||
) | ||
|
||
|
||
@validate_call | ||
def task_function(arg_A: tuple[int, int] = (1, 1)): | ||
""" | ||
Short task description | ||
|
||
Args: | ||
arg_A: Description of arg_A. | ||
""" | ||
pass | ||
|
||
|
||
def test_tuple_argument(): | ||
""" | ||
This test only asserts that `create_schema_for_single_task` runs | ||
successfully. Its goal is also to offer a quick way to experiment | ||
with new task arguments and play with the generated JSON Schema, | ||
without re-building the whole fractal-tasks-core manifest. | ||
""" | ||
schema = create_schema_for_single_task( | ||
task_function=task_function, | ||
executable=__file__, | ||
package=None, | ||
verbose=True, | ||
) | ||
debug(schema) | ||
print(json.dumps(schema, indent=2)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters