diff --git a/packages/zarr-metadata/changes/4037.misc.md b/packages/zarr-metadata/changes/4037.misc.md new file mode 100644 index 0000000000..fee5c69cca --- /dev/null +++ b/packages/zarr-metadata/changes/4037.misc.md @@ -0,0 +1 @@ +Introduces a new `JSONValue` type that models python objects that serialize directly to JSON. This type is used to annotate the contents of `attributes` and `fill_value` fields, replacing the use of the overly wide `object` type. This is technically a breaking change. \ No newline at end of file diff --git a/packages/zarr-metadata/src/zarr_metadata/__init__.py b/packages/zarr-metadata/src/zarr_metadata/__init__.py index dc9bc25c63..7c6461500e 100644 --- a/packages/zarr-metadata/src/zarr_metadata/__init__.py +++ b/packages/zarr-metadata/src/zarr_metadata/__init__.py @@ -1,6 +1,6 @@ from importlib.metadata import version -from zarr_metadata._common import NamedConfig +from zarr_metadata._common import JSONValue, NamedConfig from zarr_metadata.v2.array import ( ArrayDimensionSeparatorV2, ArrayMetadataV2, @@ -37,6 +37,7 @@ "GroupMetadataV2Partial", "GroupMetadataV3", "GroupMetadataV3Partial", + "JSONValue", "MetadataFieldV3", "NamedConfig", "ZArrayMetadata", diff --git a/packages/zarr-metadata/src/zarr_metadata/_common.py b/packages/zarr-metadata/src/zarr_metadata/_common.py index 9aa643e6d0..f6064d863f 100644 --- a/packages/zarr-metadata/src/zarr_metadata/_common.py +++ b/packages/zarr-metadata/src/zarr_metadata/_common.py @@ -9,7 +9,19 @@ from collections.abc import Mapping from typing import NotRequired -from typing_extensions import TypedDict +from typing_extensions import TypeAliasType, TypedDict + +JSONValue = TypeAliasType( + "JSONValue", + "int | float | bool | None | str | list[JSONValue] | tuple[JSONValue, ...] | Mapping[str, JSONValue]", # type: ignore[reportInvalidTypeForm] +) +"""A recursive type alias for JSON-encodable values. + +Defined via `TypeAliasType` (rather than a plain `TypeAlias`) so the +self-reference is a named recursion point that pydantic can resolve when +building a `TypeAdapter`; a bare recursive `TypeAlias` raises +`PydanticUserError`/`RecursionError` at validation time. +""" class NamedConfig(TypedDict): @@ -17,9 +29,8 @@ class NamedConfig(TypedDict): Externally-tagged union member for a metadata field. The `configuration` mapping holds arbitrary JSON-encodable values; - it is typed as `Mapping[str, object]` because the type system cannot - express or verify JSON-encodability. + it is typed as `Mapping[str, JSONValue]`. """ name: str - configuration: NotRequired[Mapping[str, object]] + configuration: NotRequired[Mapping[str, JSONValue]] diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/array.py b/packages/zarr-metadata/src/zarr_metadata/v2/array.py index 6673716f6c..999c341dc7 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/array.py @@ -5,6 +5,7 @@ from typing_extensions import TypedDict +from zarr_metadata._common import JSONValue from zarr_metadata.v2.codec import CodecMetadataV2 DataTypeMetadataV2 = str | tuple[tuple[str, str] | tuple[str, str, tuple[int, ...]], ...] @@ -61,7 +62,7 @@ class ZArrayMetadata(TypedDict): chunks: tuple[int, ...] dtype: DataTypeMetadataV2 compressor: CodecMetadataV2 | None - fill_value: object + fill_value: JSONValue order: ArrayOrderV2 filters: tuple[CodecMetadataV2, ...] | None dimension_separator: NotRequired[ArrayDimensionSeparatorV2] @@ -87,11 +88,11 @@ class ArrayMetadataV2(TypedDict): chunks: tuple[int, ...] dtype: DataTypeMetadataV2 compressor: CodecMetadataV2 | None - fill_value: object + fill_value: JSONValue order: ArrayOrderV2 filters: tuple[CodecMetadataV2, ...] | None dimension_separator: NotRequired[ArrayDimensionSeparatorV2] - attributes: NotRequired[Mapping[str, object]] + attributes: NotRequired[Mapping[str, JSONValue]] """User attributes from the sibling `.zattrs` file (not part of `.zarray`). See the class docstring for the rationale behind the merged representation. @@ -128,11 +129,11 @@ class ArrayMetadataV2Partial(TypedDict, total=False): chunks: tuple[int, ...] dtype: DataTypeMetadataV2 compressor: CodecMetadataV2 | None - fill_value: object + fill_value: JSONValue order: ArrayOrderV2 filters: tuple[CodecMetadataV2, ...] | None dimension_separator: NotRequired[ArrayDimensionSeparatorV2] - attributes: NotRequired[Mapping[str, object]] + attributes: NotRequired[Mapping[str, JSONValue]] """User attributes from the sibling `.zattrs` file (not part of `.zarray`). See the class docstring for the rationale behind the merged representation. diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py b/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py index f260537b80..18b8ded9da 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/attributes.py @@ -5,7 +5,9 @@ from collections.abc import Mapping -ZAttrsMetadata = Mapping[str, object] +from zarr_metadata._common import JSONValue + +ZAttrsMetadata = Mapping[str, JSONValue] """On-disk `.zattrs` file content. A JSON object holding user-defined attributes for a v2 array or group. diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/codec.py b/packages/zarr-metadata/src/zarr_metadata/v2/codec.py index 5b42432e8c..6d194b7e29 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/codec.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/codec.py @@ -7,8 +7,10 @@ from typing_extensions import TypedDict +from zarr_metadata._common import JSONValue -class CodecMetadataV2(TypedDict, extra_items=object): # type: ignore[call-arg] + +class CodecMetadataV2(TypedDict, extra_items=JSONValue): # type: ignore[call-arg] """ A numcodecs configuration dict, used as a v2 compressor or filter. diff --git a/packages/zarr-metadata/src/zarr_metadata/v2/group.py b/packages/zarr-metadata/src/zarr_metadata/v2/group.py index fd14960a0b..5f456fe8d3 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v2/group.py +++ b/packages/zarr-metadata/src/zarr_metadata/v2/group.py @@ -8,6 +8,8 @@ from typing_extensions import TypedDict +from zarr_metadata._common import JSONValue + class ZGroupMetadata(TypedDict): """ @@ -39,7 +41,7 @@ class GroupMetadataV2(TypedDict): """ zarr_format: Literal[2] - attributes: NotRequired[Mapping[str, object]] + attributes: NotRequired[Mapping[str, JSONValue]] class GroupMetadataV2Partial(TypedDict, total=False): @@ -69,7 +71,7 @@ class GroupMetadataV2Partial(TypedDict, total=False): """ zarr_format: Literal[2] - attributes: NotRequired[Mapping[str, object]] + attributes: NotRequired[Mapping[str, JSONValue]] __all__ = [ diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/array.py b/packages/zarr-metadata/src/zarr_metadata/v3/array.py index 30bc991f47..d9cea4aef4 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/array.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/array.py @@ -5,10 +5,11 @@ from typing_extensions import TypedDict +from zarr_metadata._common import JSONValue from zarr_metadata.v3._common import MetadataFieldV3 -class ExtensionFieldV3(TypedDict, extra_items=object): # type: ignore[call-arg] +class ExtensionFieldV3(TypedDict, extra_items=JSONValue): # type: ignore[call-arg] """ Required shape of any extension field on a v3 metadata document. @@ -55,9 +56,9 @@ class ArrayMetadataV3(TypedDict, extra_items=ExtensionFieldV3): # type: ignore[ shape: tuple[int, ...] chunk_grid: MetadataFieldV3 chunk_key_encoding: MetadataFieldV3 - fill_value: object + fill_value: JSONValue codecs: tuple[MetadataFieldV3, ...] - attributes: NotRequired[Mapping[str, object]] + attributes: NotRequired[Mapping[str, JSONValue]] storage_transformers: NotRequired[tuple[MetadataFieldV3, ...]] dimension_names: NotRequired[tuple[str | None, ...]] @@ -91,9 +92,9 @@ class ArrayMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV shape: tuple[int, ...] chunk_grid: MetadataFieldV3 chunk_key_encoding: MetadataFieldV3 - fill_value: object + fill_value: JSONValue codecs: tuple[MetadataFieldV3, ...] - attributes: NotRequired[Mapping[str, object]] + attributes: NotRequired[Mapping[str, JSONValue]] storage_transformers: NotRequired[tuple[MetadataFieldV3, ...]] dimension_names: NotRequired[tuple[str | None, ...]] diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py index 17905bf38a..fd6fb2ee4a 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/cast_value.py @@ -8,6 +8,7 @@ from typing_extensions import TypedDict +from zarr_metadata._common import JSONValue from zarr_metadata.v3._common import MetadataFieldV3 CAST_VALUE_CODEC_NAME: Final = "cast_value" @@ -46,7 +47,7 @@ OUT_OF_RANGE_MODE: Final = ("clamp", "wrap") """Tuple of permitted values for the `out_of_range` field of the `cast_value` codec.""" -ScalarMapEntry = tuple[object, object] +ScalarMapEntry = tuple[JSONValue, JSONValue] """A single `[input, output]` mapping in a `scalar_map` direction. Each scalar is JSON-encoded per its data type's fill-value rules (so diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py index 32e824ed67..9701db8497 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/codec/scale_offset.py @@ -8,6 +8,8 @@ from typing_extensions import TypedDict +from zarr_metadata._common import JSONValue + SCALE_OFFSET_CODEC_NAME: Final = "scale_offset" """The `name` field value of the `scale_offset` codec.""" @@ -26,8 +28,8 @@ class ScaleOffsetCodecConfiguration(TypedDict): permitted in addition to numbers. """ - offset: NotRequired[object] - scale: NotRequired[object] + offset: NotRequired[JSONValue] + scale: NotRequired[JSONValue] class ScaleOffsetCodecObject(TypedDict): diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py index 284ba8e482..282bcc83d6 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/data_type/struct.py @@ -9,6 +9,9 @@ from typing_extensions import ReadOnly, TypedDict +from zarr_metadata._common import JSONValue +from zarr_metadata.v3._common import MetadataFieldV3 + STRUCT_DATA_TYPE_NAME: Final = "struct" """The `name` field value of the `struct` data type.""" @@ -30,7 +33,7 @@ class StructField(TypedDict): """ name: ReadOnly[str] - data_type: ReadOnly[object] + data_type: ReadOnly[MetadataFieldV3] class StructConfiguration(TypedDict): @@ -46,7 +49,7 @@ class Struct(TypedDict): configuration: StructConfiguration -StructFillValue = Mapping[str, object] +StructFillValue = Mapping[str, JSONValue] """Permitted JSON shape of the `fill_value` field for `struct`. A JSON object mapping each field name to that field's fill value. Field diff --git a/packages/zarr-metadata/src/zarr_metadata/v3/group.py b/packages/zarr-metadata/src/zarr_metadata/v3/group.py index e44bf124e3..27186b6059 100644 --- a/packages/zarr-metadata/src/zarr_metadata/v3/group.py +++ b/packages/zarr-metadata/src/zarr_metadata/v3/group.py @@ -8,6 +8,7 @@ from typing_extensions import TypedDict +from zarr_metadata._common import JSONValue from zarr_metadata.v3.array import ExtensionFieldV3 @@ -22,7 +23,7 @@ class GroupMetadataV3(TypedDict, extra_items=ExtensionFieldV3): # type: ignore[ zarr_format: Literal[3] node_type: Literal["group"] - attributes: NotRequired[Mapping[str, object]] + attributes: NotRequired[Mapping[str, JSONValue]] class GroupMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV3): # type: ignore[call-arg] @@ -50,7 +51,7 @@ class GroupMetadataV3Partial(TypedDict, total=False, extra_items=ExtensionFieldV zarr_format: Literal[3] node_type: Literal["group"] - attributes: NotRequired[Mapping[str, object]] + attributes: NotRequired[Mapping[str, JSONValue]] __all__ = [