Skip to content

Zarr — DatasetCollection#

Backs DatasetCollection.to_zarr() and DatasetCollection.from_zarr(). The 4-D (time, band, y, x) cube, one dask chunk per Zarr chunk.

Backend methods write_collection_to_zarr, read_collection_from_zarr
Module pyramids_lazy._collection_zarr
Returns a DatasetCollection on read; None (or a Delayed) on write
from pyramids.dataset import DatasetCollection

cube = DatasetCollection.read_multiple_files("rasters", with_order=False)
cube.to_zarr("cube.zarr")
back = DatasetCollection.from_zarr("cube.zarr")

Region append#

A collection can be extended along an existing axis without rewriting the store: the new data is written into a region of the existing arrays, and the collection metadata (_meta, _files) is re-finalised afterwards so a later read reconstructs the full cube. The region is translated to per-axis slices, and an unknown dimension name is refused with a ValueError.

Chunk-grid alignment is not validated. A region that straddles a chunk boundary is written as given, so it is the caller's job to append on the chunk grid.

As with the single-Dataset path, the pyramids geobox is persisted as GeoZarr attributes so standard readers can reconstruct it.

API#

pyramids_lazy._collection_zarr.write_collection_to_zarr(collection, store, *, compute=True, mode='w', storage_options=None, compressor='auto', append_dim=None, region=None) #

Serialise the 4-D (T, B, R, C) cube of collection to a Zarr store.

Each dask chunk in collection.data lands in an independent Zarr chunk file. Geobox metadata is written as attributes on the root group + the data array. See :meth:DatasetCollection.to_zarr for the full argument contract.

Source code in src/pyramids_lazy/_collection_zarr.py
def write_collection_to_zarr(
    collection: Any,
    store: str | Path | Any,
    *,
    compute: bool = True,
    mode: str = "w",
    storage_options: dict | None = None,
    compressor: Any = "auto",
    append_dim: str | None = None,
    region: dict | None = None,
) -> Any:
    """Serialise the 4-D ``(T, B, R, C)`` cube of ``collection`` to a Zarr store.

    Each dask chunk in ``collection.data`` lands in an independent Zarr chunk file. Geobox metadata
    is written as attributes on the root group + the ``data`` array. See
    :meth:`DatasetCollection.to_zarr` for the full argument contract.
    """
    files = collection._require_files("to_zarr")
    import_zarr(
        lazy_hint("DatasetCollection.to_zarr requires the 'zarr' dependency.", "zarr")
    )
    if mode == "a" and append_dim is None and region is None:
        raise ValueError(
            "mode='a' requires append_dim='time' or region=... (append_dim / "
            "region semantics); use mode='w' to (over)write the whole cube."
        )
    data = collection.data
    resolved_store = _resolve_store(store, storage_options)
    codec_kwargs = normalize_compressors(compressor)

    if append_dim is not None:
        return _append_to_zarr(collection, resolved_store, data, append_dim, compute)
    if region is not None:
        import zarr

        existing = zarr.open_group(resolved_store, mode="a")["data"]
        return data.to_zarr(
            existing,
            region=_region_to_slices(region, data.ndim),
            overwrite=False,
            compute=compute,
        )

    write_result = data.to_zarr(
        resolved_store,
        component="data",
        overwrite=(mode == "w"),
        compute=compute,
        **codec_kwargs,
    )
    if compute:
        _finalize_collection_metadata(resolved_store, collection._meta, files)
        result: Any = None
    else:
        import dask

        result = dask.delayed(_finalize_after_write)(
            write_result,
            resolved_store,
            collection._meta,
            files,
        )
    return result

pyramids_lazy._collection_zarr.read_collection_from_zarr(collection_cls, store, *, storage_options=None) #

Open a pyramids-written cube Zarr store into a lazy DatasetCollection.

Inverse of :func:write_collection_to_zarr. The 4-D (time, band, y, x) data array is read lazily straight from the store, and the geobox is recovered from the GeoZarr spatial_ref mapping. collection_cls is the DatasetCollection class to construct.

Source code in src/pyramids_lazy/_collection_zarr.py
def read_collection_from_zarr(
    collection_cls: Any,
    store: str | Path | Any,
    *,
    storage_options: dict | None = None,
) -> Any:
    """Open a pyramids-written cube Zarr store into a lazy ``DatasetCollection``.

    Inverse of :func:`write_collection_to_zarr`. The 4-D ``(time, band, y, x)`` ``data`` array is
    read lazily straight from the store, and the geobox is recovered from the GeoZarr ``spatial_ref``
    mapping. ``collection_cls`` is the ``DatasetCollection`` class to construct.
    """
    import_zarr(
        lazy_hint(
            "DatasetCollection.from_zarr requires the 'zarr' dependency.",
            "zarr",
        )
    )
    import zarr

    resolved = _resolve_store(store, storage_options)
    root = zarr.open_group(resolved, mode="r")
    data_arr = root["data"]
    if not isinstance(data_arr, zarr.Array):
        raise TypeError(
            f"expected the 'data' node in {resolved!r} to be a zarr Array, "
            f"got a Group -- the store is not a pyramids cube"
        )
    data_attrs = dict(data_arr.attrs)
    geobox = read_geobox(root, data_name="data")
    time_length, bands, rows, cols = (int(v) for v in data_arr.shape)
    time_length = int(
        cast("int | float | str", root.attrs.get("time_length", time_length))
    )

    nodata_list = cast("list | None", data_attrs.get("nodata"))
    if nodata_list and any(v is not None for v in nodata_list):
        no_data_value: Any = list(nodata_list)
    else:
        no_data_value = None
    dtype = np.dtype(cast(str, data_attrs.get("dtype", "float32")))
    template_arr = np.zeros((bands, rows, cols), dtype=dtype)
    geo_6 = cast(
        "tuple[float, float, float, float, float, float]",
        tuple(float(v) for v in geobox["geotransform"]),
    )
    template = Dataset.from_array(
        template_arr if bands > 1 else template_arr[0],
        no_data_value=no_data_value,
        geo_ref=GeoReference(geo=geo_6, epsg=geobox_crs(geobox)),
    )
    if geobox["crs_wkt"]:
        template.crs = geobox["crs_wkt"]
    template.bands.apply_names(
        cast("list | None", data_attrs.get("band_names")),
        source="cube Zarr store",
    )
    meta = RasterMeta.from_dataset(template)
    return collection_cls(template, time_length, meta=meta, zarr_store=resolved)