Support glob patterns in open_datatree(group_filter=...) for selective group loading

doc/whats-new.rst

@@ -75,6 +75,14 @@ New Features
   or a fixed ``(width, height)`` tuple instead of computing figure size from
   ``size`` and ``aspect`` (:issue:`11103`).
   By `Kristian Kollsga <https://github.com/kkollsga>`_.
+- Added ``group_filter`` keyword to :py:func:`open_datatree` and
+  :py:func:`open_groups`, accepting an ``fnmatch``-style glob pattern
+  (e.g. ``"*/leaf_0"``) to selectively open matching groups. Mutually
+  exclusive with ``group``, which keeps its exact-path semantics.
+  Groups whose names literally contain ``*`` or ``?`` are reachable via
+  character-class escapes (``[*]``, ``[?]``)
+  (:issue:`11196`, :pull:`11302`).
+  By `Alfonso Ladino <https://github.com/aladinor>`_.
 
 Breaking Changes
 ~~~~~~~~~~~~~~~~

xarray/backends/api.py

@@ -1021,8 +1021,21 @@ def open_datatree(
         Additional keyword arguments passed on to the engine open function.
         For example:
 
-        - 'group': path to the group in the given file to open as the root group as
-          a str.
+        - 'group': path to the group in the given file to open as the root
+          group as a str. Mutually exclusive with ``'group_filter'``.
+        - 'group_filter': non-empty glob pattern matched against every
+          group path in the file. Only groups whose paths match the
+          pattern are loaded, along with their ancestors so the resulting
+          tree stays connected. Matching follows
+          :py:meth:`pathlib.PurePath.match` semantics: the pattern is
+          anchored on the right, so ``group_filter="*/leaf_0"`` matches
+          any group whose path ends in ``<segment>/leaf_0`` at any depth.
+          Group names that contain literal glob metacharacters can be
+          targeted with character-class escapes: ``[*]`` matches a literal
+          ``*``, ``[?]`` a literal ``?``, and ``[[]`` a literal ``[``. For
+          example, ``group_filter="group_[*]_01"`` matches a group
+          literally named ``group_*_01``. Mutually exclusive with
+          ``'group'``.
         - 'lock': resource lock to use when reading data from disk. Only
           relevant when using dask or another form of parallelism. By default,
           appropriate locks are chosen to safely read and write files with the
@@ -1265,8 +1278,21 @@ def open_groups(
         Additional keyword arguments passed on to the engine open function.
         For example:
 
-        - 'group': path to the group in the given file to open as the root group as
-          a str.
+        - 'group': path to the group in the given file to open as the root
+          group as a str. Mutually exclusive with ``'group_filter'``.
+        - 'group_filter': non-empty glob pattern matched against every
+          group path in the file. Only groups whose paths match the
+          pattern are loaded, along with their ancestors so the resulting
+          tree stays connected. Matching follows
+          :py:meth:`pathlib.PurePath.match` semantics: the pattern is
+          anchored on the right, so ``group_filter="*/leaf_0"`` matches
+          any group whose path ends in ``<segment>/leaf_0`` at any depth.
+          Group names that contain literal glob metacharacters can be
+          targeted with character-class escapes: ``[*]`` matches a literal
+          ``*``, ``[?]`` a literal ``?``, and ``[[]`` a literal ``[``. For
+          example, ``group_filter="group_[*]_01"`` matches a group
+          literally named ``group_*_01``. Mutually exclusive with
+          ``'group'``.
         - 'lock': resource lock to use when reading data from disk. Only
           relevant when using dask or another form of parallelism. By default,
           appropriate locks are chosen to safely read and write files with the

xarray/backends/common.py

@@ -249,6 +249,41 @@ def _iter_nc_groups(root, parent="/"):
         yield from _iter_nc_groups(group, parent=gpath)
 
 
+def _check_group_filter_mutex(group: str | None, group_filter: str | None) -> None:
+    """Validate ``group`` / ``group_filter`` are not both set, and ``group_filter``
+    is non-empty when provided.
+    """
+    if group is not None and group_filter is not None:
+        raise ValueError(
+            "group and group_filter are mutually exclusive: "
+            "group selects an exact group path while group_filter "
+            "is a glob pattern over all group paths."
+        )
+    if group_filter == "":
+        raise ValueError("group_filter must be a non-empty glob pattern")
+
+
+def _filter_group_paths(group_paths: Sequence[str], pattern: str) -> list[str]:
+    """Return the subset of ``group_paths`` whose paths match ``pattern``,
+    plus every ancestor of a match (so the resulting tree stays
+    connected). The root path ``"/"`` is always included.
+
+    ``pattern`` is matched with :py:meth:`pathlib.PurePath.match` semantics,
+    so it is anchored on the right: ``"*/leaf_0"`` matches a group whose
+    path ends in any single segment followed by ``leaf_0`` at any depth.
+    """
+    from xarray.core.treenode import NodePath
+
+    matched: set[str] = {"/"}
+    for path in group_paths:
+        np_ = NodePath(path)
+        if np_.match(pattern):
+            matched.add(path)
+            matched.update(str(p) for p in np_.parents)
+
+    return [p for p in group_paths if p in matched]
+
+
 def find_root_and_group(ds):
     """Find the root and group name of a netCDF4/h5netcdf dataset."""
     hierarchy = ()

xarray/backends/h5netcdf_.py

@@ -602,6 +602,7 @@ def open_datatree(
         decode_timedelta=None,
         format="NETCDF4",
         group: str | None = None,
+        group_filter: str | None = None,
         lock=None,
         invalid_netcdf=None,
         phony_dims=None,
@@ -621,6 +622,7 @@ def open_datatree(
             decode_timedelta=decode_timedelta,
             format=format,
             group=group,
+            group_filter=group_filter,
             lock=lock,
             invalid_netcdf=invalid_netcdf,
             phony_dims=phony_dims,
@@ -645,6 +647,7 @@ def open_groups_as_dict(
         decode_timedelta=None,
         format="NETCDF4",
         group: str | None = None,
+        group_filter: str | None = None,
         lock=None,
         invalid_netcdf=None,
         phony_dims=None,
@@ -655,15 +658,22 @@ def open_groups_as_dict(
         open_kwargs: dict[str, Any] | None = None,
         **kwargs,
     ) -> dict[str, Dataset]:
-        from xarray.backends.common import _iter_nc_groups
+        from xarray.backends.common import (
+            _check_group_filter_mutex,
+            _filter_group_paths,
+            _iter_nc_groups,
+        )
         from xarray.core.treenode import NodePath
         from xarray.core.utils import close_on_error
 
+        _check_group_filter_mutex(group, group_filter)
+
         # Keep this message for some versions
         # remove and set phony_dims="access" above
         emit_phony_dims_warning, phony_dims = _check_phony_dims(phony_dims)
 
         filename_or_obj = _normalize_filename_or_obj(filename_or_obj)
+
         store = H5NetCDFStore.open(
             filename_or_obj,
             format=format,
@@ -678,15 +688,18 @@ def open_groups_as_dict(
             open_kwargs=open_kwargs,
         )
 
-        # Check for a group and make it a parent if it exists
-        if group:
+        if group is not None:
             parent = NodePath("/") / NodePath(group)
         else:
             parent = NodePath("/")
 
         manager = store._manager
+        group_paths = list(_iter_nc_groups(store.ds, parent=parent))
+        if group_filter is not None:
+            group_paths = _filter_group_paths(group_paths, group_filter)
+
         groups_dict = {}
-        for path_group in _iter_nc_groups(store.ds, parent=parent):
+        for path_group in group_paths:
             group_store = H5NetCDFStore(manager, group=path_group, **kwargs)
             store_entrypoint = StoreBackendEntrypoint()
             with close_on_error(group_store):
@@ -701,7 +714,7 @@ def open_groups_as_dict(
                     decode_timedelta=decode_timedelta,
                 )
 
-            if group:
+            if group is not None:
                 group_name = str(NodePath(path_group).relative_to(parent))
             else:
                 group_name = str(NodePath(path_group))

xarray/backends/netCDF4_.py

@@ -807,6 +807,7 @@ def open_datatree(
         use_cftime=None,
         decode_timedelta=None,
         group: str | None = None,
+        group_filter: str | None = None,
         format="NETCDF4",
         clobber=True,
         diskless=False,
@@ -826,6 +827,7 @@ def open_datatree(
             use_cftime=use_cftime,
             decode_timedelta=decode_timedelta,
             group=group,
+            group_filter=group_filter,
             format=format,
             clobber=clobber,
             diskless=diskless,
@@ -850,6 +852,7 @@ def open_groups_as_dict(
         use_cftime=None,
         decode_timedelta=None,
         group: str | None = None,
+        group_filter: str | None = None,
         format="NETCDF4",
         clobber=True,
         diskless=False,
@@ -859,10 +862,17 @@ def open_groups_as_dict(
         autoclose=False,
         **kwargs,
     ) -> dict[str, Dataset]:
-        from xarray.backends.common import _iter_nc_groups
+        from xarray.backends.common import (
+            _check_group_filter_mutex,
+            _filter_group_paths,
+            _iter_nc_groups,
+        )
         from xarray.core.treenode import NodePath
 
+        _check_group_filter_mutex(group, group_filter)
+
         filename_or_obj = _normalize_path(filename_or_obj)
+
         store = NetCDF4DataStore.open(
             filename_or_obj,
             group=group,
@@ -875,15 +885,18 @@ def open_groups_as_dict(
             autoclose=autoclose,
         )
 
-        # Check for a group and make it a parent if it exists
-        if group:
+        if group is not None:
             parent = NodePath("/") / NodePath(group)
         else:
             parent = NodePath("/")
 
         manager = store._manager
+        group_paths = list(_iter_nc_groups(store.ds, parent=parent))
+        if group_filter is not None:
+            group_paths = _filter_group_paths(group_paths, group_filter)
+
         groups_dict = {}
-        for path_group in _iter_nc_groups(store.ds, parent=parent):
+        for path_group in group_paths:
             group_store = NetCDF4DataStore(manager, group=path_group, **kwargs)
             store_entrypoint = StoreBackendEntrypoint()
             with close_on_error(group_store):
@@ -897,7 +910,7 @@ def open_groups_as_dict(
                     use_cftime=use_cftime,
                     decode_timedelta=decode_timedelta,
                 )
-            if group:
+            if group is not None:
                 group_name = str(NodePath(path_group).relative_to(parent))
             else:
                 group_name = str(NodePath(path_group))

xarray/backends/zarr.py

@@ -680,6 +680,7 @@ def open_store(
         mode: ZarrWriteModes = "r",
         synchronizer=None,
         group=None,
+        group_filter: str | None = None,
         consolidated=False,
         consolidate_on_close=False,
         chunk_store=None,
@@ -715,8 +716,15 @@ def open_store(
 
         from zarr import Group
 
+        from xarray.backends.common import _filter_group_paths
+
         group_members: dict[str, Group] = {}
         group_paths = list(_iter_zarr_groups(zarr_group, parent=group))
+        # Filter before materializing child Group objects: each
+        # ``zarr_group[rel_path]`` lookup triggers metadata I/O, so
+        # pruning paths up-front skips the cost for groups we'd discard.
+        if group_filter is not None:
+            group_paths = _filter_group_paths(group_paths, group_filter)
         for path in group_paths:
             if path == group:
                 group_members[path] = zarr_group
@@ -1779,6 +1787,7 @@ def open_datatree(
         use_cftime=None,
         decode_timedelta=None,
         group: str | None = None,
+        group_filter: str | None = None,
         mode="r",
         synchronizer=None,
         consolidated=None,
@@ -1798,6 +1807,7 @@ def open_datatree(
             use_cftime=use_cftime,
             decode_timedelta=decode_timedelta,
             group=group,
+            group_filter=group_filter,
             mode=mode,
             synchronizer=synchronizer,
             consolidated=consolidated,
@@ -1821,6 +1831,7 @@ def open_groups_as_dict(
         use_cftime=None,
         decode_timedelta=None,
         group: str | None = None,
+        group_filter: str | None = None,
         mode="r",
         synchronizer=None,
         consolidated=None,
@@ -1829,17 +1840,21 @@ def open_groups_as_dict(
         zarr_version=None,
         zarr_format=None,
     ) -> dict[str, Dataset]:
+        from xarray.backends.common import _check_group_filter_mutex
+
+        _check_group_filter_mutex(group, group_filter)
+
         filename_or_obj = _normalize_path(filename_or_obj)
 
-        # Check for a group and make it a parent if it exists
-        if group:
+        if group is not None:
             parent = str(NodePath("/") / NodePath(group))
         else:
             parent = str(NodePath("/"))
 
         stores = ZarrStore.open_store(
             filename_or_obj,
             group=parent,
+            group_filter=group_filter,
             mode=mode,
             synchronizer=synchronizer,
             consolidated=consolidated,
@@ -1850,8 +1865,11 @@ def open_groups_as_dict(
             zarr_format=zarr_format,
         )
 
+        group_paths = list(stores.keys())
+
         groups_dict = {}
-        for path_group, store in stores.items():
+        for path_group in group_paths:
+            store = stores[path_group]
             store_entrypoint = StoreBackendEntrypoint()
 
             with close_on_error(store):
@@ -1865,7 +1883,7 @@ def open_groups_as_dict(
                     use_cftime=use_cftime,
                     decode_timedelta=decode_timedelta,
                 )
-            if group:
+            if group is not None:
                 group_name = str(NodePath(path_group).relative_to(parent))
             else:
                 group_name = str(NodePath(path_group))