Skip to content

New defaults for concat, merge, combine_* #10062

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 29 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 18 commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
5c56acf
Remove default values in private functions
jsignell Feb 14, 2025
5461a9f
Use sentinel value to change default with warnings
jsignell Feb 24, 2025
e16834f
Remove unnecessary warnings
jsignell Feb 24, 2025
9c50125
Use old kwarg values within map_blocks, concat dataarray
jsignell Feb 25, 2025
b0cf17a
Merge branch 'main' into concat_default_kwargs
jsignell Feb 25, 2025
0026ee8
Switch options back to old defaults
jsignell Feb 26, 2025
4d4deda
Update tests and add new ones to exercise options
jsignell Feb 26, 2025
5a4036b
Merge branch 'main' into concat_default_kwargs
jsignell Mar 4, 2025
912638b
Use `emit_user_level_warning` rather than `warnings.warn`
jsignell Mar 4, 2025
67fd4ff
Change hardcoded defaults
jsignell Mar 4, 2025
4f38292
Fix up test_concat
jsignell Mar 4, 2025
51ccc89
Add comment about why we allow data_vars='minimial' for concat over d…
jsignell Mar 4, 2025
aa3180e
Tidy up tests based on review
jsignell Mar 4, 2025
93d2abc
Merge branch 'main' into concat_default_kwargs
jsignell Mar 7, 2025
e517dcc
Trying to resolve mypy issues
jsignell Mar 10, 2025
0e678e5
Fix mypy in tests
jsignell Mar 10, 2025
37f0147
Fix doctests
jsignell Mar 10, 2025
dac337c
Ignore warnings on error tests
jsignell Mar 10, 2025
a0c16c3
Merge branch 'main' into concat_default_kwargs
jsignell Mar 13, 2025
4eb275c
Use typing.get_args when possible
jsignell Mar 13, 2025
03f1502
Allow `minimal` in concat options at the type level
jsignell Mar 13, 2025
f1649b8
Merge branch 'main' into concat_default_kwargs
dcherian Mar 13, 2025
7dbdd4a
Minimal docs update
jsignell Mar 13, 2025
c6a557b
Tighten up language
jsignell Mar 13, 2025
9667857
Merge branch 'main' into concat_default_kwargs
jsignell Mar 13, 2025
42cf522
Merge branch 'main' into concat_default_kwargs
jsignell Mar 17, 2025
8d0d390
Merge branch 'main' into concat_default_kwargs
jsignell Apr 18, 2025
ba45599
Add to deprecated section of whats new
jsignell Apr 18, 2025
90bd629
Merge branch 'main' into concat_default_kwargs
Illviljan May 9, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 16 additions & 9 deletions xarray/backends/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
)
from xarray.backends.locks import _get_scheduler
from xarray.coders import CFDatetimeCoder, CFTimedeltaCoder
from xarray.core import indexing
from xarray.core import dtypes, indexing
from xarray.core.chunk import _get_chunk, _maybe_chunk
from xarray.core.combine import (
_infer_concat_order_from_positions,
Expand All @@ -50,6 +50,13 @@
from xarray.core.utils import is_remote_uri
from xarray.namedarray.daskmanager import DaskManager
from xarray.namedarray.parallelcompat import guess_chunkmanager
from xarray.util.deprecation_helpers import (
_COMPAT_DEFAULT,
_COORDS_DEFAULT,
_DATA_VARS_DEFAULT,
_JOIN_DEFAULT,
CombineKwargDefault,
)

if TYPE_CHECKING:
try:
Expand Down Expand Up @@ -1404,14 +1411,16 @@ def open_mfdataset(
| Sequence[Index]
| None
) = None,
compat: CompatOptions = "no_conflicts",
compat: CompatOptions | CombineKwargDefault = _COMPAT_DEFAULT,
preprocess: Callable[[Dataset], Dataset] | None = None,
engine: T_Engine | None = None,
data_vars: Literal["all", "minimal", "different"] | list[str] = "all",
coords="different",
data_vars: Literal["all", "minimal", "different"]
| list[str]
| CombineKwargDefault = _DATA_VARS_DEFAULT,
coords=_COORDS_DEFAULT,
combine: Literal["by_coords", "nested"] = "by_coords",
parallel: bool = False,
join: JoinOptions = "outer",
join: JoinOptions | CombineKwargDefault = _JOIN_DEFAULT,
attrs_file: str | os.PathLike | None = None,
combine_attrs: CombineAttrsOptions = "override",
**kwargs,
Expand Down Expand Up @@ -1598,9 +1607,6 @@ def open_mfdataset(

paths1d: list[str | ReadBuffer]
if combine == "nested":
if isinstance(concat_dim, str | DataArray) or concat_dim is None:
concat_dim = [concat_dim] # type: ignore[assignment]

# This creates a flat list which is easier to iterate over, whilst
# encoding the originally-supplied structure as "ids".
# The "ids" are not used at all if combine='by_coords`.
Expand Down Expand Up @@ -1649,13 +1655,14 @@ def open_mfdataset(
# along each dimension, using structure given by "ids"
combined = _nested_combine(
datasets,
concat_dims=concat_dim,
concat_dim=concat_dim,
compat=compat,
data_vars=data_vars,
coords=coords,
ids=ids,
join=join,
combine_attrs=combine_attrs,
fill_value=dtypes.NA,
)
elif combine == "by_coords":
# Redo ordering from coordinates, ignoring how they were ordered
Expand Down
54 changes: 42 additions & 12 deletions xarray/core/alignment.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,9 @@
safe_cast_to_index,
)
from xarray.core.types import T_Alignable
from xarray.core.utils import is_dict_like, is_full_slice
from xarray.core.utils import emit_user_level_warning, is_dict_like, is_full_slice
from xarray.core.variable import Variable, as_compatible_data, calculate_dimensions
from xarray.util.deprecation_helpers import CombineKwargDefault

if TYPE_CHECKING:
from xarray.core.dataarray import DataArray
Expand Down Expand Up @@ -112,7 +113,7 @@ class Aligner(Generic[T_Alignable]):
objects: tuple[T_Alignable, ...]
results: tuple[T_Alignable, ...]
objects_matching_indexes: tuple[dict[MatchingIndexKey, Index], ...]
join: str
join: str | CombineKwargDefault
exclude_dims: frozenset[Hashable]
exclude_vars: frozenset[Hashable]
copy: bool
Expand All @@ -132,7 +133,7 @@ class Aligner(Generic[T_Alignable]):
def __init__(
self,
objects: Iterable[T_Alignable],
join: str = "inner",
join: str | CombineKwargDefault = "inner",
indexes: Mapping[Any, Any] | None = None,
exclude_dims: str | Iterable[Hashable] = frozenset(),
exclude_vars: Iterable[Hashable] = frozenset(),
Expand All @@ -145,7 +146,14 @@ def __init__(
self.objects = tuple(objects)
self.objects_matching_indexes = ()

if join not in ["inner", "outer", "override", "exact", "left", "right"]:
if not isinstance(join, CombineKwargDefault) and join not in [
"inner",
"outer",
"override",
"exact",
"left",
"right",
]:
raise ValueError(f"invalid value for join: {join}")
self.join = join

Expand Down Expand Up @@ -418,12 +426,34 @@ def align_indexes(self) -> None:
else:
need_reindex = False
if need_reindex:
if (
isinstance(self.join, CombineKwargDefault)
and self.join != "exact"
):
emit_user_level_warning(
self.join.warning_message(
"This change will result in the following ValueError:"
"cannot be aligned with join='exact' because "
"index/labels/sizes are not equal along "
"these coordinates (dimensions): "
+ ", ".join(
f"{name!r} {dims!r}" for name, dims in key[0]
),
recommend_set_options=False,
),
FutureWarning,
)
if self.join == "exact":
raise ValueError(
"cannot align objects with join='exact' where "
"index/labels/sizes are not equal along "
"these coordinates (dimensions): "
+ ", ".join(f"{name!r} {dims!r}" for name, dims in key[0])
+ (
self.join.error_message()
if isinstance(self.join, CombineKwargDefault)
else ""
)
)
joiner = self._get_index_joiner(index_cls)
joined_index = joiner(matching_indexes)
Expand Down Expand Up @@ -595,7 +625,7 @@ def align(
obj1: T_Obj1,
/,
*,
join: JoinOptions = "inner",
join: JoinOptions | CombineKwargDefault = "inner",
copy: bool = True,
indexes=None,
exclude: str | Iterable[Hashable] = frozenset(),
Expand All @@ -609,7 +639,7 @@ def align(
obj2: T_Obj2,
/,
*,
join: JoinOptions = "inner",
join: JoinOptions | CombineKwargDefault = "inner",
copy: bool = True,
indexes=None,
exclude: str | Iterable[Hashable] = frozenset(),
Expand All @@ -624,7 +654,7 @@ def align(
obj3: T_Obj3,
/,
*,
join: JoinOptions = "inner",
join: JoinOptions | CombineKwargDefault = "inner",
copy: bool = True,
indexes=None,
exclude: str | Iterable[Hashable] = frozenset(),
Expand All @@ -640,7 +670,7 @@ def align(
obj4: T_Obj4,
/,
*,
join: JoinOptions = "inner",
join: JoinOptions | CombineKwargDefault = "inner",
copy: bool = True,
indexes=None,
exclude: str | Iterable[Hashable] = frozenset(),
Expand All @@ -657,7 +687,7 @@ def align(
obj5: T_Obj5,
/,
*,
join: JoinOptions = "inner",
join: JoinOptions | CombineKwargDefault = "inner",
copy: bool = True,
indexes=None,
exclude: str | Iterable[Hashable] = frozenset(),
Expand All @@ -668,7 +698,7 @@ def align(
@overload
def align(
*objects: T_Alignable,
join: JoinOptions = "inner",
join: JoinOptions | CombineKwargDefault = "inner",
copy: bool = True,
indexes=None,
exclude: str | Iterable[Hashable] = frozenset(),
Expand All @@ -678,7 +708,7 @@ def align(

def align(
*objects: T_Alignable,
join: JoinOptions = "inner",
join: JoinOptions | CombineKwargDefault = "inner",
copy: bool = True,
indexes=None,
exclude: str | Iterable[Hashable] = frozenset(),
Expand Down Expand Up @@ -886,7 +916,7 @@ def align(

def deep_align(
objects: Iterable[Any],
join: JoinOptions = "inner",
join: JoinOptions | CombineKwargDefault = "inner",
copy: bool = True,
indexes=None,
exclude: str | Iterable[Hashable] = frozenset(),
Expand Down
Loading
Loading