-
Notifications
You must be signed in to change notification settings - Fork 49
Open
Description
We see tests going OOM when comparing large datasets.
There exists at least one solution that claims to solve this problem:
https://cubed-dev.github.io/cubed/examples/xarray.html
tests/test_benchmarks.py:146:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
venv/lib/python3.11/site-packages/openeo/testing/results.py:517: in assert_job_results_allclose
issues = _compare_job_results(
venv/lib/python3.11/site-packages/openeo/testing/results.py:449: in _compare_job_results
issues = _compare_xarray_dataarray(
venv/lib/python3.11/site-packages/openeo/testing/results.py:238: in _compare_xarray_dataarray
threshold = abs(expected * rtol) + atol
^^^^^^^^^^^^^^^
venv/lib/python3.11/site-packages/xarray/core/_typed_ops.py:551: in __mul__
return self._binary_op(other, operator.mul)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
venv/lib/python3.11/site-packages/xarray/core/dataarray.py:4817: in _binary_op
f(self.variable, other_variable_or_arraylike)
venv/lib/python3.11/site-packages/xarray/core/_typed_ops.py:934: in __mul__
return self._binary_op(other, operator.mul)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <xarray.Variable (band: 1, y: 68355, x: 33904)> Size: 5GB
array([[[-32768, -32768, ..., -32768, -32768],
[-327...lValue: -32768
scale_factor: 0.01
add_offset: 0.0
long_name: biomass
other = 0.001, f = <built-in function mul>, reflexive = False
def _binary_op(self, other, f, reflexive=False):
if isinstance(other, xr.DataTree | xr.DataArray | xr.Dataset):
return NotImplemented
if reflexive and issubclass(type(self), type(other)):
other_data, self_data, dims = _broadcast_compat_data(other, self)
else:
self_data, other_data, dims = _broadcast_compat_data(self, other)
keep_attrs = _get_keep_attrs(default=False)
attrs = self._attrs if keep_attrs else None
with np.errstate(all="ignore"):
new_data = (
> f(self_data, other_data) if not reflexive else f(other_data, self_data)
^^^^^^^^^^^^^^^^^^^^^^^^
)
E numpy._core._exceptions._ArrayMemoryError: Unable to allocate 17.3 GiB for an array with shape (1, 68355, 33904) and data type float64
venv/lib/python3.11/site-packages/xarray/core/variable.py:2389: MemoryError
Reactions are currently unavailable
Metadata
Metadata
Assignees
Labels
No labels