Support passing UDF as context to run_udf

[EmileSonneveld]

It is possible to run this when manually constructing the process graph, but the python client raises an error.

cwl = Path("apex-force-openeo/material/force-l2.cwl").read_text()
datacube = connection.datacube_from_process(
    "run_udf",
    data=None,
    udf=cwl,
    runtime="EOAP-CWL",
    context={"input": udf_process_graph}, # udf_process_graph is a UDF producing a StructuredData (list of strings)
)
datacube.to_json() # fails

stacktrace

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[53], line 1
----> 1 datacube.to_json()

File ~/micromamba/envs/testing/lib/python3.11/site-packages/openeo/internal/graph_building.py:53, in FlatGraphableMixin.to_json(self, indent, separators)
   40 """
   41 Get interoperable JSON representation of the process graph.
   42 
 (...)
   50 :return: JSON string
   51 """
   52 pg = {"process_graph": self.flat_graph()}
---> 53 return json.dumps(pg, indent=indent, separators=separators)

File ~/micromamba/envs/testing/lib/python3.11/json/__init__.py:238, in dumps(obj, skipkeys, ensure_ascii, check_circular, allow_nan, cls, indent, separators, default, sort_keys, **kw)
  232 if cls is None:
  233     cls = JSONEncoder
  234 return cls(
  235     skipkeys=skipkeys, ensure_ascii=ensure_ascii,
  236     check_circular=check_circular, allow_nan=allow_nan, indent=indent,
  237     separators=separators, default=default, sort_keys=sort_keys,
--> 238     **kw).encode(obj)

File ~/micromamba/envs/testing/lib/python3.11/json/encoder.py:202, in JSONEncoder.encode(self, o)
  200 chunks = self.iterencode(o, _one_shot=True)
  201 if not isinstance(chunks, (list, tuple)):
--> 202     chunks = list(chunks)
  203 return ''.join(chunks)

File ~/micromamba/envs/testing/lib/python3.11/json/encoder.py:432, in _make_iterencode.<locals>._iterencode(o, _current_indent_level)
  430     yield from _iterencode_list(o, _current_indent_level)
  431 elif isinstance(o, dict):
--> 432     yield from _iterencode_dict(o, _current_indent_level)
  433 else:
  434     if markers is not None:

File ~/micromamba/envs/testing/lib/python3.11/json/encoder.py:406, in _make_iterencode.<locals>._iterencode_dict(dct, _current_indent_level)
  404         else:
  405             chunks = _iterencode(value, _current_indent_level)
--> 406         yield from chunks
  407 if newline_indent is not None:
  408     _current_indent_level -= 1

File ~/micromamba/envs/testing/lib/python3.11/json/encoder.py:406, in _make_iterencode.<locals>._iterencode_dict(dct, _current_indent_level)
  404         else:
  405             chunks = _iterencode(value, _current_indent_level)
--> 406         yield from chunks
  407 if newline_indent is not None:
  408     _current_indent_level -= 1

  [... skipping similar frames: _make_iterencode.<locals>._iterencode_dict at line 406 (2 times)]

File ~/micromamba/envs/testing/lib/python3.11/json/encoder.py:406, in _make_iterencode.<locals>._iterencode_dict(dct, _current_indent_level)
  404         else:
  405             chunks = _iterencode(value, _current_indent_level)
--> 406         yield from chunks
  407 if newline_indent is not None:
  408     _current_indent_level -= 1

File ~/micromamba/envs/testing/lib/python3.11/json/encoder.py:439, in _make_iterencode.<locals>._iterencode(o, _current_indent_level)
  437         raise ValueError("Circular reference detected")
  438     markers[markerid] = o
--> 439 o = _default(o)
  440 yield from _iterencode(o, _current_indent_level)
  441 if markers is not None:

File ~/micromamba/envs/testing/lib/python3.11/json/encoder.py:180, in JSONEncoder.default(self, o)
  161 def default(self, o):
  162     """Implement this method in a subclass such that it returns
  163     a serializable object for ``o``, or calls the base implementation
  164     (to raise a ``TypeError``).
 (...)
  178 
  179     """
--> 180     raise TypeError(f'Object of type {o.__class__.__name__} '
  181                     f'is not JSON serializable')

TypeError: Object of type DataCube is not JSON serializable

[Hneuschmidt]

Related: https://forum.dataspace.copernicus.eu/t/alternative-to-resample-cube-temporal-to-match-the-time-series-on-independent-cubes/4191/5

[EmileSonneveld]

@Hneuschmidt The solution might be to simply pass the first run_udf process as context without wrapping it. For example:

connection = openeo.connect("https://openeo.dataspace.copernicus.eu").authenticate_oidc()

runudf1 = connection.datacube_from_process(
    "run_udf",
    context={},
    data=[],
    runtime="Python",
    udf="""
from openeo.udf.udf_data import UdfData
from openeo.udf.structured_data import StructuredData

def apply_udf_data(data: UdfData):
    sd = StructuredData(
        data={
            "input": [
                "s3://EODATA/Sentinel-2/MSI/L1C/2024/12/28/S2B_MSIL1C_20241228T101339_N0511_R022_T32TPQ_20241228T120532.SAFE",
                "s3://EODATA/Sentinel-2/MSI/L1C/2024/12/28/S2B_MSIL1C_20241228T101339_N0511_R022_T32TQQ_20241228T120532.SAFE",
            ],
            "resolution:": 20,
        }
    )
    data.set_structured_data_list(structured_data_list=[sd])
""",
)
runudf2 = runudf1.process(
    "run_udf",
    context=runudf1,
    data=None,
    runtime="EOAP-CWL",
    udf=cwl,
)

Does that work for you?

[Hneuschmidt]

It does work, however, we are looking for a solution to pass both the result of a process and additional configuration in the context dict. The example is a little too minimal. The desired usage would be (for example):

datacube = connection.datacube_from_process(
    "run_udf",
    data=None,
    udf=cwl,
    runtime="EOAP-CWL",
    context={
        "input": udf_process_graph,
        "projection": "EQUI7", # <--- additional parameter
    }
)
datacube.to_json() # fails