diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e5c948d..250ef4ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ and start a new "In Progress" section above it. - Include `job_options` as top-level properties in `GET /jobs/{job_id}` response ([#470](https://github.com/Open-EO/openeo-python-driver/issues/470)) - Bump STAC version from `0.9.0` to `1.0.0` in capabilities endpoint, collection metadata, job results and ML model metadata ([#363](https://github.com/Open-EO/openeo-python-driver/issues/363)) - `load_collection`: add check on malformed `spatial_extent`. ([#284](https://github.com/Open-EO/openeo-python-driver/issues/284)) +- Support logging added value for synchronous requests ([Open-EO/openeo-geopyspark-driver#1436](https://github.com/Open-EO/openeo-geopyspark-driver/issues/1436)) ## 0.139.0 diff --git a/openeo_driver/_version.py b/openeo_driver/_version.py index 2d37b924..8cec8eba 100644 --- a/openeo_driver/_version.py +++ b/openeo_driver/_version.py @@ -1 +1 @@ -__version__ = "0.140.0a1" +__version__ = "0.140.0a2" diff --git a/openeo_driver/backend.py b/openeo_driver/backend.py index 763ece1a..7df0ca94 100644 --- a/openeo_driver/backend.py +++ b/openeo_driver/backend.py @@ -19,6 +19,7 @@ from typing import List, Union, NamedTuple, Dict, Optional, Callable, Iterable, Container, Any, Tuple import flask +from openeo_driver.processgraph.definitions import ProcessGraphFlatDict import openeo_driver.util.view_helpers from openeo.utils.version import ComparableVersion @@ -1072,6 +1073,8 @@ def request_costs( job_options: Union[dict, None] = None, request_id: str, success: bool, + process_graph: Union[ProcessGraphFlatDict, None] = None, + tracer: Union[DryRunDataTracer, None] = None, ) -> Optional[float]: """ Report resource usage of (current) synchronous processing request and get associated cost. diff --git a/openeo_driver/processgraph/evaluator.py b/openeo_driver/processgraph/evaluator.py index 120e10d0..88ed3b21 100644 --- a/openeo_driver/processgraph/evaluator.py +++ b/openeo_driver/processgraph/evaluator.py @@ -44,7 +44,7 @@ from openeo_driver.util import UNSET from openeo_driver.util.http import is_http_url from openeo_driver.utils import EvalEnv, smart_bool -from openeo_driver.views import OPENEO_API_VERSION_DEFAULT +from openeo_driver.views import ENV_SYNC_DRY_RUN_TRACER, OPENEO_API_VERSION_DEFAULT _log = logging.getLogger(__name__) @@ -116,7 +116,11 @@ def evaluate( env = env.push({ENV_FINAL_RESULT: [None], ENV_MAX_BUFFER: {}}) if do_dry_run: - dry_run_tracer = do_dry_run if isinstance(do_dry_run, DryRunDataTracer) else DryRunDataTracer() + dry_run_tracer = ( + env.get(ENV_SYNC_DRY_RUN_TRACER) or do_dry_run + if isinstance(do_dry_run, DryRunDataTracer) + else DryRunDataTracer() + ) _log.info("Doing dry run") dry_run_env = env.push( { diff --git a/openeo_driver/views.py b/openeo_driver/views.py index ccd9dac4..77dd6eac 100644 --- a/openeo_driver/views.py +++ b/openeo_driver/views.py @@ -55,6 +55,7 @@ LINK_REL, ) from openeo_driver.datacube import DriverMlModel +from openeo_driver.dry_run import DryRunDataTracer from openeo_driver.errors import ( FeatureUnsupportedException, FilePathInvalidException, @@ -105,6 +106,8 @@ STREAM_CHUNK_SIZE_DEFAULT = 10 * 1024 +ENV_SYNC_DRY_RUN_TRACER = "sync_dry_run_tracer" + class OpenEoApiApp(Flask): @@ -725,8 +728,21 @@ def result(user: User): } ) + tracer = DryRunDataTracer() + env = env.push({ENV_SYNC_DRY_RUN_TRACER: tracer}) + + request_costs = functools.partial( + backend_implementation.request_costs, + user=user, + job_options=job_options, + request_id=request_id, + process_graph=process_graph, + tracer=tracer, + ) + try: - result = backend_implementation.processing.evaluate(process_graph=process_graph, env=env) + result = backend_implementation.processing.evaluate(process_graph=copy.deepcopy(process_graph), env=env) + _log.info(f"`POST /result`: {type(result)}") if result is None: @@ -742,18 +758,14 @@ def result(user: User): result = to_save_result(data=result) response = result.create_flask_response() - costs = backend_implementation.request_costs( - success=True, user=user, request_id=request_id, job_options=job_options - ) + costs = request_costs(success=True) if costs: # TODO not all costs are accounted for so don't expose in "OpenEO-Costs" yet response.headers["OpenEO-Costs-experimental"] = costs except Exception: # TODO: also send "OpenEO-Costs" header on failure - backend_implementation.request_costs( - success=False, user=user, request_id=request_id, job_options=job_options - ) + request_costs(success=False) raise # Add request id as "OpenEO-Identifier" like we do for batch jobs. diff --git a/tests/test_views_execute.py b/tests/test_views_execute.py index faa522a0..8dd1e1a1 100644 --- a/tests/test_views_execute.py +++ b/tests/test_views_execute.py @@ -22,7 +22,7 @@ from openeo_driver.datacube import DriverDataCube, DriverVectorCube from openeo_driver.datastructs import ResolutionMergeArgs, SarBackscatterArgs -from openeo_driver.dry_run import ProcessType +from openeo_driver.dry_run import ProcessType, DryRunDataTracer from openeo_driver.dummy import dummy_backend from openeo_driver.dummy.dummy_backend import DummyVisitor from openeo_driver.errors import ( @@ -4308,13 +4308,13 @@ def test_vector_buffer_returns_error_on_empty_result_geometry(api): (None, None, None), # request_costs override ( - lambda user, request_id, success, job_options: 1234 + isinstance(user, User), + lambda user, job_options, request_id, success, process_graph, tracer: 1234 + isinstance(user, User), None, "1235", ), # Extra job options handling ( - lambda user, request_id, success, job_options: 1234 * job_options.get("extra", 0), + lambda user, job_options, request_id, success, process_graph, tracer: 1234 * job_options.get("extra", 0), {"extra": 2}, "2468", ), @@ -4363,6 +4363,8 @@ def test_synchronous_processing_request_costs( job_options=job_options, success=success, request_id="r-abc123", + process_graph=pg, + tracer=dirty_equals.IsInstance(DryRunDataTracer), )