diff --git a/agentplatform/_genai/evals.py b/agentplatform/_genai/evals.py index 4b600f7054..754c859bc1 100644 --- a/agentplatform/_genai/evals.py +++ b/agentplatform/_genai/evals.py @@ -263,6 +263,20 @@ def _DeleteEvaluationMetricParameters_to_vertex( return to_object +def _DeleteEvaluationSetParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["name"]) is not None: + setv(to_object, ["_url", "name"], getv(from_object, ["name"])) + + if getv(from_object, ["config"]) is not None: + setv(to_object, ["config"], getv(from_object, ["config"])) + + return to_object + + def _EvaluateInstancesRequestParameters_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -951,6 +965,44 @@ def _ListEvaluationMetricsResponse_from_vertex( return to_object +def _ListEvaluationSetsConfig_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + + if getv(from_object, ["page_size"]) is not None: + setv(parent_object, ["_query", "pageSize"], getv(from_object, ["page_size"])) + + if getv(from_object, ["page_token"]) is not None: + setv(parent_object, ["_query", "pageToken"], getv(from_object, ["page_token"])) + + if getv(from_object, ["filter"]) is not None: + setv(parent_object, ["_query", "filter"], getv(from_object, ["filter"])) + + if getv(from_object, ["order_by"]) is not None: + setv(parent_object, ["_query", "orderBy"], getv(from_object, ["order_by"])) + + return to_object + + +def _ListEvaluationSetsParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["config"]) is not None: + setv( + to_object, + ["config"], + _ListEvaluationSetsConfig_to_vertex( + getv(from_object, ["config"]), to_object + ), + ) + + return to_object + + def _RubricBasedMetricInput_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -1729,6 +1781,78 @@ def _delete_evaluation_metric( self._api_client._verify_response(return_value) return return_value + def _delete_evaluation_set( + self, + *, + name: str, + config: Optional[types.DeleteEvaluationSetConfigOrDict] = None, + ) -> types.DeleteEvaluationSetOperation: + """ + Deletes an EvaluationSet. + """ + + parameter_model = types._DeleteEvaluationSetParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _DeleteEvaluationSetParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationSets/{name}".format_map(request_url_dict) + else: + path = "evaluationSets/{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("delete", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.DeleteEvaluationSetOperation._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + def _evaluate_instances( self, *, @@ -2607,6 +2731,74 @@ def _list_evaluation_metrics( self._api_client._verify_response(return_value) return return_value + def _list_evaluation_sets( + self, *, config: Optional[types.ListEvaluationSetsConfigOrDict] = None + ) -> types.ListEvaluationSetsResponse: + """ + Lists EvaluationSets. + """ + + parameter_model = types._ListEvaluationSetsParameters( + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _ListEvaluationSetsParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationSets".format_map(request_url_dict) + else: + path = "evaluationSets" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("get", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.ListEvaluationSetsResponse._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + def update_evaluation_experiment( self, *, @@ -3551,6 +3743,63 @@ def create_evaluation_set( config=config, ) + def list_evaluation_sets( + self, + *, + filter: Optional[str] = None, + order_by: Optional[str] = None, + config: Optional[types.ListEvaluationSetsConfigOrDict] = None, + ) -> types.ListEvaluationSetsResponse: + """Lists EvaluationSets. + + Args: + filter: An expression for filtering the results of the request. For + field names both snake_case and camelCase are supported. For more + information about filter syntax, see + `AIP-160 `_. + Example: ``'display_name="my_dataset"'``. + order_by: A comma-separated list of fields to order by, sorted in + ascending order by default. Use ``desc`` after a field name for + descending. Example: ``"create_time desc"``. + config: Optional configuration for the list operation, including + pagination (``page_size``, ``page_token``), ``filter``, and + ``order_by``. Top-level ``filter`` and ``order_by`` arguments + take precedence over values set in ``config``. + + Returns: + The list evaluation sets response. + """ + if config is None: + config = types.ListEvaluationSetsConfig() + if isinstance(config, dict): + config = types.ListEvaluationSetsConfig.model_validate(config) + if filter is not None: + config.filter = filter + if order_by is not None: + config.order_by = order_by + return self._list_evaluation_sets( + config=config, + ) + + def delete_evaluation_set( + self, + *, + name: str, + config: Optional[types.DeleteEvaluationSetConfigOrDict] = None, + ) -> None: + """Deletes an EvaluationSet. + + Args: + name: The resource name of the EvaluationSet to delete. Format: + `projects/{project}/locations/{location}/evaluationSets/{evaluation_set}` + config: The optional configuration for the delete operation. + """ + if not name: + raise ValueError("name cannot be empty.") + if name.startswith("projects/"): + name = name.split("/")[-1] + self._delete_evaluation_set(name=name, config=config) + def generate_conversation_scenarios( self, *, @@ -4381,6 +4630,80 @@ async def _delete_evaluation_metric( self._api_client._verify_response(return_value) return return_value + async def _delete_evaluation_set( + self, + *, + name: str, + config: Optional[types.DeleteEvaluationSetConfigOrDict] = None, + ) -> types.DeleteEvaluationSetOperation: + """ + Deletes an EvaluationSet. + """ + + parameter_model = types._DeleteEvaluationSetParameters( + name=name, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _DeleteEvaluationSetParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationSets/{name}".format_map(request_url_dict) + else: + path = "evaluationSets/{name}" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "delete", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.DeleteEvaluationSetOperation._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def _evaluate_instances( self, *, @@ -5281,6 +5604,76 @@ async def _list_evaluation_metrics( self._api_client._verify_response(return_value) return return_value + async def _list_evaluation_sets( + self, *, config: Optional[types.ListEvaluationSetsConfigOrDict] = None + ) -> types.ListEvaluationSetsResponse: + """ + Lists EvaluationSets. + """ + + parameter_model = types._ListEvaluationSetsParameters( + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _ListEvaluationSetsParameters_to_vertex(parameter_model) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationSets".format_map(request_url_dict) + else: + path = "evaluationSets" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "get", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.ListEvaluationSetsResponse._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def update_evaluation_experiment( self, *, @@ -5837,6 +6230,63 @@ async def create_evaluation_set( ) return result + async def list_evaluation_sets( + self, + *, + filter: Optional[str] = None, + order_by: Optional[str] = None, + config: Optional[types.ListEvaluationSetsConfigOrDict] = None, + ) -> types.ListEvaluationSetsResponse: + """Lists EvaluationSets. + + Args: + filter: An expression for filtering the results of the request. For + field names both snake_case and camelCase are supported. For more + information about filter syntax, see + `AIP-160 `_. + Example: ``'display_name="my_dataset"'``. + order_by: A comma-separated list of fields to order by, sorted in + ascending order by default. Use ``desc`` after a field name for + descending. Example: ``"create_time desc"``. + config: Optional configuration for the list operation, including + pagination (``page_size``, ``page_token``), ``filter``, and + ``order_by``. Top-level ``filter`` and ``order_by`` arguments + take precedence over values set in ``config``. + + Returns: + The list evaluation sets response. + """ + if config is None: + config = types.ListEvaluationSetsConfig() + if isinstance(config, dict): + config = types.ListEvaluationSetsConfig.model_validate(config) + if filter is not None: + config.filter = filter + if order_by is not None: + config.order_by = order_by + return await self._list_evaluation_sets( + config=config, + ) + + async def delete_evaluation_set( + self, + *, + name: str, + config: Optional[types.DeleteEvaluationSetConfigOrDict] = None, + ) -> None: + """Deletes an EvaluationSet. + + Args: + name: The resource name of the EvaluationSet to delete. Format: + `projects/{project}/locations/{location}/evaluationSets/{evaluation_set}` + config: The optional configuration for the delete operation. + """ + if not name: + raise ValueError("name cannot be empty.") + if name.startswith("projects/"): + name = name.split("/")[-1] + await self._delete_evaluation_set(name=name, config=config) + async def generate_conversation_scenarios( self, *, diff --git a/agentplatform/_genai/types/__init__.py b/agentplatform/_genai/types/__init__.py index 9e9278030a..d27abe4a5c 100644 --- a/agentplatform/_genai/types/__init__.py +++ b/agentplatform/_genai/types/__init__.py @@ -58,6 +58,7 @@ from .common import _DeleteDatasetRequestParameters from .common import _DeleteEvaluationExperimentParameters from .common import _DeleteEvaluationMetricParameters +from .common import _DeleteEvaluationSetParameters from .common import _DeleteMemoryBankRequestParameters from .common import _DeleteMemoryRequestParameters from .common import _DeleteMultimodalDatasetRequestParameters @@ -134,6 +135,7 @@ from .common import _ListDatasetVersionsRequestParameters from .common import _ListEvaluationExperimentsParameters from .common import _ListEvaluationMetricsParameters +from .common import _ListEvaluationSetsParameters from .common import _ListMemoriesRequestParameters from .common import _ListMemoryRevisionsRequestParameters from .common import _ListMultimodalDatasetsRequestParameters @@ -441,6 +443,12 @@ from .common import DeleteEvaluationMetricOperation from .common import DeleteEvaluationMetricOperationDict from .common import DeleteEvaluationMetricOperationOrDict +from .common import DeleteEvaluationSetConfig +from .common import DeleteEvaluationSetConfigDict +from .common import DeleteEvaluationSetConfigOrDict +from .common import DeleteEvaluationSetOperation +from .common import DeleteEvaluationSetOperationDict +from .common import DeleteEvaluationSetOperationOrDict from .common import DeleteMemoryBankConfig from .common import DeleteMemoryBankConfigDict from .common import DeleteMemoryBankConfigOrDict @@ -931,6 +939,12 @@ from .common import ListEvaluationMetricsResponse from .common import ListEvaluationMetricsResponseDict from .common import ListEvaluationMetricsResponseOrDict +from .common import ListEvaluationSetsConfig +from .common import ListEvaluationSetsConfigDict +from .common import ListEvaluationSetsConfigOrDict +from .common import ListEvaluationSetsResponse +from .common import ListEvaluationSetsResponseDict +from .common import ListEvaluationSetsResponseOrDict from .common import ListMemoriesConfig from .common import ListMemoriesConfigDict from .common import ListMemoriesConfigOrDict @@ -2234,6 +2248,12 @@ "DeleteEvaluationMetricOperation", "DeleteEvaluationMetricOperationDict", "DeleteEvaluationMetricOperationOrDict", + "DeleteEvaluationSetConfig", + "DeleteEvaluationSetConfigDict", + "DeleteEvaluationSetConfigOrDict", + "DeleteEvaluationSetOperation", + "DeleteEvaluationSetOperationDict", + "DeleteEvaluationSetOperationOrDict", "BleuInstance", "BleuInstanceDict", "BleuInstanceOrDict", @@ -2426,6 +2446,12 @@ "ListEvaluationMetricsResponse", "ListEvaluationMetricsResponseDict", "ListEvaluationMetricsResponseOrDict", + "ListEvaluationSetsConfig", + "ListEvaluationSetsConfigDict", + "ListEvaluationSetsConfigOrDict", + "ListEvaluationSetsResponse", + "ListEvaluationSetsResponseDict", + "ListEvaluationSetsResponseOrDict", "UpdateEvaluationExperimentConfig", "UpdateEvaluationExperimentConfigDict", "UpdateEvaluationExperimentConfigOrDict", @@ -3813,6 +3839,7 @@ "_CreateEvaluationSetParameters", "_DeleteEvaluationExperimentParameters", "_DeleteEvaluationMetricParameters", + "_DeleteEvaluationSetParameters", "_EvaluateInstancesRequestParameters", "_GenerateUserScenariosParameters", "_GenerateLossClustersParameters", @@ -3824,6 +3851,7 @@ "_GetEvaluationItemParameters", "_ListEvaluationExperimentsParameters", "_ListEvaluationMetricsParameters", + "_ListEvaluationSetsParameters", "_UpdateEvaluationExperimentParameters", "_OptimizeRequestParameters", "_CustomJobParameters", diff --git a/agentplatform/_genai/types/common.py b/agentplatform/_genai/types/common.py index b0343429c1..ca83d113d5 100644 --- a/agentplatform/_genai/types/common.py +++ b/agentplatform/_genai/types/common.py @@ -4320,6 +4320,92 @@ class DeleteEvaluationMetricOperationDict(TypedDict, total=False): ] +class DeleteEvaluationSetConfig(_common.BaseModel): + """Config for deleting an evaluation set.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + + +class DeleteEvaluationSetConfigDict(TypedDict, total=False): + """Config for deleting an evaluation set.""" + + http_options: Optional[genai_types.HttpOptions] + """Used to override HTTP request options.""" + + +DeleteEvaluationSetConfigOrDict = Union[ + DeleteEvaluationSetConfig, DeleteEvaluationSetConfigDict +] + + +class _DeleteEvaluationSetParameters(_common.BaseModel): + """Parameters for deleting an evaluation set.""" + + name: Optional[str] = Field(default=None, description="""""") + config: Optional[DeleteEvaluationSetConfig] = Field( + default=None, description="""""" + ) + + +class _DeleteEvaluationSetParametersDict(TypedDict, total=False): + """Parameters for deleting an evaluation set.""" + + name: Optional[str] + """""" + + config: Optional[DeleteEvaluationSetConfigDict] + """""" + + +_DeleteEvaluationSetParametersOrDict = Union[ + _DeleteEvaluationSetParameters, _DeleteEvaluationSetParametersDict +] + + +class DeleteEvaluationSetOperation(_common.BaseModel): + """Operation for deleting an evaluation set.""" + + name: Optional[str] = Field( + default=None, + description="""The server-assigned name, which is only unique within the same service that originally returns it. If you use the default HTTP mapping, the `name` should be a resource name ending with `operations/{unique_id}`.""", + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, + description="""Service-specific metadata associated with the operation. It typically contains progress information and common metadata such as create time. Some services might not provide such metadata. Any method that returns a long-running operation should document the metadata type, if any.""", + ) + done: Optional[bool] = Field( + default=None, + description="""If the value is `false`, it means the operation is still in progress. If `true`, the operation is completed, and either `error` or `response` is available.""", + ) + error: Optional[dict[str, Any]] = Field( + default=None, + description="""The error result of the operation in case of failure or cancellation.""", + ) + + +class DeleteEvaluationSetOperationDict(TypedDict, total=False): + """Operation for deleting an evaluation set.""" + + name: Optional[str] + """The server-assigned name, which is only unique within the same service that originally returns it. If you use the default HTTP mapping, the `name` should be a resource name ending with `operations/{unique_id}`.""" + + metadata: Optional[dict[str, Any]] + """Service-specific metadata associated with the operation. It typically contains progress information and common metadata such as create time. Some services might not provide such metadata. Any method that returns a long-running operation should document the metadata type, if any.""" + + done: Optional[bool] + """If the value is `false`, it means the operation is still in progress. If `true`, the operation is completed, and either `error` or `response` is available.""" + + error: Optional[dict[str, Any]] + """The error result of the operation in case of failure or cancellation.""" + + +DeleteEvaluationSetOperationOrDict = Union[ + DeleteEvaluationSetOperation, DeleteEvaluationSetOperationDict +] + + class BleuInstance(_common.BaseModel): """Bleu instance.""" @@ -6447,6 +6533,109 @@ class ListEvaluationMetricsResponseDict(TypedDict, total=False): ] +class ListEvaluationSetsConfig(_common.BaseModel): + """Config for listing evaluation sets.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + page_size: Optional[int] = Field(default=None, description="""""") + page_token: Optional[str] = Field(default=None, description="""""") + filter: Optional[str] = Field( + default=None, + description="""An expression for filtering the results of the request. + For field names both snake_case and camelCase are supported. + For more information about filter syntax, see + `AIP-160 `_.""", + ) + order_by: Optional[str] = Field( + default=None, + description="""A comma-separated list of fields to order by, sorted in ascending + order by default. Use ``desc`` after a field name for descending. + Example: ``"create_time desc"``.""", + ) + + +class ListEvaluationSetsConfigDict(TypedDict, total=False): + """Config for listing evaluation sets.""" + + http_options: Optional[genai_types.HttpOptions] + """Used to override HTTP request options.""" + + page_size: Optional[int] + """""" + + page_token: Optional[str] + """""" + + filter: Optional[str] + """An expression for filtering the results of the request. + For field names both snake_case and camelCase are supported. + For more information about filter syntax, see + `AIP-160 `_.""" + + order_by: Optional[str] + """A comma-separated list of fields to order by, sorted in ascending + order by default. Use ``desc`` after a field name for descending. + Example: ``"create_time desc"``.""" + + +ListEvaluationSetsConfigOrDict = Union[ + ListEvaluationSetsConfig, ListEvaluationSetsConfigDict +] + + +class _ListEvaluationSetsParameters(_common.BaseModel): + """Parameters for listing evaluation sets.""" + + config: Optional[ListEvaluationSetsConfig] = Field(default=None, description="""""") + + +class _ListEvaluationSetsParametersDict(TypedDict, total=False): + """Parameters for listing evaluation sets.""" + + config: Optional[ListEvaluationSetsConfigDict] + """""" + + +_ListEvaluationSetsParametersOrDict = Union[ + _ListEvaluationSetsParameters, _ListEvaluationSetsParametersDict +] + + +class ListEvaluationSetsResponse(_common.BaseModel): + """Response for listing evaluation sets.""" + + sdk_http_response: Optional[genai_types.HttpResponse] = Field( + default=None, description="""Used to retain the full HTTP response.""" + ) + next_page_token: Optional[str] = Field(default=None, description="""""") + evaluation_sets: Optional[list[EvaluationSet]] = Field( + default=None, + description="""List of evaluation sets. + """, + ) + + +class ListEvaluationSetsResponseDict(TypedDict, total=False): + """Response for listing evaluation sets.""" + + sdk_http_response: Optional[genai_types.HttpResponse] + """Used to retain the full HTTP response.""" + + next_page_token: Optional[str] + """""" + + evaluation_sets: Optional[list[EvaluationSetDict]] + """List of evaluation sets. + """ + + +ListEvaluationSetsResponseOrDict = Union[ + ListEvaluationSetsResponse, ListEvaluationSetsResponseDict +] + + class UpdateEvaluationExperimentConfig(_common.BaseModel): """Config for updating an evaluation experiment.""" diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index 39c5224f36..9dd0b86321 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -12370,6 +12370,98 @@ def test_delete_evaluation_experiment_uses_delete_and_full_name(self): assert call_args[0][1] == self.experiment_name +class TestListEvaluationSets: + + def setup_method(self, method): + self.mock_api_client = mock.MagicMock() + self.mock_api_client.vertexai = True + self.mock_response = mock.MagicMock() + self.mock_response.body = json.dumps( + { + "evaluationSets": [ + { + "name": "projects/123/locations/us-central1/evaluationSets/1", + "displayName": "set_1", + }, + { + "name": "projects/123/locations/us-central1/evaluationSets/2", + "displayName": "set_2", + }, + ] + } + ) + self.mock_api_client.request.return_value = self.mock_response + + def test_list_evaluation_sets_returns_sets(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + response = evals_module.list_evaluation_sets() + + assert len(response.evaluation_sets) == 2 + assert isinstance( + response.evaluation_sets[0], agentplatform_genai_types.EvaluationSet + ) + assert response.evaluation_sets[0].display_name == "set_1" + assert response.evaluation_sets[1].display_name == "set_2" + + def test_list_evaluation_sets_gets_evaluation_sets_path(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.list_evaluation_sets() + + self.mock_api_client.request.assert_called_once() + call_args = self.mock_api_client.request.call_args + assert call_args[0][0] == "get" + assert call_args[0][1].startswith("evaluationSets") + + def test_list_evaluation_sets_passes_filter_and_order_by(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.list_evaluation_sets( + filter='display_name="set_1"', order_by="create_time desc" + ) + + self.mock_api_client.request.assert_called_once() + path = self.mock_api_client.request.call_args[0][1] + assert path.startswith("evaluationSets?") + assert "orderBy=create_time+desc" in path + + +class TestDeleteEvaluationSet: + + def setup_method(self, method): + self.mock_api_client = mock.MagicMock() + self.mock_api_client.vertexai = True + self.set_name = "projects/123/locations/us-central1/evaluationSets/456" + self.mock_response = mock.MagicMock() + self.mock_response.body = json.dumps({"name": "operations/789"}) + self.mock_api_client.request.return_value = self.mock_response + + def test_delete_evaluation_set_uses_delete_and_short_name(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.delete_evaluation_set(name=self.set_name) + + self.mock_api_client.request.assert_called_once() + call_args = self.mock_api_client.request.call_args + assert call_args[0][0] == "delete" + assert call_args[0][1] == "evaluationSets/456" + + def test_delete_evaluation_set_accepts_short_name(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.delete_evaluation_set(name="456") + + call_args = self.mock_api_client.request.call_args + assert call_args[0][1] == "evaluationSets/456" + + def test_delete_evaluation_set_raises_on_empty_name(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + with pytest.raises(ValueError): + evals_module.delete_evaluation_set(name="") + + class TestCreateEvaluationRunAutoExperiment: def setup_method(self, method):