diff --git a/python/lib/sift_client/_internal/low_level_wrappers/rules.py b/python/lib/sift_client/_internal/low_level_wrappers/rules.py index 08a279b85..5e3794066 100644 --- a/python/lib/sift_client/_internal/low_level_wrappers/rules.py +++ b/python/lib/sift_client/_internal/low_level_wrappers/rules.py @@ -3,9 +3,18 @@ import logging from typing import TYPE_CHECKING, Any, Sequence, cast -from sift.common.type.v1.resource_identifier_pb2 import ResourceIdentifier, ResourceIdentifiers +from sift.common.type.v1.resource_identifier_pb2 import ( + NamedResources, + Names, + ResourceIdentifier, + ResourceIdentifiers, +) from sift.rule_evaluation.v1.rule_evaluation_pb2 import ( AssetsTimeRange, + EvaluateRulesAnnotationOptions, + EvaluateRulesFromCurrentRuleVersions, + EvaluateRulesFromReportTemplate, + EvaluateRulesFromRuleVersions, EvaluateRulesRequest, EvaluateRulesResponse, RunTimeRange, @@ -15,6 +24,8 @@ ArchiveRuleRequest, BatchArchiveRulesRequest, BatchGetRulesRequest, + BatchGetRuleVersionsRequest, + BatchGetRuleVersionsResponse, BatchUnarchiveRulesRequest, BatchUpdateRulesRequest, BatchUpdateRulesResponse, @@ -24,7 +35,11 @@ CreateRuleResponse, GetRuleRequest, GetRuleResponse, + GetRuleVersionRequest, + GetRuleVersionResponse, ListRulesRequest, + ListRuleVersionsRequest, + ListRuleVersionsResponse, RuleAssetConfiguration, RuleConditionExpression, UnarchiveRuleRequest, @@ -45,6 +60,7 @@ Rule, RuleCreate, RuleUpdate, + RuleVersion, ) from sift_client.sift_types.tag import Tag from sift_client.transport import GrpcClient, WithGrpcClient @@ -506,6 +522,57 @@ async def list_all_rules( max_results=max_results, ) + async def list_rule_versions( + self, + rule_id: str, + *, + filter_query: str | None = None, + order_by: str | None = None, + page_size: int | None = None, + page_token: str | None = None, + ) -> tuple[list[RuleVersion], str]: + """List rule versions for a rule. + + Args: + rule_id: The rule ID to list versions for. + filter_query: Optional CEL filter (fields: rule_version_id, user_notes, change_message). + order_by: Unused, for _handle_pagination compatibility. + page_size: Maximum number of versions per page. + page_token: Token for the next page. + + Returns: + Tuple of (list of RuleVersions, next page token or empty string). + """ + _ = order_by + request_kwargs: dict[str, Any] = { + "rule_id": rule_id, + "page_size": page_size or DEFAULT_PAGE_SIZE, + "page_token": page_token or "", + } + if filter_query: + request_kwargs["filter"] = filter_query + request = ListRuleVersionsRequest(**request_kwargs) + response = await self._grpc_client.get_stub(RuleServiceStub).ListRuleVersions(request) + response = cast("ListRuleVersionsResponse", response) + versions = [RuleVersion._from_proto(p) for p in response.rule_versions] + return versions, response.next_page_token or "" + + async def list_all_rule_versions( + self, + rule_id: str, + *, + filter_query: str | None = None, + max_results: int | None = None, + page_size: int | None = DEFAULT_PAGE_SIZE, + ) -> list[RuleVersion]: + """List all rule versions for a rule, with optional CEL filter.""" + return await self._handle_pagination( + self.list_rule_versions, + kwargs={"rule_id": rule_id, "filter_query": filter_query}, + page_size=page_size, + max_results=max_results, + ) + async def evaluate_rules( self, *, @@ -571,13 +638,22 @@ async def evaluate_rules( if all_applicable_rules: kwargs["all_applicable_rules"] = all_applicable_rules if rule_ids: - kwargs["rules"] = {"rules": ResourceIdentifiers(ids={"ids": rule_ids})} # type: ignore + kwargs["rules"] = EvaluateRulesFromCurrentRuleVersions( + rules=ResourceIdentifiers(ids={"ids": rule_ids}) # type: ignore[arg-type] + ) if rule_version_ids: - kwargs["rule_versions"] = rule_version_ids + kwargs["rule_versions"] = EvaluateRulesFromRuleVersions( + rule_version_ids=rule_version_ids + ) if report_template_id: - kwargs["report_template"] = report_template_id + kwargs["report_template"] = EvaluateRulesFromReportTemplate( + report_template=ResourceIdentifier(id=report_template_id) + ) if tags: - kwargs["tags"] = [tag.name if isinstance(tag, Tag) else tag for tag in tags] + tag_names = [tag.name if isinstance(tag, Tag) else tag for tag in tags] + kwargs["annotation_options"] = EvaluateRulesAnnotationOptions( + tags=NamedResources(names=Names(names=tag_names)) # type: ignore[arg-type] + ) if report_name: kwargs["report_name"] = report_name if organization_id: @@ -595,3 +671,31 @@ async def evaluate_rules( report = await ReportsLowLevelClient(self._grpc_client).get_report(report_id=report_id) return created_annotation_count, report, job_id return created_annotation_count, None, job_id + + async def get_rule_version(self, rule_version_id: str) -> Rule: + """Get a rule at a specific version by rule_version_id. + + Args: + rule_version_id: The rule version ID to get. + + Returns: + The Rule at that version. + """ + request = GetRuleVersionRequest(rule_version_id=rule_version_id) + response = await self._grpc_client.get_stub(RuleServiceStub).GetRuleVersion(request) + grpc_rule = cast("GetRuleVersionResponse", response).rule + return Rule._from_proto(grpc_rule) + + async def batch_get_rule_versions(self, rule_version_ids: list[str]) -> list[Rule]: + """Get multiple rules at specific versions by rule_version_ids. + + Args: + rule_version_ids: The rule version IDs to get. + + Returns: + List of Rules at those versions (order may match request order). + """ + request = BatchGetRuleVersionsRequest(rule_version_ids=rule_version_ids) + response = await self._grpc_client.get_stub(RuleServiceStub).BatchGetRuleVersions(request) + response = cast("BatchGetRuleVersionsResponse", response) + return [Rule._from_proto(r) for r in response.rules] diff --git a/python/lib/sift_client/_tests/resources/test_reports.py b/python/lib/sift_client/_tests/resources/test_reports.py index 8472ed276..3eb3e6f05 100644 --- a/python/lib/sift_client/_tests/resources/test_reports.py +++ b/python/lib/sift_client/_tests/resources/test_reports.py @@ -54,6 +54,35 @@ def test_client_binding(sift_client): @pytest.mark.integration class TestReports: + def test_create_from_rule_versions(self, nostromo_run, test_rule, sift_client): + """Create a report from specific rule version IDs.""" + rule_versions = sift_client.rules.list_rule_versions(test_rule) + assert rule_versions, "test_rule should have at least one version" + report = sift_client.reports.create_from_rule_versions( + name="report_from_rule_versions", + run=nostromo_run, + organization_id=nostromo_run.organization_id, + rule_versions=[rule_versions[0].rule_version_id], + ) + assert report is not None + assert report.run_id == nostromo_run.id_ + assert report.name == "report_from_rule_versions" + + def test_create_from_rule_versions_with_rule_version_objects( + self, nostromo_run, test_rule, sift_client + ): + """Create a report passing RuleVersion instances.""" + rule_versions = sift_client.rules.list_rule_versions(test_rule) + assert rule_versions + report = sift_client.reports.create_from_rule_versions( + name="report_from_rule_versions_objs", + run=nostromo_run, + organization_id=nostromo_run.organization_id, + rule_versions=rule_versions[:1], + ) + assert report is not None + assert report.run_id == nostromo_run.id_ + def test_create_from_rules(self, nostromo_run, test_rule, sift_client): report_from_rules = sift_client.reports.create_from_rules( name="report_from_rules", @@ -146,17 +175,16 @@ def test_archive(self, nostromo_run, test_rule, sift_client): assert archived_report is not None assert archived_report.is_archived == True - def test_unarchive(self, sift_client): - reports_from_rules = sift_client.reports.list_( - name="report_from_rules", include_archived=True + def test_unarchive(self, nostromo_run, test_rule, sift_client): + # create a report, archive it, then unarchive it + report_from_rules = sift_client.reports.create_from_rules( + name="report_from_rules_unarchive", + run=nostromo_run, + rules=[test_rule], ) - report_from_rules = None - for report_from_rules in reports_from_rules: - if report_from_rules.is_archived: - report_from_rules = report_from_rules - break assert report_from_rules is not None - assert report_from_rules.is_archived == True - unarchived_report = sift_client.reports.unarchive(report=report_from_rules) + archived_report = sift_client.reports.archive(report=report_from_rules) + assert archived_report.is_archived is True + unarchived_report = sift_client.reports.unarchive(report=archived_report) assert unarchived_report is not None - assert unarchived_report.is_archived == False + assert unarchived_report.is_archived is False diff --git a/python/lib/sift_client/_tests/resources/test_rules.py b/python/lib/sift_client/_tests/resources/test_rules.py index c0fc581ea..d181d80d1 100644 --- a/python/lib/sift_client/_tests/resources/test_rules.py +++ b/python/lib/sift_client/_tests/resources/test_rules.py @@ -22,6 +22,7 @@ RuleAnnotationType, RuleCreate, RuleUpdate, + RuleVersion, ) pytestmark = pytest.mark.integration @@ -215,6 +216,118 @@ async def test_list_with_time_filters(self, rules_api_async): for rule in rules: assert rule.created_date >= one_year_ago + class TestListRuleVersions: + """Tests for the async list_rule_versions method.""" + + @pytest.mark.asyncio + async def test_list_rule_versions_by_rule(self, rules_api_async, test_rule): + """Test listing rule versions for a rule.""" + versions = await rules_api_async.list_rule_versions(test_rule) + assert isinstance(versions, list) + assert len(versions) >= 1 + for v in versions: + assert isinstance(v, RuleVersion) + assert v.rule_id == test_rule.id_ + assert v.rule_version_id + assert v.version + assert v.created_date + + @pytest.mark.asyncio + async def test_list_rule_versions_by_rule_id_str(self, rules_api_async, test_rule): + """Test listing rule versions by rule ID string.""" + versions = await rules_api_async.list_rule_versions(test_rule.id_) + assert isinstance(versions, list) + assert len(versions) >= 1 + for v in versions: + assert v.rule_id == test_rule.id_ + + @pytest.mark.asyncio + async def test_list_rule_versions_with_limit(self, rules_api_async, test_rule): + """Test listing rule versions with limit.""" + versions = await rules_api_async.list_rule_versions(test_rule, limit=1) + assert isinstance(versions, list) + assert len(versions) <= 1 + if versions: + assert isinstance(versions[0], RuleVersion) + + @pytest.mark.asyncio + async def test_list_rule_versions_with_rule_version_ids_filter( + self, rules_api_async, test_rule + ): + """Test listing rule versions filtered by rule_version_ids.""" + all_versions = await rules_api_async.list_rule_versions(test_rule) + assert all_versions + first_id = all_versions[0].rule_version_id + versions = await rules_api_async.list_rule_versions( + test_rule, rule_version_ids=[first_id] + ) + assert len(versions) == 1 + assert versions[0].rule_version_id == first_id + + class TestGetRuleVersion: + """Tests for the async get_rule_version method.""" + + @pytest.mark.asyncio + async def test_get_rule_version_by_id(self, rules_api_async, test_rule): + """Test getting a rule at a specific version by rule_version_id.""" + versions = await rules_api_async.list_rule_versions(test_rule) + assert versions + rule_at_version = await rules_api_async.get_rule_version(versions[0].rule_version_id) + assert rule_at_version is not None + assert rule_at_version.id_ == test_rule.id_ + assert rule_at_version.rule_version is not None + assert rule_at_version.rule_version.rule_version_id == versions[0].rule_version_id + + @pytest.mark.asyncio + async def test_get_rule_version_by_rule_version_instance(self, rules_api_async, test_rule): + """Test getting a rule at a specific version by passing RuleVersion instance.""" + versions = await rules_api_async.list_rule_versions(test_rule) + assert versions + rule_at_version = await rules_api_async.get_rule_version(versions[0]) + assert rule_at_version is not None + assert rule_at_version.id_ == test_rule.id_ + assert rule_at_version.rule_version.rule_version_id == versions[0].rule_version_id + + class TestBatchGetRuleVersions: + """Tests for the async batch_get_rule_versions method.""" + + @pytest.mark.asyncio + async def test_batch_get_rule_versions_by_ids(self, rules_api_async, test_rule): + """Test batch getting rules by rule_version_id strings.""" + versions = await rules_api_async.list_rule_versions(test_rule) + assert versions + ids = [v.rule_version_id for v in versions[:2]] + rules = await rules_api_async.batch_get_rule_versions(ids) + assert len(rules) == len(ids) + returned_ids = {r.rule_version.rule_version_id for r in rules if r.rule_version} + assert returned_ids >= set(ids) + for r in rules: + assert r.id_ == test_rule.id_ + + @pytest.mark.asyncio + async def test_batch_get_rule_versions_by_rule_version_instances( + self, rules_api_async, test_rule + ): + """Test batch getting rules by passing RuleVersion instances.""" + versions = await rules_api_async.list_rule_versions(test_rule) + assert versions + rules = await rules_api_async.batch_get_rule_versions(versions[:2]) + assert len(rules) <= 2 + for r in rules: + assert r.id_ == test_rule.id_ + if len(versions) >= 2: + assert len(rules) == 2 + + @pytest.mark.asyncio + async def test_batch_get_rule_versions_single(self, rules_api_async, test_rule): + """Test batch_get_rule_versions with a single version ID.""" + versions = await rules_api_async.list_rule_versions(test_rule) + assert versions + rules = await rules_api_async.batch_get_rule_versions([versions[0].rule_version_id]) + assert len(rules) == 1 + assert rules[0].id_ == test_rule.id_ + assert rules[0].rule_version.rule_version_id == versions[0].rule_version_id + class TestFind: """Tests for the async find method.""" diff --git a/python/lib/sift_client/resources/reports.py b/python/lib/sift_client/resources/reports.py index 86a19d5ae..df3b4f0d1 100644 --- a/python/lib/sift_client/resources/reports.py +++ b/python/lib/sift_client/resources/reports.py @@ -6,7 +6,7 @@ from sift_client._internal.low_level_wrappers.rules import RulesLowLevelClient from sift_client.resources._base import ResourceBase from sift_client.sift_types.report import Report, ReportUpdate -from sift_client.sift_types.rule import Rule +from sift_client.sift_types.rule import Rule, RuleVersion from sift_client.sift_types.run import Run from sift_client.util import cel_utils as cel @@ -266,6 +266,45 @@ async def create_from_applicable_rules( return self._apply_client_to_instance(created_report) return None + async def create_from_rule_versions( + self, + *, + name: str, + run: Run | str | None = None, + organization_id: str | None = None, + rule_versions: list[RuleVersion] | list[str], + ) -> Report | None: + """Create a new report from rule versions. + + Args: + name: The name of the report. + run: The run or run ID to associate with the report. + organization_id: The organization ID. + rule_versions: List of RuleVersions or rule_version IDs to include in the report. + + Returns: + The created Report or None if no report was created. + """ + ( + created_annotation_count, + created_report, + job_id, + ) = await self._rules_low_level_client.evaluate_rules( + run_id=run._id_or_error if isinstance(run, Run) else run, + organization_id=organization_id, + rule_version_ids=[ + rule_version.rule_version_id + if isinstance(rule_version, RuleVersion) + else rule_version + for rule_version in rule_versions + ] + or [], + report_name=name, + ) + if created_report: + return self._apply_client_to_instance(created_report) + return None + async def rerun( self, *, diff --git a/python/lib/sift_client/resources/rules.py b/python/lib/sift_client/resources/rules.py index b6734e610..08c5815e3 100644 --- a/python/lib/sift_client/resources/rules.py +++ b/python/lib/sift_client/resources/rules.py @@ -7,7 +7,7 @@ from sift_client.errors import SiftWarning from sift_client.resources._base import ResourceBase from sift_client.sift_types.asset import Asset -from sift_client.sift_types.rule import Rule, RuleCreate, RuleUpdate +from sift_client.sift_types.rule import Rule, RuleCreate, RuleUpdate, RuleVersion from sift_client.util import cel_utils as cel if TYPE_CHECKING: @@ -328,3 +328,84 @@ async def batch_update_or_create_rules( # Fetch the rules. updated_rules = await self._low_level_client.batch_get_rules(rule_ids=final_rule_ids) return self._apply_client_to_instances(updated_rules) + + async def list_rule_versions( + self, + rule: Rule | str, + *, + user_notes_contains: str | None = None, + change_message_contains: str | None = None, + rule_version_ids: list[str] | None = None, + filter_query: str | None = None, + limit: int | None = None, + ) -> list[RuleVersion]: + """List versions of a rule with optional filtering. + + Args: + rule: The Rule instance or rule ID. + user_notes_contains: Filter by user notes (notes for a given version) containing this string. + change_message_contains: Filter by change messages containing this string. + rule_version_ids: Limit to these rule version IDs. + filter_query: Raw CEL filter (fields: rule_version_id, user_notes, change_message). + limit: Maximum number of versions to return. If None, returns all matches. + + Returns: + A list of RuleVersion objects matching the filters, ordered by newest versions first. + """ + if isinstance(rule, Rule): + rule_id = rule._id_or_error + else: + rule_id = rule + + filter_parts: list[str] = [] + if user_notes_contains: + filter_parts.append(cel.contains("user_notes", user_notes_contains)) + if change_message_contains: + filter_parts.append(cel.contains("change_message", change_message_contains)) + if rule_version_ids: + filter_parts.append(cel.in_("rule_version_id", rule_version_ids)) + if filter_query: + filter_parts.append(filter_query) + query_filter = cel.and_(*filter_parts) if filter_parts else None + + return await self._low_level_client.list_all_rule_versions( + rule_id=rule_id, + filter_query=query_filter, + max_results=limit, + page_size=limit, + ) + + async def get_rule_version(self, rule_version: RuleVersion | str) -> Rule: + """Get a rule at a specific version by rule version ID. + + Args: + rule_version: The RuleVersion instance or rule version ID. + + Returns: + The Rule at that version. + """ + if isinstance(rule_version, RuleVersion): + rule_version_id = rule_version.rule_version_id + else: + rule_version_id = rule_version + rule = await self._low_level_client.get_rule_version(rule_version_id=rule_version_id) + return self._apply_client_to_instance(rule) + + async def batch_get_rule_versions( + self, rule_versions: list[RuleVersion] | list[str] + ) -> list[Rule]: + """Get multiple rules at specific versions by rule version IDs. + + Args: + rule_versions: List of RuleVersion instances or rule version IDs. + + Returns: + List of Rules at those versions. + """ + rule_version_ids = [ + rv.rule_version_id if isinstance(rv, RuleVersion) else rv for rv in rule_versions + ] + rules = await self._low_level_client.batch_get_rule_versions( + rule_version_ids=rule_version_ids + ) + return self._apply_client_to_instances(rules) diff --git a/python/lib/sift_client/resources/sync_stubs/__init__.pyi b/python/lib/sift_client/resources/sync_stubs/__init__.pyi index 35399d4cd..0fe3a628f 100644 --- a/python/lib/sift_client/resources/sync_stubs/__init__.pyi +++ b/python/lib/sift_client/resources/sync_stubs/__init__.pyi @@ -28,7 +28,7 @@ if TYPE_CHECKING: ) from sift_client.sift_types.job import Job, JobStatus, JobType from sift_client.sift_types.report import Report, ReportUpdate - from sift_client.sift_types.rule import Rule, RuleCreate, RuleUpdate + from sift_client.sift_types.rule import Rule, RuleCreate, RuleUpdate, RuleVersion from sift_client.sift_types.run import Run, RunCreate, RunUpdate from sift_client.sift_types.tag import Tag, TagUpdate from sift_client.sift_types.test_report import ( @@ -819,6 +819,27 @@ class ReportsAPI: """ ... + def create_from_rule_versions( + self, + *, + name: str, + run: Run | str | None = None, + organization_id: str | None = None, + rule_versions: list[RuleVersion] | list[str], + ) -> Report | None: + """Create a new report from rule versions. + + Args: + name: The name of the report. + run: The run or run ID to associate with the report. + organization_id: The organization ID. + rule_versions: List of RuleVersions or rule_version IDs to include in the report. + + Returns: + The created Report or None if no report was created. + """ + ... + def create_from_rules( self, *, @@ -995,6 +1016,17 @@ class RulesAPI: """ ... + def batch_get_rule_versions(self, rule_versions: list[RuleVersion] | list[str]) -> list[Rule]: + """Get multiple rules at specific versions by rule version IDs. + + Args: + rule_versions: List of RuleVersion instances or rule version IDs. + + Returns: + List of Rules at those versions. + """ + ... + def batch_update_or_create_rules( self, rules: Sequence[RuleCreate | RuleUpdate], @@ -1062,6 +1094,17 @@ class RulesAPI: """ ... + def get_rule_version(self, rule_version: RuleVersion | str) -> Rule: + """Get a rule at a specific version by rule version ID. + + Args: + rule_version: The RuleVersion instance or rule version ID. + + Returns: + The Rule at that version. + """ + ... + def list_( self, *, @@ -1115,6 +1158,31 @@ class RulesAPI: """ ... + def list_rule_versions( + self, + rule: Rule | str, + *, + user_notes_contains: str | None = None, + change_message_contains: str | None = None, + rule_version_ids: list[str] | None = None, + filter_query: str | None = None, + limit: int | None = None, + ) -> list[RuleVersion]: + """List versions of a rule with optional filtering. + + Args: + rule: The Rule instance or rule ID. + user_notes_contains: Filter by user notes (notes for a given version) containing this string. + change_message_contains: Filter by change messages containing this string. + rule_version_ids: Limit to these rule version IDs. + filter_query: Raw CEL filter (fields: rule_version_id, user_notes, change_message). + limit: Maximum number of versions to return. If None, returns all matches. + + Returns: + A list of RuleVersion objects matching the filters, ordered by newest versions first. + """ + ... + def unarchive(self, rule: str | Rule) -> Rule: """Unarchive a rule. diff --git a/python/lib/sift_py/ingestion/_internal/ingest.py b/python/lib/sift_py/ingestion/_internal/ingest.py index b086e1889..1da6268c4 100644 --- a/python/lib/sift_py/ingestion/_internal/ingest.py +++ b/python/lib/sift_py/ingestion/_internal/ingest.py @@ -520,11 +520,11 @@ def _update_flow_configs( # We can have multiple channels of the same name but different data-type. This will create a completely unique channel # identifier by creating a composite key of the fully qualified channel name with the channel's data-type. - sift_channel_identifier: Callable[[ChannelConfigPb], str] = ( - lambda x: f"{channel_fqn(x)}.{x.data_type}" + sift_channel_identifier: Callable[[ChannelConfigPb], str] = lambda x: ( + f"{channel_fqn(x)}.{x.data_type}" ) - config_channel_identifier: Callable[[ChannelConfig], str] = ( - lambda x: f"{channel_fqn(x)}.{x.data_type.value}" + config_channel_identifier: Callable[[ChannelConfig], str] = lambda x: ( + f"{channel_fqn(x)}.{x.data_type.value}" ) for config_flow in config_flows: