diff --git a/src/sentry/seer/night_shift/delivery.py b/src/sentry/seer/night_shift/delivery.py index 84676c82600e..435594d16430 100644 --- a/src/sentry/seer/night_shift/delivery.py +++ b/src/sentry/seer/night_shift/delivery.py @@ -12,18 +12,26 @@ from sentry.models.group import Group from sentry.models.organization import Organization from sentry.seer.agent.types import FeatureRunStatus +from sentry.seer.autofix.autofix_agent import AutofixStep, trigger_autofix_agent +from sentry.seer.autofix.constants import SeerAutomationSource +from sentry.seer.autofix.issue_summary import referrer_map from sentry.seer.autofix.utils import AutofixStoppingPoint, bulk_read_preferences_from_sentry_db from sentry.seer.models.night_shift import ( SeerNightShiftRun, SeerNightShiftRunResult, SeerNightShiftRunShard, ) -from sentry.seer.night_shift.models import TriageResponse -from sentry.tasks.seer.night_shift.models import TriageAction, TriageResult +from sentry.seer.models.run import SeerRun +from sentry.seer.models.workflow import SeerWorkflowStrategy +from sentry.seer.night_shift.models import TriageResponse, TriageVerdict +from sentry.tasks.seer.night_shift.models import TriageAction from sentry.tasks.seer.night_shift.skip_cache import mark_skipped logger = logging.getLogger(__name__) +# Verdict reasons are LLM-generated free text; cap what we persist per row. +REASON_MAX_CHARS = 2048 + def deliver_night_shift_result( organization_id: int, @@ -102,10 +110,8 @@ def _process_verdicts( dry_run: bool, log_extra: Mapping[str, object], ) -> None: - """Mark SKIPs, fire autofix for fixable verdicts, persist result rows.""" - # Import here to avoid circular import - from sentry.tasks.seer.night_shift.cron import _run_autofix_for_candidates - + """Mark SKIPs, fire autofix for fixable verdicts, and persist one result row + per verdict (every action, dry runs included) for later analysis.""" group_ids = [v.group_id for v in triage_response.verdicts] groups_by_id: dict[int, Group] = { g.id: g @@ -123,33 +129,45 @@ def _process_verdicts( extra={**log_extra, "unknown_group_ids": unknown_group_ids}, ) + # Groups this run already has a result row for (e.g. Seer redelivered a + # shard result): don't re-trigger autofix or write duplicate rows for them. + recorded_group_ids = set( + SeerNightShiftRunResult.objects.filter(run=run, group_id__in=group_ids).values_list( + "group_id", flat=True + ) + ) + # SKIP and ROOT_CAUSE_ONLY are both suppressed from future runs via the skip # cache. ROOT_CAUSE_ONLY keeps its own action value for tracking, but is # otherwise treated identically to SKIP (it does not trigger autofix). + verdicts: list[TriageVerdict] = [] + fixable_groups: list[Group] = [] for v in triage_response.verdicts: - if ( - v.action in (TriageAction.SKIP, TriageAction.ROOT_CAUSE_ONLY) - and v.group_id in groups_by_id - ): + group = groups_by_id.get(v.group_id) + if group is None or v.group_id in recorded_group_ids: + continue + verdicts.append(v) + if v.action in (TriageAction.SKIP, TriageAction.ROOT_CAUSE_ONLY): mark_skipped(v.group_id) + elif v.action == TriageAction.AUTOFIX: + fixable_groups.append(group) - # Convert verdicts to TriageResult objects for the shared function - fixable_candidates = [ - TriageResult(group=groups_by_id[v.group_id], action=v.action, reason=v.reason) - for v in triage_response.verdicts - if v.action == TriageAction.AUTOFIX and v.group_id in groups_by_id - ] - - sentry_sdk.metrics.distribution("night_shift.candidates_selected", len(fixable_candidates)) + sentry_sdk.metrics.distribution("night_shift.candidates_selected", len(fixable_groups)) + if not fixable_groups: + logger.info( + "night_shift.no_fixable_candidates", + extra={**log_extra, "num_candidates": len(verdicts)}, + ) - results: list[SeerNightShiftRunResult] = [] - if not dry_run and fixable_candidates: + reason_by_group_id = {v.group_id: v.reason for v in verdicts} + state_id_by_group: dict[int, int] = {} + if not dry_run and fixable_groups: # Cache organization on each group's project to avoid N+1 queries for group in groups_by_id.values(): group.project.organization = organization # Build stopping_point_by_project_id from project preferences (bulk query) - project_ids = {c.group.project_id for c in fixable_candidates} + project_ids = {group.project_id for group in fixable_groups} preferences = bulk_read_preferences_from_sentry_db(organization.id, list(project_ids)) default_stopping_point = AutofixStoppingPoint(SEER_AUTOMATED_RUN_STOPPING_POINT_DEFAULT) stopping_point_by_project_id = { @@ -161,25 +179,79 @@ def _process_verdicts( for pid in project_ids } - results = _run_autofix_for_candidates( - run=run, - candidates=fixable_candidates, - stopping_point_by_project_id=stopping_point_by_project_id, - log_extra=dict(log_extra), + referrer = referrer_map[SeerAutomationSource.NIGHT_SHIFT] + for group in fixable_groups: + reason = reason_by_group_id[group.id] + user_context = ( + f"Night-shift triage already investigated this issue and concluded:\n{reason}" + if reason + else None + ) + try: + state_id_by_group[group.id] = trigger_autofix_agent( + group=group, + step=AutofixStep.ROOT_CAUSE, + referrer=referrer, + stopping_point=stopping_point_by_project_id[group.project_id], + user_context=user_context, + ) + except Exception: + logger.exception( + "night_shift.autofix_trigger_failed", + extra={**log_extra, "group_id": group.id}, + ) + + sentry_sdk.metrics.count("night_shift.autofix_triggered", len(state_id_by_group)) + + # TODO: have trigger_autofix_agent return the SeerRun directly to avoid this lookup. + seer_run_by_state_id = { + sr.seer_run_state_id: sr + for sr in SeerRun.objects.filter(seer_run_state_id__in=state_id_by_group.values()) + } + + rows: list[SeerNightShiftRunResult] = [] + for v in verdicts: + extras: dict[str, Any] = {"action": str(v.action)} + if v.reason: + extras["reason"] = v.reason[:REASON_MAX_CHARS] + seer_run_id: str | None = None + result_seer_run: SeerRun | None = None + if v.action == TriageAction.AUTOFIX and not dry_run: + state_id = state_id_by_group.get(v.group_id) + if state_id is None: + extras["trigger_error"] = True + else: + seer_run_id = str(state_id) + result_seer_run = seer_run_by_state_id.get(state_id) + rows.append( + SeerNightShiftRunResult( + run=run, + kind=SeerWorkflowStrategy.AGENTIC_TRIAGE, + group=groups_by_id[v.group_id], + seer_run_id=seer_run_id, + result_seer_run=result_seer_run, + extras=extras, + ) ) + # ignore_conflicts: concurrent redeliveries can race past the recorded-rows check. + SeerNightShiftRunResult.objects.bulk_create(rows, ignore_conflicts=True) - seer_run_id_by_group = {r.group_id: r.seer_run_id for r in results} logger.info( "night_shift.candidates_selected", extra={ **log_extra, "num_verdicts": len(triage_response.verdicts), + "num_already_recorded": len(recorded_group_ids), "dry_run": dry_run, "candidates": [ { "group_id": v.group_id, "action": v.action, - "seer_run_id": seer_run_id_by_group.get(v.group_id), + "seer_run_id": ( + str(state_id) + if (state_id := state_id_by_group.get(v.group_id)) is not None + else None + ), } for v in triage_response.verdicts ], diff --git a/src/sentry/tasks/seer/night_shift/cron.py b/src/sentry/tasks/seer/night_shift/cron.py index 94cccffa6e04..648edf459151 100644 --- a/src/sentry/tasks/seer/night_shift/cron.py +++ b/src/sentry/tasks/seer/night_shift/cron.py @@ -21,17 +21,13 @@ from sentry.models.organization import Organization, OrganizationStatus from sentry.models.project import Project from sentry.seer.agent.client import SeerAgentClient -from sentry.seer.autofix.autofix_agent import AutofixStep, trigger_autofix_agent from sentry.seer.autofix.constants import ( AutofixAutomationTuningSettings, - SeerAutomationSource, ) -from sentry.seer.autofix.issue_summary import referrer_map from sentry.seer.autofix.utils import AutofixStoppingPoint, bulk_read_preferences_from_sentry_db from sentry.seer.models import SeerPermissionError from sentry.seer.models.night_shift import ( SeerNightShiftRun, - SeerNightShiftRunResult, SeerNightShiftRunShard, ) from sentry.seer.models.project_repository import SeerProjectRepository @@ -39,7 +35,6 @@ from sentry.seer.models.workflow import SeerWorkflowConfig, SeerWorkflowStrategy from sentry.seer.night_shift.models import NightShiftPayload, TriageCandidate, TriageTweaks from sentry.tasks.base import instrumented_task -from sentry.tasks.seer.night_shift.models import TriageAction, TriageResult from sentry.tasks.seer.night_shift.simple_triage import ( ScoredCandidate, fixability_score_strategy, @@ -313,6 +308,8 @@ def run_night_shift_execution( return None sentry_sdk.metrics.distribution("night_shift.eligible_projects", len(eligible)) + # Stamped so zero-shard runs are distinguishable: no eligible projects vs. no candidates. + run.update(extras={**(run.extras or {}), "num_eligible_projects": len(eligible)}) if not eligible: logger.info("night_shift.no_eligible_projects", extra=log_extra) @@ -560,6 +557,7 @@ def _dispatch_to_seer_feature( eligible_projects = [ep.project for ep in eligible] repos_by_project = {ep.project.id: ep.connected_repos for ep in eligible} scored = fixability_score_strategy(eligible_projects, resolved_options["max_candidates"]) + run.update(extras={**(run.extras or {}), "num_candidates": len(scored)}) if not scored: logger.info("night_shift.no_candidates", extra=log_extra) return @@ -620,69 +618,3 @@ def _link_shard(created: SeerRun) -> None: "num_shards_dispatched": dispatched, }, ) - - -def _run_autofix_for_candidates( - run: SeerNightShiftRun, - candidates: Sequence[TriageResult], - stopping_point_by_project_id: Mapping[int, AutofixStoppingPoint], - log_extra: dict[str, object], -) -> list[SeerNightShiftRunResult]: - """ - For each fixable triage candidate, trigger a Seer autofix run and persist - the resulting run id onto a newly created SeerNightShiftRunResult row. - Returns the list of rows that were created. - """ - fixable_candidates = [c for c in candidates if c.action == TriageAction.AUTOFIX] - if not fixable_candidates: - logger.info( - "night_shift.no_fixable_candidates", - extra={**log_extra, "num_candidates": len(candidates)}, - ) - return [] - - referrer = referrer_map[SeerAutomationSource.NIGHT_SHIFT] - - results: list[SeerNightShiftRunResult] = [] - for c in fixable_candidates: - stopping_point = stopping_point_by_project_id[c.group.project_id] - - user_context = ( - f"Night-shift triage already investigated this issue and concluded:\n{c.reason}" - if c.reason - else None - ) - - try: - seer_run_id = trigger_autofix_agent( - group=c.group, - step=AutofixStep.ROOT_CAUSE, - referrer=referrer, - stopping_point=stopping_point, - user_context=user_context, - ) - except Exception: - logger.exception( - "night_shift.autofix_trigger_failed", - extra={**log_extra, "group_id": c.group.id}, - ) - continue - - # TODO: have trigger_autofix_agent return the SeerRun directly to avoid this lookup. - result_seer_run = SeerRun.objects.filter(seer_run_state_id=seer_run_id).first() - results.append( - SeerNightShiftRunResult( - run=run, - kind=SeerWorkflowStrategy.AGENTIC_TRIAGE, - group=c.group, - seer_run_id=str(seer_run_id), - result_seer_run=result_seer_run, - extras={"action": str(c.action)}, - ) - ) - - SeerNightShiftRunResult.objects.bulk_create(results) - - sentry_sdk.metrics.count("night_shift.autofix_triggered", len(results)) - - return results diff --git a/tests/sentry/seer/night_shift/test_delivery.py b/tests/sentry/seer/night_shift/test_delivery.py index 44b2cdeb2db7..d576e0f6f783 100644 --- a/tests/sentry/seer/night_shift/test_delivery.py +++ b/tests/sentry/seer/night_shift/test_delivery.py @@ -8,7 +8,7 @@ SeerNightShiftRunResult, SeerNightShiftRunShard, ) -from sentry.seer.night_shift.delivery import deliver_night_shift_result +from sentry.seer.night_shift.delivery import REASON_MAX_CHARS, deliver_night_shift_result from sentry.tasks.seer.night_shift.models import TriageAction from sentry.tasks.seer.night_shift.skip_cache import key as skip_cache_key from sentry.testutils.cases import TestCase @@ -91,7 +91,7 @@ def test_sibling_shard_success_keeps_other_shard_error(self) -> None: result=None, error="shard failed", ) - with patch("sentry.tasks.seer.night_shift.cron.trigger_autofix_agent", return_value=1): + with patch("sentry.seer.night_shift.delivery.trigger_autofix_agent", return_value=1): deliver_night_shift_result( organization_id=org.id, run_uuid=str(ok_seer_run.uuid), @@ -126,7 +126,8 @@ def test_invalid_result_logs_exception(self) -> None: assert not SeerNightShiftRunResult.objects.filter(run=run).exists() def test_skip_verdict_marks_group_skipped(self) -> None: - """SKIP verdicts should mark the group in skip cache.""" + """SKIP verdicts mark the group in the skip cache and persist a result + row without a seer run.""" org = self.create_organization() project = self.create_project(organization=org) group = self.create_group(project=project) @@ -138,7 +139,7 @@ def test_skip_verdict_marks_group_skipped(self) -> None: ] } - with patch("sentry.tasks.seer.night_shift.cron.trigger_autofix_agent") as mock_trigger: + with patch("sentry.seer.night_shift.delivery.trigger_autofix_agent") as mock_trigger: deliver_night_shift_result( organization_id=org.id, run_uuid=self._run_uuid(run), @@ -156,8 +157,13 @@ def test_skip_verdict_marks_group_skipped(self) -> None: finally: redis.delete(skip_cache_key(group.id)) - # No results persisted for SKIP verdicts - assert not SeerNightShiftRunResult.objects.filter(run=run).exists() + skip_result = SeerNightShiftRunResult.objects.get(run=run) + assert skip_result.group_id == group.id + assert skip_result.seer_run_id is None + assert skip_result.result_seer_run is None + assert skip_result.extras["action"] == TriageAction.SKIP.value + assert skip_result.extras["reason"] == "not fixable" + assert "trigger_error" not in skip_result.extras def test_autofix_verdict_triggers_autofix(self) -> None: """AUTOFIX verdicts should trigger autofix with project stopping point.""" @@ -176,7 +182,7 @@ def test_autofix_verdict_triggers_autofix(self) -> None: } with patch( - "sentry.tasks.seer.night_shift.cron.trigger_autofix_agent", return_value=42 + "sentry.seer.night_shift.delivery.trigger_autofix_agent", return_value=42 ) as mock_trigger: deliver_night_shift_result( organization_id=org.id, @@ -195,6 +201,7 @@ def test_autofix_verdict_triggers_autofix(self) -> None: assert results[0].group_id == group.id assert results[0].seer_run_id == "42" assert results[0].extras["action"] == TriageAction.AUTOFIX.value + assert results[0].extras["reason"] == "looks good" def test_root_cause_only_verdict_marks_group_skipped(self) -> None: """ROOT_CAUSE_ONLY verdicts are treated like SKIP: marked in the skip @@ -214,7 +221,7 @@ def test_root_cause_only_verdict_marks_group_skipped(self) -> None: ] } - with patch("sentry.tasks.seer.night_shift.cron.trigger_autofix_agent") as mock_trigger: + with patch("sentry.seer.night_shift.delivery.trigger_autofix_agent") as mock_trigger: deliver_night_shift_result( organization_id=org.id, run_uuid=self._run_uuid(run), @@ -232,11 +239,13 @@ def test_root_cause_only_verdict_marks_group_skipped(self) -> None: finally: redis.delete(skip_cache_key(group.id)) - # No results persisted for ROOT_CAUSE_ONLY verdicts - assert not SeerNightShiftRunResult.objects.filter(run=run).exists() + result_row = SeerNightShiftRunResult.objects.get(run=run) + assert result_row.group_id == group.id + assert result_row.seer_run_id is None + assert result_row.extras["action"] == TriageAction.ROOT_CAUSE_ONLY.value def test_dry_run_skips_autofix(self) -> None: - """Dry run mode should not trigger autofix or persist results.""" + """Dry run mode should not trigger autofix but still persist verdict rows.""" org = self.create_organization() project = self.create_project(organization=org) group = self.create_group(project=project) @@ -248,7 +257,7 @@ def test_dry_run_skips_autofix(self) -> None: ] } - with patch("sentry.tasks.seer.night_shift.cron.trigger_autofix_agent") as mock_trigger: + with patch("sentry.seer.night_shift.delivery.trigger_autofix_agent") as mock_trigger: deliver_night_shift_result( organization_id=org.id, run_uuid=self._run_uuid(run), @@ -259,7 +268,12 @@ def test_dry_run_skips_autofix(self) -> None: mock_trigger.assert_not_called() - assert not SeerNightShiftRunResult.objects.filter(run=run).exists() + result_row = SeerNightShiftRunResult.objects.get(run=run) + assert result_row.group_id == group.id + assert result_row.seer_run_id is None + assert result_row.extras["action"] == TriageAction.AUTOFIX.value + # An untriggered dry-run verdict is not a trigger failure. + assert "trigger_error" not in result_row.extras def test_trigger_failure_continues_with_other_groups(self) -> None: """If trigger fails for one group, continue processing others.""" @@ -291,10 +305,10 @@ def trigger_side_effect(**kwargs: Any) -> int: with ( patch( - "sentry.tasks.seer.night_shift.cron.trigger_autofix_agent", + "sentry.seer.night_shift.delivery.trigger_autofix_agent", side_effect=trigger_side_effect, ), - patch("sentry.tasks.seer.night_shift.cron.logger") as mock_logger, + patch("sentry.seer.night_shift.delivery.logger") as mock_logger, ): deliver_night_shift_result( organization_id=org.id, @@ -307,10 +321,13 @@ def trigger_side_effect(**kwargs: Any) -> int: exception_calls = [call.args[0] for call in mock_logger.exception.call_args_list] assert "night_shift.autofix_trigger_failed" in exception_calls - results = list(SeerNightShiftRunResult.objects.filter(run=run)) - assert len(results) == 1 - assert results[0].group_id == ok_group.id - assert results[0].seer_run_id == "7" + results = {r.group_id: r for r in SeerNightShiftRunResult.objects.filter(run=run)} + assert set(results) == {failing_group.id, ok_group.id} + assert results[ok_group.id].seer_run_id == "7" + assert "trigger_error" not in results[ok_group.id].extras + assert results[failing_group.id].seer_run_id is None + assert results[failing_group.id].extras["action"] == TriageAction.AUTOFIX.value + assert results[failing_group.id].extras["trigger_error"] is True def test_unknown_group_ids_logged(self) -> None: """Groups not belonging to the org should be logged and skipped.""" @@ -331,7 +348,7 @@ def test_unknown_group_ids_logged(self) -> None: } with ( - patch("sentry.tasks.seer.night_shift.cron.trigger_autofix_agent") as mock_trigger, + patch("sentry.seer.night_shift.delivery.trigger_autofix_agent") as mock_trigger, patch("sentry.seer.night_shift.delivery.logger") as mock_logger, ): deliver_night_shift_result( @@ -346,6 +363,8 @@ def test_unknown_group_ids_logged(self) -> None: warning_calls = [call.args[0] for call in mock_logger.warning.call_args_list] assert "night_shift.delivery.unknown_group_ids" in warning_calls + assert not SeerNightShiftRunResult.objects.filter(run=run).exists() + def test_user_context_passed_to_autofix(self) -> None: """Verdict reason should be passed as user_context to autofix.""" org = self.create_organization() @@ -364,7 +383,7 @@ def test_user_context_passed_to_autofix(self) -> None: } with patch( - "sentry.tasks.seer.night_shift.cron.trigger_autofix_agent", return_value=1 + "sentry.seer.night_shift.delivery.trigger_autofix_agent", return_value=1 ) as mock_trigger: deliver_night_shift_result( organization_id=org.id, @@ -391,7 +410,7 @@ def test_successful_delivery_clears_stale_error_message(self) -> None: ] } - with patch("sentry.tasks.seer.night_shift.cron.trigger_autofix_agent", return_value=1): + with patch("sentry.seer.night_shift.delivery.trigger_autofix_agent", return_value=1): deliver_night_shift_result( organization_id=org.id, run_uuid=self._run_uuid(run), @@ -403,6 +422,94 @@ def test_successful_delivery_clears_stale_error_message(self) -> None: shard.refresh_from_db() assert "error_message" not in shard.extras + def test_redelivery_is_idempotent(self) -> None: + """Redelivering the same shard result must not re-trigger autofix or + create duplicate rows.""" + org = self.create_organization() + project = self.create_project(organization=org) + group = self.create_group(project=project) + run = self._create_night_shift_run(organization=org) + + result = { + "verdicts": [ + {"group_id": group.id, "action": TriageAction.AUTOFIX.value, "reason": "fixable"} + ] + } + + with patch( + "sentry.seer.night_shift.delivery.trigger_autofix_agent", return_value=11 + ) as mock_trigger: + for _ in range(2): + deliver_night_shift_result( + organization_id=org.id, + run_uuid=self._run_uuid(run), + status="completed", + result=result, + error=None, + ) + + mock_trigger.assert_called_once() + + assert SeerNightShiftRunResult.objects.filter(run=run).count() == 1 + + def test_result_links_seer_run(self) -> None: + """When the SeerRun mirror row exists, the result row links it.""" + org = self.create_organization() + project = self.create_project(organization=org) + group = self.create_group(project=project) + run = self._create_night_shift_run(organization=org) + autofix_seer_run = self.create_seer_run(organization=org, seer_run_state_id=99) + + result = { + "verdicts": [ + {"group_id": group.id, "action": TriageAction.AUTOFIX.value, "reason": "fixable"} + ] + } + + with patch("sentry.seer.night_shift.delivery.trigger_autofix_agent", return_value=99): + deliver_night_shift_result( + organization_id=org.id, + run_uuid=self._run_uuid(run), + status="completed", + result=result, + error=None, + ) + + result_row = SeerNightShiftRunResult.objects.get(run=run) + assert result_row.seer_run_id == "99" + assert result_row.result_seer_run_id == autofix_seer_run.id + + def test_reason_truncated(self) -> None: + """Persisted reasons are capped at REASON_MAX_CHARS.""" + org = self.create_organization() + project = self.create_project(organization=org) + group = self.create_group(project=project) + run = self._create_night_shift_run(organization=org) + + result = { + "verdicts": [ + { + "group_id": group.id, + "action": TriageAction.SKIP.value, + "reason": "x" * (REASON_MAX_CHARS + 100), + } + ] + } + + deliver_night_shift_result( + organization_id=org.id, + run_uuid=self._run_uuid(run), + status="completed", + result=result, + error=None, + ) + + result_row = SeerNightShiftRunResult.objects.get(run=run) + assert result_row.extras["reason"] == "x" * REASON_MAX_CHARS + + redis = redis_clusters.get("default") + redis.delete(skip_cache_key(group.id)) + def test_empty_reason_no_user_context(self) -> None: """Empty reason should result in no user_context.""" org = self.create_organization() @@ -415,7 +522,7 @@ def test_empty_reason_no_user_context(self) -> None: } with patch( - "sentry.tasks.seer.night_shift.cron.trigger_autofix_agent", return_value=1 + "sentry.seer.night_shift.delivery.trigger_autofix_agent", return_value=1 ) as mock_trigger: deliver_night_shift_result( organization_id=org.id, diff --git a/tests/sentry/tasks/seer/test_night_shift.py b/tests/sentry/tasks/seer/test_night_shift.py index cec284ef92c7..efcba2ad89d7 100644 --- a/tests/sentry/tasks/seer/test_night_shift.py +++ b/tests/sentry/tasks/seer/test_night_shift.py @@ -615,7 +615,7 @@ def test_dispatches_candidates_to_seer_feature(self) -> None: with ( self.feature("organizations:gen-ai-features"), - patch("sentry.tasks.seer.night_shift.cron.trigger_autofix_agent") as mock_autofix, + patch("sentry.seer.night_shift.delivery.trigger_autofix_agent") as mock_autofix, ): run_night_shift_for_org(org.id) @@ -863,6 +863,7 @@ def test_extras_contain_options_and_target_project_ids(self) -> None: "extra_triage_instructions": "", }, "target_project_ids": [project.id], + "num_eligible_projects": 0, } def test_extras_contain_triggering_user_id_when_provided(self) -> None: