Skip to content

feat(seer): Record a night shift result row for every triage verdict #118976

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
trevor-e merged 6 commits into from
Jul 8, 2026
+235 −123
Merged

feat(seer): Record a night shift result row for every triage verdict #118976

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
835d807
feat(seer): Record a night shift result row for every triage verdict
trevor-e Jul 2, 2026
d9d940e
style(seer): Trim verbose comments in night shift verdict recording
trevor-e Jul 2, 2026
62c155a
ref(seer): Move unique constraint and dedupe to a follow-up PR
trevor-e Jul 7, 2026
cac8002
ref(seer): Collapse verdict filtering into a single pass
trevor-e Jul 7, 2026
f2539db
fix(seer): Rename shadowed result variable in night shift delivery test
trevor-e Jul 7, 2026
6497876
ref(seer): Inline the night shift autofix trigger loop
trevor-e Jul 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 100 additions & 28 deletions src/sentry/seer/night_shift/delivery.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,26 @@
from sentry.models.group import Group
from sentry.models.organization import Organization
from sentry.seer.agent.types import FeatureRunStatus
from sentry.seer.autofix.autofix_agent import AutofixStep, trigger_autofix_agent
from sentry.seer.autofix.constants import SeerAutomationSource
from sentry.seer.autofix.issue_summary import referrer_map
from sentry.seer.autofix.utils import AutofixStoppingPoint, bulk_read_preferences_from_sentry_db
from sentry.seer.models.night_shift import (
SeerNightShiftRun,
SeerNightShiftRunResult,
SeerNightShiftRunShard,
)
from sentry.seer.night_shift.models import TriageResponse
from sentry.tasks.seer.night_shift.models import TriageAction, TriageResult
from sentry.seer.models.run import SeerRun
from sentry.seer.models.workflow import SeerWorkflowStrategy
from sentry.seer.night_shift.models import TriageResponse, TriageVerdict
from sentry.tasks.seer.night_shift.models import TriageAction
from sentry.tasks.seer.night_shift.skip_cache import mark_skipped

logger = logging.getLogger(__name__)

# Verdict reasons are LLM-generated free text; cap what we persist per row.
REASON_MAX_CHARS = 2048


def deliver_night_shift_result(
organization_id: int,
Expand Down Expand Up @@ -102,10 +110,8 @@ def _process_verdicts(
dry_run: bool,
log_extra: Mapping[str, object],
) -> None:
"""Mark SKIPs, fire autofix for fixable verdicts, persist result rows."""
# Import here to avoid circular import
from sentry.tasks.seer.night_shift.cron import _run_autofix_for_candidates

"""Mark SKIPs, fire autofix for fixable verdicts, and persist one result row
per verdict (every action, dry runs included) for later analysis."""
group_ids = [v.group_id for v in triage_response.verdicts]
groups_by_id: dict[int, Group] = {
g.id: g
Expand All @@ -123,33 +129,45 @@ def _process_verdicts(
extra={**log_extra, "unknown_group_ids": unknown_group_ids},
)

# Groups this run already has a result row for (e.g. Seer redelivered a
# shard result): don't re-trigger autofix or write duplicate rows for them.
recorded_group_ids = set(
SeerNightShiftRunResult.objects.filter(run=run, group_id__in=group_ids).values_list(
"group_id", flat=True
)
)

# SKIP and ROOT_CAUSE_ONLY are both suppressed from future runs via the skip
# cache. ROOT_CAUSE_ONLY keeps its own action value for tracking, but is
# otherwise treated identically to SKIP (it does not trigger autofix).
verdicts: list[TriageVerdict] = []
fixable_groups: list[Group] = []
for v in triage_response.verdicts:
if (
v.action in (TriageAction.SKIP, TriageAction.ROOT_CAUSE_ONLY)
and v.group_id in groups_by_id
):
group = groups_by_id.get(v.group_id)
if group is None or v.group_id in recorded_group_ids:
continue
verdicts.append(v)
if v.action in (TriageAction.SKIP, TriageAction.ROOT_CAUSE_ONLY):
mark_skipped(v.group_id)
elif v.action == TriageAction.AUTOFIX:
fixable_groups.append(group)

# Convert verdicts to TriageResult objects for the shared function
fixable_candidates = [
TriageResult(group=groups_by_id[v.group_id], action=v.action, reason=v.reason)
for v in triage_response.verdicts
if v.action == TriageAction.AUTOFIX and v.group_id in groups_by_id
]

sentry_sdk.metrics.distribution("night_shift.candidates_selected", len(fixable_candidates))
sentry_sdk.metrics.distribution("night_shift.candidates_selected", len(fixable_groups))
if not fixable_groups:
logger.info(
"night_shift.no_fixable_candidates",
extra={**log_extra, "num_candidates": len(verdicts)},
)

results: list[SeerNightShiftRunResult] = []
if not dry_run and fixable_candidates:
reason_by_group_id = {v.group_id: v.reason for v in verdicts}
state_id_by_group: dict[int, int] = {}
if not dry_run and fixable_groups:
# Cache organization on each group's project to avoid N+1 queries
for group in groups_by_id.values():
group.project.organization = organization

# Build stopping_point_by_project_id from project preferences (bulk query)
project_ids = {c.group.project_id for c in fixable_candidates}
project_ids = {group.project_id for group in fixable_groups}
preferences = bulk_read_preferences_from_sentry_db(organization.id, list(project_ids))
default_stopping_point = AutofixStoppingPoint(SEER_AUTOMATED_RUN_STOPPING_POINT_DEFAULT)
stopping_point_by_project_id = {
Expand All @@ -161,25 +179,79 @@ def _process_verdicts(
for pid in project_ids
}

results = _run_autofix_for_candidates(
run=run,
candidates=fixable_candidates,
stopping_point_by_project_id=stopping_point_by_project_id,
log_extra=dict(log_extra),
referrer = referrer_map[SeerAutomationSource.NIGHT_SHIFT]
for group in fixable_groups:
reason = reason_by_group_id[group.id]
user_context = (
f"Night-shift triage already investigated this issue and concluded:\n{reason}"
if reason
else None
)
try:
state_id_by_group[group.id] = trigger_autofix_agent(
group=group,
step=AutofixStep.ROOT_CAUSE,
referrer=referrer,
stopping_point=stopping_point_by_project_id[group.project_id],
user_context=user_context,
)
except Exception:
logger.exception(
"night_shift.autofix_trigger_failed",
extra={**log_extra, "group_id": group.id},
)

sentry_sdk.metrics.count("night_shift.autofix_triggered", len(state_id_by_group))

# TODO: have trigger_autofix_agent return the SeerRun directly to avoid this lookup.
seer_run_by_state_id = {
sr.seer_run_state_id: sr
for sr in SeerRun.objects.filter(seer_run_state_id__in=state_id_by_group.values())
}

rows: list[SeerNightShiftRunResult] = []
for v in verdicts:
extras: dict[str, Any] = {"action": str(v.action)}
if v.reason:
extras["reason"] = v.reason[:REASON_MAX_CHARS]
seer_run_id: str | None = None
result_seer_run: SeerRun | None = None
if v.action == TriageAction.AUTOFIX and not dry_run:
state_id = state_id_by_group.get(v.group_id)
if state_id is None:
extras["trigger_error"] = True
else:
seer_run_id = str(state_id)
result_seer_run = seer_run_by_state_id.get(state_id)
rows.append(
SeerNightShiftRunResult(
run=run,
kind=SeerWorkflowStrategy.AGENTIC_TRIAGE,
group=groups_by_id[v.group_id],
seer_run_id=seer_run_id,
result_seer_run=result_seer_run,
extras=extras,
)
)
# ignore_conflicts: concurrent redeliveries can race past the recorded-rows check.
SeerNightShiftRunResult.objects.bulk_create(rows, ignore_conflicts=True)

@sentry sentry Bot Jul 7, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Bug: bulk_create with ignore_conflicts=True is used without a database unique constraint, allowing concurrent redeliveries to create duplicate rows due to a race condition.
Severity: MEDIUM

Suggested Fix

Deploy the database migration that adds the UniqueConstraint on (run, group) to the SeerNightShiftRunResult model. This will make the ignore_conflicts=True parameter effective and prevent duplicate row creation at the database level, resolving the race condition.

Prompt for AI Agent 
Review the code at the location below. A potential bug has been identified by an AI
agent. Verify if this is a real issue. If it is, propose a fix; if not, explain why it's
not valid.

Location: src/sentry/seer/night_shift/delivery.py#L237

Potential issue: The code at `delivery.py:237` calls `bulk_create` with
`ignore_conflicts=True`, but the corresponding `(run, group)` unique constraint on the
`SeerNightShiftRunResult` model does not exist in the database yet. Without this
constraint, `ignore_conflicts=True` has no effect. An idempotency check on lines 134-138
is insufficient to prevent duplicates under concurrent redeliveries. A
time-of-check/time-of-use (TOCTOU) race condition exists where two workers can both
check for existing rows (finding none), and then both proceed to insert the same data,
resulting in duplicate `SeerNightShiftRunResult` rows.

Did we get this right? 👍 / 👎 to inform future reviews.

@trevor-e trevor-e Jul 8, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixing in a stacked PR follow-up


seer_run_id_by_group = {r.group_id: r.seer_run_id for r in results}
logger.info(
"night_shift.candidates_selected",
extra={
**log_extra,
"num_verdicts": len(triage_response.verdicts),
"num_already_recorded": len(recorded_group_ids),
"dry_run": dry_run,
"candidates": [
{
"group_id": v.group_id,
"action": v.action,
"seer_run_id": seer_run_id_by_group.get(v.group_id),
"seer_run_id": (
str(state_id)
if (state_id := state_id_by_group.get(v.group_id)) is not None
else None
),
}
for v in triage_response.verdicts
],
Expand Down
74 changes: 3 additions & 71 deletions src/sentry/tasks/seer/night_shift/cron.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,25 +21,20 @@
from sentry.models.organization import Organization, OrganizationStatus
from sentry.models.project import Project
from sentry.seer.agent.client import SeerAgentClient
from sentry.seer.autofix.autofix_agent import AutofixStep, trigger_autofix_agent
from sentry.seer.autofix.constants import (
AutofixAutomationTuningSettings,
SeerAutomationSource,
)
from sentry.seer.autofix.issue_summary import referrer_map
from sentry.seer.autofix.utils import AutofixStoppingPoint, bulk_read_preferences_from_sentry_db
from sentry.seer.models import SeerPermissionError
from sentry.seer.models.night_shift import (
SeerNightShiftRun,
SeerNightShiftRunResult,
SeerNightShiftRunShard,
)
from sentry.seer.models.project_repository import SeerProjectRepository
from sentry.seer.models.run import SeerRun
from sentry.seer.models.workflow import SeerWorkflowConfig, SeerWorkflowStrategy
from sentry.seer.night_shift.models import NightShiftPayload, TriageCandidate, TriageTweaks
from sentry.tasks.base import instrumented_task
from sentry.tasks.seer.night_shift.models import TriageAction, TriageResult
from sentry.tasks.seer.night_shift.simple_triage import (
ScoredCandidate,
fixability_score_strategy,
Expand Down Expand Up @@ -313,6 +308,8 @@ def run_night_shift_execution(
return None

sentry_sdk.metrics.distribution("night_shift.eligible_projects", len(eligible))
# Stamped so zero-shard runs are distinguishable: no eligible projects vs. no candidates.
run.update(extras={**(run.extras or {}), "num_eligible_projects": len(eligible)})

if not eligible:
logger.info("night_shift.no_eligible_projects", extra=log_extra)
Expand Down Expand Up @@ -560,6 +557,7 @@ def _dispatch_to_seer_feature(
eligible_projects = [ep.project for ep in eligible]
repos_by_project = {ep.project.id: ep.connected_repos for ep in eligible}
scored = fixability_score_strategy(eligible_projects, resolved_options["max_candidates"])
run.update(extras={**(run.extras or {}), "num_candidates": len(scored)})
if not scored:
logger.info("night_shift.no_candidates", extra=log_extra)
return
Expand Down Expand Up @@ -620,69 +618,3 @@ def _link_shard(created: SeerRun) -> None:
"num_shards_dispatched": dispatched,
},
)


def _run_autofix_for_candidates(
run: SeerNightShiftRun,
candidates: Sequence[TriageResult],
stopping_point_by_project_id: Mapping[int, AutofixStoppingPoint],
log_extra: dict[str, object],
) -> list[SeerNightShiftRunResult]:
"""
For each fixable triage candidate, trigger a Seer autofix run and persist
the resulting run id onto a newly created SeerNightShiftRunResult row.
Returns the list of rows that were created.
"""
fixable_candidates = [c for c in candidates if c.action == TriageAction.AUTOFIX]
if not fixable_candidates:
logger.info(
"night_shift.no_fixable_candidates",
extra={**log_extra, "num_candidates": len(candidates)},
)
return []

referrer = referrer_map[SeerAutomationSource.NIGHT_SHIFT]

results: list[SeerNightShiftRunResult] = []
for c in fixable_candidates:
stopping_point = stopping_point_by_project_id[c.group.project_id]

user_context = (
f"Night-shift triage already investigated this issue and concluded:\n{c.reason}"
if c.reason
else None
)

try:
seer_run_id = trigger_autofix_agent(
group=c.group,
step=AutofixStep.ROOT_CAUSE,
referrer=referrer,
stopping_point=stopping_point,
user_context=user_context,
)
except Exception:
logger.exception(
"night_shift.autofix_trigger_failed",
extra={**log_extra, "group_id": c.group.id},
)
continue

# TODO: have trigger_autofix_agent return the SeerRun directly to avoid this lookup.
result_seer_run = SeerRun.objects.filter(seer_run_state_id=seer_run_id).first()
results.append(
SeerNightShiftRunResult(
run=run,
kind=SeerWorkflowStrategy.AGENTIC_TRIAGE,
group=c.group,
seer_run_id=str(seer_run_id),
result_seer_run=result_seer_run,
extras={"action": str(c.action)},
)
)

SeerNightShiftRunResult.objects.bulk_create(results)

sentry_sdk.metrics.count("night_shift.autofix_triggered", len(results))

return results
Loading
Loading