ccx-incident-145

I'll search for the Grafana alerting evaluation scheduler and overrun detection code.

{"matches": ["mcp__sourcegraph__sg_keyword_search", "mcp__sourcegraph__sg_nls_search"], "query": "select:mcp__sourcegraph__sg_keyword_search,mcp__sourcegraph__sg_nls_search", "total_deferred_tools": 33}

{"text":"# github.com/sg-evals/grafana--26d36ec – packages/grafana-alerting/src/grafana/api/rules/v0alpha1/rules.api.gen.ts\n502:   fieldValidation?: string;\n503:   alertRule: AlertRule;\n504: };\n505: export type UpdateAlertRuleStatusApiResponse = /** status 200 OK */ AlertRule | /** status 201 Created */ AlertRule;\n506: export type UpdateAlertRuleStatusApiArg = {\n507:   /** name of the AlertRule */\n508:   name: string;\n509: \n448:   fieldValidation?: string;\n449:   alertRule: AlertRule;\n450: };\n451: export type DeleteAlertRuleApiResponse = /** status 200 OK */ Status | /** status 202 Accepted */ Status;\n452: export type DeleteAlertRuleApiArg = {\n453:   /** name of the AlertRule */\n454:   name: string;\n455: \n376:   fieldValidation?: string;\n377:   alertRule: AlertRule;\n378: };\n379: export type DeletecollectionAlertRuleApiResponse = /** status 200 OK */ Status;\n380: export type DeletecollectionAlertRuleApiArg = {\n381:   /** If 'true', then the output is pretty printed. Defaults to 'false' unless the user-agent indicates a browser or command-line HTTP tool (curl and wget). */\n382: \n\n# github.com/sg-evals/grafana--26d36ec – pkg/tests/apis/openapi_snapshots/rules.alerting.grafana.app-v0alpha1.json\n2280:     \"schemas\": {\n2281:       \"com.github.grafana.grafana.apps.alerting.rules.pkg.apis.alerting.v0alpha1.AlertRule\": {\n2282:         \"type\": \"object\",\n2283: \n2369:       },\n2370:       \"com.github.grafana.grafana.apps.alerting.rules.pkg.apis.alerting.v0alpha1.AlertRuleList\": {\n2371:         \"type\": \"object\",\n2372: \n2469:       },\n2470:       \"com.github.grafana.grafana.apps.alerting.rules.pkg.apis.alerting.v0alpha1.AlertRuleSpec\": {\n2471:         \"type\": \"object\",\n2472: \n\n# github.com/sg-evals/grafana--26d36ec – pkg/test...

{"text":"No results.\n\nNo results found."}

Let me search for the specific log message and alerting scheduler patterns.

{"text":"# github.com/sg-evals/grafana--26d36ec – pkg/services/ngalert/writer/prom.go\n126: \n127: \t// ExpectedErrors are user-level write errors like trying to write an invalid series.\n128: \tExpectedErrors = []string{\n129: \t\tMimirDistributorMaxWriteMessageSizeError,\n130: \n74: \t// https://github.com/grafana/mimir/blob/1ff367ef58987cd1941de03a8d6923fde82dfdd3/pkg/distributor/validate.go#L491-L491\n75: \tMimirMetricMetadataMetricNameTooLongError = \"err-mimir-metric-name-too-long\"\n76: \tMimirMetricMetadataMissingMetricNameError = \"err-mimir-metadata-missing-metric-name\"\n77: \tMimirMetricMetadataUnitTooLongError       = \"err-mimir-unit-too-long\"\n78: \n79: \n111: var (\n112: \t// Unexpected, 500-like write errors.\n113: \tErrUnexpectedWriteFailure = errors.New(\"failed to write time series\")\n114: \t// Expected, user-level write errors like trying to write an invalid series.\n115: \tErrRejectedWrite          = errors.New(\"series was rejected\")\n116: \n\n# github.com/sg-evals/grafana--26d36ec – pkg/services/ngalert/api/util.go\n79: \tdefault:\n80: \t\treturn nil, unexpectedDatasourceTypeError(ds.Type, expectedType.String())\n81: \t}\n82: \n69: \t}\n70: \tswitch expectedType {\n71: \tcase apimodels.AlertmanagerBackend:\n72: \n203: \t\tif !strings.Contains(contentType, \"yaml\") {\n204: \t\t\treturn nil, fmt.Errorf(\"unexpected content type from upstream. expected YAML, got %v\", contentType)\n205: \t\t}\n206: \n\n# github.com/sg-evals/grafana--26d36ec – pkg/expr/sql_command.go\n268: //   - No numeric field is found.\n269: //   - More than one numeric field exists.\n270: //   - Any label set appears more than once.\n271: func extractNumberSetFromSQLForAlerting(frame *data.Frame) ([]mathexp.Number, error) {\n272: \n446: \n447: \t// If meta.type is not suppor...

{"text":"No results.\n\nNo results found."}

{"text":"# github.com/sg-evals/grafana--26d36ec – pkg/services/ngalert/metrics/scheduler.go\n6: \n7: \t\"github.com/grafana/grafana/pkg/services/ngalert/schedule/ticker\"\n8: )\n9: \n181: \t\t\t\tSubsystem: Subsystem,\n182: \t\t\t\tName:      \"schedule_rule_evaluations_missed_total\",\n183: \t\t\t\tHelp:      \"The total number of rule evaluations missed due to a slow rule evaluation.\",\n184: \t\t\t},\n185: \n14: \n15: type Scheduler struct {\n16: \tRegisterer                          prometheus.Registerer\n17: \n\n# github.com/sg-evals/grafana--26d36ec – pkg/services/ngalert/schedule/schedule.go\n16: \t\"github.com/grafana/grafana/pkg/services/featuremgmt\"\n17: \t\"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions\"\n18: \t\"github.com/grafana/grafana/pkg/services/ngalert/eval\"\n19: \t\"github.com/grafana/grafana/pkg/services/ngalert/metrics\"\n20: \tngmodels \"github.com/grafana/grafana/pkg/services/ngalert/models\"\n21: \t\"github.com/grafana/grafana/pkg/services/ngalert/schedule/ticker\"\n22: \t\"github.com/grafana/grafana/pkg/services/ngalert/state\"\n23: \t\"github.com/grafana/grafana/pkg/setting\"\n24: \n245: \n246: func (sch *schedule) schedulePeriodic(ctx context.Context, t *ticker.T) error {\n247: \tdispatcherGroup, ctx := errgroup.WithContext(ctx)\n248: \n25: \n26: // ScheduleService is an interface for a service that schedules the evaluation\n27: // of alert rules.\n28: type ScheduleService interface {\n29: \t// Run the scheduler until the context is canceled or the scheduler returns\n30: \t// an error. The scheduler is terminated when this function returns.\n31: \tRun(context.Context) error\n32: \n\n# github.com/sg-evals/grafana--26d36ec – pkg/services/ngalert/ngalert.go\n342: \t\tTracer:               ng.tracer,\n343: \t\tLog:    ...

{"text":"# github.com/sg-evals/grafana--26d36ec – pkg/services/ngalert/schedule/schedule.go\n437: \t\tif dropped != nil {\n438: \t\t\tsch.log.Warn(\"Tick dropped because alert rule evaluation is too slow\", append(key.LogContext(), \"time\", next.scheduledAt, \"droppedTick\", dropped.scheduledAt)...)\n439: \t\t\torgID := fmt.Sprint(key.OrgID)\n440: \t\t\tsch.metrics.EvaluationMissed.WithLabelValues(orgID, next.rule.Title).Inc()\n441: \t\t}\n442: \n274: // TODO refactor to accept a callback for tests that will be called with things that are returned currently, and return nothing.\n275: // Returns a slice of rules that were scheduled for evaluation, map of stopped rules, and a slice of updated rules\n276: func (sch *schedule) processTick(ctx context.Context, dispatcherGroup *errgroup.Group, tick time.Time) ([]readyToRunItem, map[ngmodels.AlertRuleKey]struct{}, []ngmodels.AlertRuleKeyWithVersion) {\n277: \ttickNum := tick.Unix() / int64(sch.baseInterval.Seconds())\n278: \n279: \n95: \n96: \tmetrics *metrics.Scheduler\n97: \n98: \talertsSender    AlertsSender\n99: \tminRuleInterval time.Duration\n100: \n101: \t// schedulableAlertRules contains the alert rules that are considered for\n102: \t// evaluation in the current tick. The evaluation of an alert rule in the\n103: \t// current tick depends on its evaluation interval and when it was\n104: \t// last evaluated.\n105: \tschedulableAlertRules alertRulesRegistry\n106: \n107: \n\n# github.com/sg-evals/grafana--26d36ec – pkg/services/ngalert/metrics/scheduler.go\n27: \tGroups                              *prometheus.GaugeVec\n28: \tSchedulePeriodicDuration            prometheus.Histogram\n29: \tSchedulableAlertRules               prometheus.Gauge\n30: \tSchedulableAlertRulesHash           prometheus.Gauge\n31: \tUpdateSchedula...

I found the relevant schedule files. Let me read the key files in detail.

{"matches": ["mcp__sourcegraph__sg_read_file"], "query": "select:mcp__sourcegraph__sg_read_file", "total_deferred_tools": 33}

{"text":"URL: https://demo.sourcegraph.com/github.com/sg-evals/grafana--26d36ec@7b5876467dc156dd3117dd0b3c15f994100e9597/-/blob/pkg/services/ngalert/schedule/schedule.go\n1: package schedule\n2: \n3: import (\n4: \t\"context\"\n5: \t\"fmt\"\n6: \t\"net/url\"\n7: \t\"time\"\n8: \n9: \t\"github.com/benbjohnson/clock\"\n10: \t\"golang.org/x/sync/errgroup\"\n11: \n12: \t\"github.com/grafana/grafana-plugin-sdk-go/data\"\n13: \n14: \t\"github.com/grafana/grafana/pkg/infra/log\"\n15: \t\"github.com/grafana/grafana/pkg/infra/tracing\"\n16: \t\"github.com/grafana/grafana/pkg/services/featuremgmt\"\n17: \t\"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions\"\n18: \t\"github.com/grafana/grafana/pkg/services/ngalert/eval\"\n19: \t\"github.com/grafana/grafana/pkg/services/ngalert/metrics\"\n20: \tngmodels \"github.com/grafana/grafana/pkg/services/ngalert/models\"\n21: \t\"github.com/grafana/grafana/pkg/services/ngalert/schedule/ticker\"\n22: \t\"github.com/grafana/grafana/pkg/services/ngalert/state\"\n23: \t\"github.com/grafana/grafana/pkg/setting\"\n24: )\n25: \n26: // ScheduleService is an interface for a service that schedules the evaluation\n27: // of alert rules.\n28: type ScheduleService interface {\n29: \t// Run the scheduler until the context is canceled or the scheduler returns\n30: \t// an error. The scheduler is terminated when this function returns.\n31: \tRun(context.Context) error\n32: }\n33: \n34: // AlertsSender is an interface for a service that is responsible for sending notifications to the end-user.\n35: //\n36: //go:generate mockery --name AlertsSender --structname AlertsSenderMock --inpackage --filename alerts_sender_mock.go --with-expecter\n37: type AlertsSender interface {\n38: \tSend(ctx context.Context, key ngmodels.AlertRuleKey, alert...

{"text":"URL: https://demo.sourcegraph.com/github.com/sg-evals/grafana--26d36ec@7b5876467dc156dd3117dd0b3c15f994100e9597/-/blob/pkg/services/ngalert/schedule/alert_rule.go\n1: package schedule\n2: \n3: import (\n4: \t\"context\"\n5: \t\"errors\"\n6: \t\"fmt\"\n7: \t\"net/url\"\n8: \t\"time\"\n9: \n10: \t\"github.com/benbjohnson/clock\"\n11: \t\"github.com/prometheus/alertmanager/api/v2/models\"\n12: \t\"go.opentelemetry.io/otel/attribute\"\n13: \t\"go.opentelemetry.io/otel/codes\"\n14: \t\"go.opentelemetry.io/otel/trace\"\n15: \n16: \t\"github.com/grafana/grafana/pkg/infra/log\"\n17: \t\"github.com/grafana/grafana/pkg/infra/tracing\"\n18: \t\"github.com/grafana/grafana/pkg/services/datasources\"\n19: \t\"github.com/grafana/grafana/pkg/services/featuremgmt\"\n20: \t\"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions\"\n21: \t\"github.com/grafana/grafana/pkg/services/ngalert/eval\"\n22: \t\"github.com/grafana/grafana/pkg/services/ngalert/metrics\"\n23: \tngmodels \"github.com/grafana/grafana/pkg/services/ngalert/models\"\n24: \t\"github.com/grafana/grafana/pkg/services/ngalert/state\"\n25: \t\"github.com/grafana/grafana/pkg/services/org\"\n26: \t\"github.com/grafana/grafana/pkg/services/user\"\n27: \t\"github.com/grafana/grafana/pkg/setting\"\n28: \t\"github.com/grafana/grafana/pkg/util\"\n29: )\n30: \n31: // Rule represents a single piece of work that is executed periodically by the ruler.\n32: type Rule interface {\n33: \t// Run creates the resources that will perform the rule's work, and starts it. It blocks indefinitely, until Stop is called or another signal is sent.\n34: \tRun() error\n35: \t// Stop shuts down the rule's execution with an optional reason. It has no effect if the rule has not yet been Run.\n36: \tStop(reason error)\n37: \t// Eva...

Now let me look at the ticker package which is part of the scheduler's time-tracking mechanism.

{"text":"# github.com/sg-evals/grafana--26d36ec – pkg/services/ngalert/metrics/scheduler.go\n31: \tUpdateSchedulableAlertRulesDuration prometheus.Histogram\n32: \tTicker                              *ticker.Metrics\n33: \tEvaluationMissed                    *prometheus.CounterVec\n34: \n16: \tRegisterer                          prometheus.Registerer\n17: \tBehindSeconds                       prometheus.Gauge\n18: \tEvalTotal                           *prometheus.CounterVec\n19: \n176: \t\t),\n177: \t\tTicker: ticker.NewMetrics(r, \"alerting\"),\n178: \t\tEvaluationMissed: promauto.With(r).NewCounterVec(\n179: \n\n# github.com/sg-evals/grafana--26d36ec – pkg/services/ngalert/schedule/ticker/metrics.go\n1: package ticker\n2: \n3: \n30: \t\t\tSubsystem: subsystem,\n31: \t\t\tName:      \"ticker_interval_seconds\",\n32: \t\t\tHelp:      \"Interval at which the ticker is meant to tick.\",\n33: \t\t}),\n34: \n24: \t\t\tSubsystem: subsystem,\n25: \t\t\tName:      \"ticker_next_tick_timestamp_seconds\",\n26: \t\t\tHelp:      \"Timestamp of the next tick in seconds before it is consumed.\",\n27: \n\n# github.com/sg-evals/grafana--26d36ec – pkg/plugins/manager/process/process.go\n12: type Service struct {\n13: \tkeepPluginAliveTickerDuration time.Duration\n14: }\n15: \n9: \n10: const defaultKeepPluginAliveTickerDuration = time.Second\n11: \n12: \n67: func (s *Service) keepPluginAlive(p *plugins.Plugin) error {\n68: \tticker := time.NewTicker(s.keepPluginAliveTickerDuration)\n69: \n70: \n\n# github.com/sg-evals/grafana--26d36ec – pkg/util/retryer/retryer.go\n19: \tcurrentDelay := minDelay\n20: \tvar ticker *time.Ticker\n21: \n22: \n36: \n37: \t\tif ticker == nil {\n38: \t\t\tticker = time.NewTicker(currentDelay)\n39: \t\t\tdefer ticker.Stop()\n40: \t\t} else {\n41: \n44: \n45: \t\...

{"text":"URL: https://demo.sourcegraph.com/github.com/sg-evals/grafana--26d36ec@7b5876467dc156dd3117dd0b3c15f994100e9597/-/blob/pkg/services/ngalert/schedule/ticker\n1: 100644 blob 9c8a60e385042e6990bce11ed09eb7cc66372a1c\tmetrics.go\n2: 100644 blob a52b9c4e5598362de287ff26bc1f036a9cce460c\tticker.go\n3: 100644 blob a8eb28afd772faa4bfe1b6b8a647f048613435e2\tticker_test.go\n4: "}

{"text":"URL: https://demo.sourcegraph.com/github.com/sg-evals/grafana--26d36ec@7b5876467dc156dd3117dd0b3c15f994100e9597/-/blob/pkg/services/ngalert/schedule/ticker/ticker.go\n1: package ticker\n2: \n3: import (\n4: \t\"fmt\"\n5: \t\"time\"\n6: \n7: \t\"github.com/benbjohnson/clock\"\n8: \n9: \t\"github.com/grafana/grafana/pkg/infra/log\"\n10: )\n11: \n12: // Ticker emits ticks at regular time intervals. it's like a time.Ticker, except:\n13: //   - it doesn't drop ticks for slow receivers, rather, it queues up.  so that callers are in control to instrument what's going on.\n14: //   - it ticks on interval marks or very shortly after. this provides a predictable load pattern\n15: //     (this shouldn't cause too much load contention issues because the next steps in the pipeline just process at their own pace)\n16: //   - the timestamps are used to mark \"last datapoint to query for\" and as such, are a configurable amount of seconds in the past\n17: type T struct {\n18: \tC        chan time.Time\n19: \tclock    clock.Clock\n20: \tlast     time.Time\n21: \tinterval time.Duration\n22: \tmetrics  *Metrics\n23: \tstopCh   chan struct{}\n24: \tlogger   log.Logger\n25: }\n26: \n27: // NewTicker returns a Ticker that ticks on interval marks (or very shortly after) starting at c.Now(), and never drops ticks. interval should not be negative or zero.\n28: func New(c clock.Clock, interval time.Duration, metric *Metrics, logger log.Logger) *T {\n29: \tif interval <= 0 {\n30: \t\tpanic(fmt.Errorf(\"non-positive interval [%v] is not allowed\", interval))\n31: \t}\n32: \tt := &T{\n33: \t\tC:        make(chan time.Time),\n34: \t\tclock:    c,\n35: \t\tlast:     getStartTick(c, interval),\n36: \t\tinterval: interval,\n37: \t\tmetrics:  metric,\n38: \t\tstopCh:   make(chan struct{}),\n39:...

{"text":"URL: https://demo.sourcegraph.com/github.com/sg-evals/grafana--26d36ec@7b5876467dc156dd3117dd0b3c15f994100e9597/-/blob/pkg/services/ngalert/schedule/recording_rule.go\n1: package schedule\n2: \n3: import (\n4: \tcontext \"context\"\n5: \t\"fmt\"\n6: \t\"time\"\n7: \n8: \t\"github.com/benbjohnson/clock\"\n9: \t\"github.com/grafana/grafana-plugin-sdk-go/backend\"\n10: \t\"github.com/grafana/grafana-plugin-sdk-go/data\"\n11: \t\"go.opentelemetry.io/otel/attribute\"\n12: \t\"go.opentelemetry.io/otel/codes\"\n13: \t\"go.opentelemetry.io/otel/trace\"\n14: \t\"go.uber.org/atomic\"\n15: \n16: \t\"github.com/grafana/grafana/pkg/infra/log\"\n17: \t\"github.com/grafana/grafana/pkg/infra/tracing\"\n18: \t\"github.com/grafana/grafana/pkg/services/ngalert/eval\"\n19: \t\"github.com/grafana/grafana/pkg/services/ngalert/metrics\"\n20: \tngmodels \"github.com/grafana/grafana/pkg/services/ngalert/models\"\n21: \t\"github.com/grafana/grafana/pkg/setting\"\n22: \t\"github.com/grafana/grafana/pkg/util\"\n23: )\n24: \n25: type RuleStatus struct {\n26: \tHealth              string\n27: \tLastError           error\n28: \tEvaluationTimestamp time.Time\n29: \tEvaluationDuration  time.Duration\n30: }\n31: \n32: type recordingRule struct {\n33: \tkey ngmodels.AlertRuleKeyWithGroup\n34: \n35: \tctx                 context.Context\n36: \tevalCh              chan *Evaluation\n37: \tstopFn              util.CancelCauseFunc\n38: \thealth              *atomic.String\n39: \tlastError           *atomic.Error\n40: \tevaluationTimestamp *atomic.Time\n41: \tevaluationDuration  *atomic.Duration\n42: \n43: \tretryConfig RetryConfig\n44: \n45: \tclock       clock.Clock\n46: \tevalFactory eval.EvaluatorFactory\n47: \tcfg         setting.RecordingRuleSettings\n48: \twriter      RecordingWriter\n49: \n50:...

Let me also check the metrics file to complete the picture.

{"text":"URL: https://demo.sourcegraph.com/github.com/sg-evals/grafana--26d36ec@7b5876467dc156dd3117dd0b3c15f994100e9597/-/blob/pkg/services/ngalert/metrics/scheduler.go\n1: package metrics\n2: \n3: import (\n4: \t\"github.com/prometheus/client_golang/prometheus\"\n5: \t\"github.com/prometheus/client_golang/prometheus/promauto\"\n6: \n7: \t\"github.com/grafana/grafana/pkg/services/ngalert/schedule/ticker\"\n8: )\n9: \n10: const (\n11: \tAlertRuleActiveLabelValue = \"active\"\n12: \tAlertRulePausedLabelValue = \"paused\"\n13: )\n14: \n15: type Scheduler struct {\n16: \tRegisterer                          prometheus.Registerer\n17: \tBehindSeconds                       prometheus.Gauge\n18: \tEvalTotal                           *prometheus.CounterVec\n19: \tEvalFailures                        *prometheus.CounterVec\n20: \tEvalDuration                        *prometheus.HistogramVec\n21: \tEvalAttemptTotal                    *prometheus.CounterVec\n22: \tEvalAttemptFailures                 *prometheus.CounterVec\n23: \tProcessDuration                     *prometheus.HistogramVec\n24: \tSendDuration                        *prometheus.HistogramVec\n25: \tSimpleNotificationRules             *prometheus.GaugeVec\n26: \tGroupRules                          *prometheus.GaugeVec\n27: \tGroups                              *prometheus.GaugeVec\n28: \tSchedulePeriodicDuration            prometheus.Histogram\n29: \tSchedulableAlertRules               prometheus.Gauge\n30: \tSchedulableAlertRulesHash           prometheus.Gauge\n31: \tUpdateSchedulableAlertRulesDuration prometheus.Histogram\n32: \tTicker                              *ticker.Metrics\n33: \tEvaluationMissed                    *prometheus.CounterVec\n34: \tSimplifiedEditorRules               *prometheus.GaugeVec\n35: \tPro...

{"text":"URL: https://demo.sourcegraph.com/github.com/sg-evals/grafana--26d36ec@7b5876467dc156dd3117dd0b3c15f994100e9597/-/blob/pkg/services/ngalert/schedule/ticker/metrics.go\n1: package ticker\n2: \n3: import (\n4: \t\"github.com/prometheus/client_golang/prometheus\"\n5: \t\"github.com/prometheus/client_golang/prometheus/promauto\"\n6: )\n7: \n8: type Metrics struct {\n9: \tLastTickTime    prometheus.Gauge\n10: \tNextTickTime    prometheus.Gauge\n11: \tIntervalSeconds prometheus.Gauge\n12: }\n13: \n14: func NewMetrics(reg prometheus.Registerer, subsystem string) *Metrics {\n15: \treturn &Metrics{\n16: \t\tLastTickTime: promauto.With(reg).NewGauge(prometheus.GaugeOpts{\n17: \t\t\tNamespace: \"grafana\",\n18: \t\t\tSubsystem: subsystem,\n19: \t\t\tName:      \"ticker_last_consumed_tick_timestamp_seconds\",\n20: \t\t\tHelp:      \"Timestamp of the last consumed tick in seconds.\",\n21: \t\t}),\n22: \t\tNextTickTime: promauto.With(reg).NewGauge(prometheus.GaugeOpts{\n23: \t\t\tNamespace: \"grafana\",\n24: \t\t\tSubsystem: subsystem,\n25: \t\t\tName:      \"ticker_next_tick_timestamp_seconds\",\n26: \t\t\tHelp:      \"Timestamp of the next tick in seconds before it is consumed.\",\n27: \t\t}),\n28: \t\tIntervalSeconds: promauto.With(reg).NewGauge(prometheus.GaugeOpts{\n29: \t\t\tNamespace: \"grafana\",\n30: \t\t\tSubsystem: subsystem,\n31: \t\t\tName:      \"ticker_interval_seconds\",\n32: \t\t\tHelp:      \"Interval at which the ticker is meant to tick.\",\n33: \t\t}),\n34: \t}\n35: }\n36: "}

I now have a comprehensive understanding of the alerting evaluation scheduler and overrun detection. Let me write the answer.

File created successfully at: /workspace/answer.json (file state is current in your context — no need to Read it back)

The answer is complete. Here's a summary of findings:

## Grafana Alert Evaluation Cycle Overrun — Key Files

The overrun detection spans these files in `pkg/services/ngalert/schedule/`:

**Scheduler tick implementation:**
- **`schedule/ticker/ticker.go`** — Custom `T` ticker that queues ticks without dropping them (unlike `time.Ticker`), giving callers full control over overrun measurement.
- **`schedule/schedule.go`** — The core scheduler:
  - `schedulePeriodic()` — main loop; reads ticks and sets `BehindSeconds` metric (`start.Sub(tick).Seconds()`)
  - `processTick()` — per-tick handler that identifies rules due to run
  - `runJobFn()` — **logs the overrun**: emits `"Tick dropped because alert rule evaluation is too slow"` (Warn) and increments `EvaluationMissed` counter when `Eval()` returns a dropped tick

**Overrun detection mechanism:**
- **`schedule/alert_rule.go`** — `alertRule.Eval()` does a **non-blocking drain** of the unbuffered `evalCh` before sending. If a previous evaluation is still pending (rule hasn't consumed the last tick yet), that pending message is captured as `droppedMsg` and returned to `runJobFn`.
- **`schedule/recording_rule.go`** — `recordingRule.Eval()` has identical logic for recording rules.

**Metrics:**
- **`metrics/scheduler.go`** — Defines `BehindSeconds`, `EvaluationMissed`, and `SchedulePeriodicDuration`; **`schedule/ticker/metrics.go`** defines per-tick timing gauges.

I'll look at the workspace structure first to find the grafana repo.

grafana--26d36ec

AGENTS.md
CHANGELOG.md
CODE_OF_CONDUCT.md
CONTRIBUTING.md
Dockerfile
GOVERNANCE.md
HALL_OF_FAME.md
LICENSE
LICENSING.md
MAINTAINERS.md
Makefile
NOTICE.md
README.md
ROADMAP.md
SECURITY.md
SUPPORT.md
WORKFLOW.md
apps
build.go
conf
contribute
crowdin.yml
cue.mod
cypress.config.js
devenv
docs
e2e
e2e-playwright
emails
embed.go
alert_rule.go
alert_rule_test.go
alerts_sender_mock.go
fetcher.go
jitter.go
jitter_test.go
loaded_metrics_reader.go
loaded_metrics_reader_test.go
metrics.go
metrics_test.go
recording_rule.go
recording_rule_test.go
registry.go
registry_bench_test.go
registry_test.go
retry.go
retry_test.go
schedule.go
schedule_unit_test.go
sequence.go
sequence_test.go
testing.go
ticker

(Bash completed with no output)

metrics.go
ticker.go
ticker_test.go

/workspace/grafana--26d36ec/pkg/services/ngalert/schedule/schedule.go:256:			sch.metrics.BehindSeconds.Set(start.Sub(tick).Seconds())
/workspace/grafana--26d36ec/pkg/services/ngalert/schedule/ticker/ticker_test.go:102:	t.Run("should put the tick in the channel immediately if it is behind", func(t *testing.T) {

1	package ticker
2	
3	import (
4		"fmt"
5		"time"
6	
7		"github.com/benbjohnson/clock"
8	
9		"github.com/grafana/grafana/pkg/infra/log"
10	)
11	
12	// Ticker emits ticks at regular time intervals. it's like a time.Ticker, except:
13	//   - it doesn't drop ticks for slow receivers, rather, it queues up.  so that callers are in control to instrument what's going on.
14	//   - it ticks on interval marks or very shortly after. this provides a predictable load pattern
15	//     (this shouldn't cause too much load contention issues because the next steps in the pipeline just process at their own pace)
16	//   - the timestamps are used to mark "last datapoint to query for" and as such, are a configurable amount of seconds in the past
17	type T struct {
18		C        chan time.Time
19		clock    clock.Clock
20		last     time.Time
21		interval time.Duration
22		metrics  *Metrics
23		stopCh   chan struct{}
24		logger   log.Logger
25	}
26	
27	// NewTicker returns a Ticker that ticks on interval marks (or very shortly after) starting at c.Now(), and never drops ticks. interval should not be negative or zero.
28	func New(c clock.Clock, interval time.Duration, metric *Metrics, logger log.Logger) *T {
29		if interval <= 0 {
30			panic(fmt.Errorf("non-positive interval [%v] is not allowed", interval))
31		}
32		t := &T{
33			C:        make(chan time.Time),
34			clock:    c,
35			last:     getStartTick(c, interval),
36			interval: interval,
37			metrics:  metric,
38			stopCh:   make(chan struct{}),
39			logger:   logger,
40		}
41		metric.IntervalSeconds.Set(t.interval.Seconds()) // Seconds report fractional part as well, so it matches the format of the timestamp we report below
42		go t.run()
43		return t
44	}
45	
46	func getStartTick(clk clock.Clock, interval time.Duration) time.Time {
4...

1	package schedule
2	
3	import (
4		"context"
5		"fmt"
6		"net/url"
7		"time"
8	
9		"github.com/benbjohnson/clock"
10		"golang.org/x/sync/errgroup"
11	
12		"github.com/grafana/grafana-plugin-sdk-go/data"
13	
14		"github.com/grafana/grafana/pkg/infra/log"
15		"github.com/grafana/grafana/pkg/infra/tracing"
16		"github.com/grafana/grafana/pkg/services/featuremgmt"
17		"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
18		"github.com/grafana/grafana/pkg/services/ngalert/eval"
19		"github.com/grafana/grafana/pkg/services/ngalert/metrics"
20		ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
21		"github.com/grafana/grafana/pkg/services/ngalert/schedule/ticker"
22		"github.com/grafana/grafana/pkg/services/ngalert/state"
23		"github.com/grafana/grafana/pkg/setting"
24	)
25	
26	// ScheduleService is an interface for a service that schedules the evaluation
27	// of alert rules.
28	type ScheduleService interface {
29		// Run the scheduler until the context is canceled or the scheduler returns
30		// an error. The scheduler is terminated when this function returns.
31		Run(context.Context) error
32	}
33	
34	// AlertsSender is an interface for a service that is responsible for sending notifications to the end-user.
35	//
36	//go:generate mockery --name AlertsSender --structname AlertsSenderMock --inpackage --filename alerts_sender_mock.go --with-expecter
37	type AlertsSender interface {
38		Send(ctx context.Context, key ngmodels.AlertRuleKey, alerts definitions.PostableAlerts)
39	}
40	
41	// RulesStore is a store that provides alert rules for scheduling
42	type RulesStore interface {
43		GetAlertRulesKeysForScheduling(ctx context.Context) ([]ngmodels.AlertRuleKeyWithVersion, error)
44		GetAlertRulesForScheduling(ctx context.Context, query *ngm...

/workspace/grafana--26d36ec/pkg/services/ngalert/schedule/recording_rule.go:29:	EvaluationDuration  time.Duration
/workspace/grafana--26d36ec/pkg/services/ngalert/schedule/recording_rule.go:41:	evaluationDuration  *atomic.Duration
/workspace/grafana--26d36ec/pkg/services/ngalert/schedule/recording_rule.go:82:		evaluationDuration:  atomic.NewDuration(0),
/workspace/grafana--26d36ec/pkg/services/ngalert/schedule/recording_rule.go:109:		EvaluationDuration:  r.evaluationDuration.Load(),
/workspace/grafana--26d36ec/pkg/services/ngalert/schedule/recording_rule.go:115:	var droppedMsg *Evaluation
/workspace/grafana--26d36ec/pkg/services/ngalert/schedule/recording_rule.go:117:	case droppedMsg = <-r.evalCh:
/workspace/grafana--26d36ec/pkg/services/ngalert/schedule/recording_rule.go:123:		return true, droppedMsg
/workspace/grafana--26d36ec/pkg/services/ngalert/schedule/recording_rule.go:125:		return false, droppedMsg
/workspace/grafana--26d36ec/pkg/services/ngalert/schedule/recording_rule.go:174:	evalDuration := r.metrics.EvalDuration.WithLabelValues(orgID)
/workspace/grafana--26d36ec/pkg/services/ngalert/schedule/recording_rule.go:185:		evalDuration.Observe(dur.Seconds())
/workspace/grafana--26d36ec/pkg/services/ngalert/schedule/recording_rule.go:187:		r.evaluationDuration.Store(dur)
/workspace/grafana--26d36ec/pkg/services/ngalert/schedule/recording_rule.go:284:	logger.Debug("Recording rule query completed", "resultCount", len(result.Responses), "duration", evalDur)
/workspace/grafana--26d36ec/pkg/services/ngalert/schedule/recording_rule.go:311:	logger.Debug("Metrics written", "duration", writeDur)
/workspace/grafana--26d36ec/pkg/services/ngalert/schedule/recording_rule.go:328:		logger.Error("Failed to evaluate rule", "error", err, "duration", r.clock.Now().Sub(start))
/workspac...

/workspace/grafana--26d36ec/pkg/apiserver/rest/dualwriter_syncer.go:115:	// ensure that execution takes no longer than necessary
/workspace/grafana--26d36ec/pkg/registry/apis/dashboard/authorizer.go:19:			// Note that we will return Allow more than expected.
/workspace/grafana--26d36ec/pkg/services/ngalert/metrics/scheduler.go:17:	BehindSeconds                       prometheus.Gauge
/workspace/grafana--26d36ec/pkg/services/ngalert/metrics/scheduler.go:41:		BehindSeconds: promauto.With(r).NewGauge(prometheus.GaugeOpts{
/workspace/grafana--26d36ec/pkg/services/ngalert/schedule/schedule.go:256:			sch.metrics.BehindSeconds.Set(start.Sub(tick).Seconds())
/workspace/grafana--26d36ec/pkg/services/secrets/manager/manager.go:567:	// is longer than a certain "caution period", because cache "by label"
/workspace/grafana--26d36ec/pkg/storage/unified/apistore/store.go:671:// If watch is lagging, progress status might:
/workspace/grafana--26d36ec/pkg/util/shortid_generator.go:22:	ErrUIDTooLong       = fmt.Errorf("UID is longer than %d symbols", MaxUIDLength)

195	// Returns a tuple where first element is
196	//   - true when message was sent
197	//   - false when the send operation is stopped
198	//
199	// the second element contains a dropped message that was sent by a concurrent sender.
200	func (a *alertRule) Eval(eval *Evaluation) (bool, *Evaluation) {
201		if a.key.AlertRuleKey != eval.rule.GetKey() {
202			// Make sure that rule has the same key. This should not happen
203			panic(fmt.Sprintf("Invalid rule sent for evaluating. Expected rule key %s, got %s", a.key.AlertRuleKey, eval.rule.GetKey()))
204		}
205		// read the channel in unblocking manner to make sure that there is no concurrent send operation.
206		var droppedMsg *Evaluation
207		select {
208		case droppedMsg = <-a.evalCh:
209		default:
210		}
211	
212		select {
213		case a.evalCh <- eval:
214			return true, droppedMsg
215		case <-a.ctx.Done():
216			return false, droppedMsg
217		}
218	}
219	
220	// update sends an instruction to the rule evaluation routine to update the scheduled rule to the specified version. The specified version must be later than the current version, otherwise no update will happen.
221	func (a *alertRule) Update(eval *Evaluation) bool {
222		// check if the channel is not empty.
223		select {
224		case <-a.updateCh:

1	package ticker
2	
3	import (
4		"github.com/prometheus/client_golang/prometheus"
5		"github.com/prometheus/client_golang/prometheus/promauto"
6	)
7	
8	type Metrics struct {
9		LastTickTime    prometheus.Gauge
10		NextTickTime    prometheus.Gauge
11		IntervalSeconds prometheus.Gauge
12	}
13	
14	func NewMetrics(reg prometheus.Registerer, subsystem string) *Metrics {
15		return &Metrics{
16			LastTickTime: promauto.With(reg).NewGauge(prometheus.GaugeOpts{
17				Namespace: "grafana",
18				Subsystem: subsystem,
19				Name:      "ticker_last_consumed_tick_timestamp_seconds",
20				Help:      "Timestamp of the last consumed tick in seconds.",
21			}),
22			NextTickTime: promauto.With(reg).NewGauge(prometheus.GaugeOpts{
23				Namespace: "grafana",
24				Subsystem: subsystem,
25				Name:      "ticker_next_tick_timestamp_seconds",
26				Help:      "Timestamp of the next tick in seconds before it is consumed.",
27			}),
28			IntervalSeconds: promauto.With(reg).NewGauge(prometheus.GaugeOpts{
29				Namespace: "grafana",
30				Subsystem: subsystem,
31				Name:      "ticker_interval_seconds",
32				Help:      "Interval at which the ticker is meant to tick.",
33			}),
34		}
35	}
36	

I have the full picture. Now writing the answer file.

File created successfully at: /workspace/answer.json (file state is current in your context — no need to Read it back)

Found them. The answer is written to `/workspace/answer.json`.

**TLDR:** In grafana/grafana (checkout at `/workspace/grafana--26d36ec`), the alerting evaluation scheduler tick and the overrun detection both live in the ngalert scheduler package:

1. **Scheduler tick implementation**
   - `pkg/services/ngalert/schedule/ticker/ticker.go` — the custom `ticker.T`. Unlike `time.Ticker`, it never drops ticks for slow receivers; it queues them precisely so callers can detect when evaluation falls behind. Its `run()` loop delivers any tick whose time has already passed immediately.
   - `pkg/services/ngalert/schedule/schedule.go` — `Run()` creates the ticker at the base interval, and `schedulePeriodic()` is the loop that consumes each tick from `t.C` and calls `processTick()` to dispatch rule evaluations.

2. **Overrun detection and logging**
   - `schedule.go:256` — each tick, `schedulePeriodic()` measures `start.Sub(tick)` and sets the `BehindSeconds` gauge (how late the cycle started), plus observes `SchedulePeriodicDuration`.
   - `alert_rule.go:200` (`alertRule.Eval`) and the analogous `recording_rule.go` — before sending a new evaluation into the rule's channel, `Eval` drains any still-unconsumed previous evaluation, which means the prior cycle overran its interval; it returns that as a dropped message.
   - `schedule.go:438` — `runJobFn()` checks the dropped result and logs the warning **"Tick dropped because alert rule evaluation is too slow"**, incrementing the `EvaluationMissed` counter (metrics defined in `pkg/services/ngalert/metrics/scheduler.go`).

One caveat, noted in the answer file: the literal string "alert evaluation took longer than expected" doesn't appear in this revision — the corresponding overrun warning in this code is the dropped-tick message above, ...