1Set the failure count that escalates
AlertFailureThreshold = 2: the ThresholdReached alert fires when OccurrenceCount reaches 2. It is a JobsOptions setting for the whole host, 3 by default.
var builder = Host.CreateApplicationBuilder(args);
builder.Services.UseActa(j =>
{
j.UseLocalDatabase(builder.Configuration);
j.ConfigureOptions(o => o.AlertFailureThreshold = 2);
j.Run<AlertEscalationJobs>("alert-escalation");
});
using var host = builder.Build();
await host.StartAsync();2Read each stage as an alert of its own kind
The sys.alerts sweep writes them once a minute, so they appear after the failures, not with them.
Console.WriteLine($"alerts ({alertItems.Count}):");
foreach (var alert in alertItems.OrderBy(a => a.CreatedAtUtc))
{
Console.WriteLine($" [{alert.Severity}] {alert.Kind} x{alert.OccurrenceCount} channel='{alert.ChannelName}'");
}
await host.StopAsync();3The job it acts on
public sealed class UnstableReportJob
{
// AlertProfile = OnFailure (default): the first failure raises a FirstFailure alert and repeats count
// up on it, the count reaching AlertFailureThreshold raises ThresholdReached, and the last attempt
// raises FinalFailure.
[Job("unstable-report", MaxAttempts = 4, Backoff = "0s")]
public static Task Handle(UnstableReport input, CancellationToken ct)
{
Console.WriteLine($"[{input.Name}] attempt failed");
throw new InvalidOperationException($"report {input.Name} unavailable");
}
}4Run it and check the output
From a clone of the Acta repository, with the .NET 10 SDK:
dotnet run --project learn/400-observability-and-alerts/411-alert-escalationjob reached Failed
] FirstFailure x
] ThresholdReached x
] FinalFailure xChecked every release. Before each release, Acta Verify runs this program and checks that its output contains these lines.
This page shows the code. The rules it relies on, starting with the handler running again from its first line, are in the handler contract.
The whole program
Every file of learn/400-observability-and-alerts/411-alert-escalation, as it is compiled.
UseLocalDatabase is a helper of the Acta repository, not of the packages. In your own app, register the database with UseSqlite, UsePostgres or UseSqlServer, as in how to run your first job.
alert-escalation.cs89 lines
// How to escalate an alert as failures add up
// Automatic failure alerts escalate in stages: the first failure, the failure count reaching
// `AlertFailureThreshold`, and the final failure when the retry budget is spent.
using Acta;
using Acta.Learn.AlertEscalation;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
var builder = Host.CreateApplicationBuilder(args);
builder.Services.UseActa(j =>
{
j.UseLocalDatabase(builder.Configuration);
// learn: 1. Set the failure count that escalates
// AlertFailureThreshold = 2: the ThresholdReached alert fires when OccurrenceCount reaches 2. It is a
// `JobsOptions` setting for the whole host, 3 by default.
j.ConfigureOptions(o => o.AlertFailureThreshold = 2);
// learn: end
j.Run<AlertEscalationJobs>("alert-escalation");
});
using var host = builder.Build();
await host.StartAsync();
var jobs = host.Services.GetRequiredService<IJobs>();
var queries = host.Services.GetRequiredService<IActaOperations>();
// MaxAttempts = 4, Backoff = "0s": four quick failures exhaust the budget.
var outcome = await jobs.EnqueueAsync(new UnstableReport("quarterly"));
Console.WriteLine($"enqueued {outcome.JobRef}");
// Wait for all 4 attempts to exhaust and the job to reach Failed.
Console.WriteLine("waiting for terminal failure...");
var jobId = outcome.JobId;
while (true)
{
var status = (await jobs.GetStatusAsync(JobLookup.ById(jobId)))?.Status;
if (status == JobStatusCode.Failed)
break;
await Task.Delay(200);
}
Console.WriteLine("job reached Failed");
// Automatic alerts (FirstFailure, ThresholdReached, FinalFailure) are generated by the sys.alerts
// Cron.EveryMinute sweep -- they do not appear at failure time. Poll until FinalFailure appears.
Console.WriteLine("waiting for the sys.alerts sweep...");
IReadOnlyList<ListAlertsItem> alertItems = [];
while (true)
{
var page = await queries.Alerts.ListAsync(new ListAlertsQuery(JobNamespace: "alert-escalation"));
alertItems = page.Items;
if (alertItems.Any(a => a.Kind == AlertKindCode.FinalFailure))
break;
Console.WriteLine("waiting for the sys.alerts sweep...");
await Task.Delay(5_000);
}
Console.WriteLine($"alerts ({alertItems.Count}):");
// learn: 2. Read each stage as an alert of its own kind
// The `sys.alerts` sweep writes them once a minute, so they appear after the failures, not with them.
foreach (var alert in alertItems.OrderBy(a => a.CreatedAtUtc))
{
Console.WriteLine($" [{alert.Severity}] {alert.Kind} x{alert.OccurrenceCount} channel='{alert.ChannelName}'");
}
// learn: end
await host.StopAsync();
namespace Acta.Learn.AlertEscalation
{
public sealed record UnstableReport(string Name);
public sealed class UnstableReportJob
{
// AlertProfile = OnFailure (default): the first failure raises a FirstFailure alert and repeats count
// up on it, the count reaching AlertFailureThreshold raises ThresholdReached, and the last attempt
// raises FinalFailure.
[Job("unstable-report", MaxAttempts = 4, Backoff = "0s")]
public static Task Handle(UnstableReport input, CancellationToken ct)
{
Console.WriteLine($"[{input.Name}] attempt failed");
throw new InvalidOperationException($"report {input.Name} unavailable");
}
}
}