diff --git a/Directory.Packages.props b/Directory.Packages.props index 041b190c6..fa8e6acf9 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -5,6 +5,7 @@ + diff --git a/src/FluentCMS.Core/DependencyInjection/FluentCmsBuilder.cs b/src/FluentCMS.Core/DependencyInjection/FluentCmsBuilder.cs index a3562fac6..aac783f6c 100644 --- a/src/FluentCMS.Core/DependencyInjection/FluentCmsBuilder.cs +++ b/src/FluentCMS.Core/DependencyInjection/FluentCmsBuilder.cs @@ -1,5 +1,6 @@ using System.Reflection; using FluentCMS.Core.Authorization; +using FluentCMS.Core.Jobs; using FluentCMS.Core.Storage; using Microsoft.Extensions.DependencyInjection; @@ -10,12 +11,15 @@ public sealed class FluentCmsBuilder internal FluentCmsBuilder(IServiceCollection services) { Services = services; + Jobs = new JobsBuilder(services); } public IServiceCollection Services { get; } public AuthorizationBuilder Authorization { get; } = new(); + public JobsBuilder Jobs { get; } + internal IDataProvider? DataProvider { get; private set; } internal Assembly? ContentMigrationsAssembly { get; private set; } diff --git a/src/FluentCMS.Core/DependencyInjection/FluentCmsServiceCollectionExtensions.cs b/src/FluentCMS.Core/DependencyInjection/FluentCmsServiceCollectionExtensions.cs index a6a993f0c..a44f6923c 100644 --- a/src/FluentCMS.Core/DependencyInjection/FluentCmsServiceCollectionExtensions.cs +++ b/src/FluentCMS.Core/DependencyInjection/FluentCmsServiceCollectionExtensions.cs @@ -4,10 +4,12 @@ using FluentCMS.Core.Authorization; using FluentCMS.Core.Content; using FluentCMS.Core.Content.Query; +using FluentCMS.Core.Jobs; using FluentCMS.Core.Localization; using FluentCMS.Core.Schema; using FluentCMS.Core.Sites; using FluentCMS.Core.Storage; +using FluentCMS.Jobs; using FluentCMS.Localization; using FluentCMS.Schema; using FluentCMS.Sites; @@ -81,6 +83,17 @@ public static IServiceCollection AddFluentCms(this IServiceCollection services, services.AddScoped(); services.AddScoped(); + services.AddSingleton(builder.Jobs.Options); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.AddHostedService(); + return services; } } diff --git a/src/FluentCMS.Core/FluentCMS.Core.csproj b/src/FluentCMS.Core/FluentCMS.Core.csproj index 059b2f6e1..fa3823f3e 100644 --- a/src/FluentCMS.Core/FluentCMS.Core.csproj +++ b/src/FluentCMS.Core/FluentCMS.Core.csproj @@ -6,6 +6,7 @@ + diff --git a/src/FluentCMS.Core/Jobs/JobDeadLetters.cs b/src/FluentCMS.Core/Jobs/JobDeadLetters.cs new file mode 100644 index 000000000..a08ed8afe --- /dev/null +++ b/src/FluentCMS.Core/Jobs/JobDeadLetters.cs @@ -0,0 +1,49 @@ +using FluentCMS.Core.Storage; +using FluentCMS.Core.Storage.Entities; +using FluentCMS.Jobs; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; + +namespace FluentCMS.Core.Jobs; + +internal sealed class JobDeadLetters(IServiceScopeFactory scopes, JobRunnerOptions options, TimeProvider time) : IJobDeadLetters +{ + public async Task> ListAsync(int offset = 0, int limit = 50, CancellationToken cancellationToken = default) + { + ArgumentOutOfRangeException.ThrowIfNegative(offset); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(limit); + + await using var scope = scopes.CreateAsyncScope(); + var context = scope.ServiceProvider.GetRequiredService(); + + return await context.Set().AsNoTracking() + .Where(job => job.Status == JobStatus.DeadLettered) + .OrderByDescending(job => job.CompletedAt) + .ThenBy(job => job.Id) + .Skip(offset) + .Take(limit) + .Select(job => new DeadLetteredJob(job.Id, job.Type, job.Queue, job.Attempts, job.LastError, job.CompletedAt)) + .ToListAsync(cancellationToken); + } + + // Attempts count on rather than start over, because cms_job_attempt keeps the rows of the earlier ones. + public async Task RetryAsync(Guid id, CancellationToken cancellationToken = default) + { + await using var scope = scopes.CreateAsyncScope(); + var context = scope.ServiceProvider.GetRequiredService(); + + var now = time.GetUtcNow(); + var further = options.DefaultMaxAttempts; + var queued = await context.Set() + .Where(job => job.Id == id && job.Status == JobStatus.DeadLettered) + .ExecuteUpdateAsync( + job => job + .SetProperty(row => row.Status, JobStatus.Pending) + .SetProperty(row => row.RunAt, now) + .SetProperty(row => row.MaxAttempts, row => row.Attempts + further) + .SetProperty(row => row.CompletedAt, (DateTimeOffset?)null), + cancellationToken); + + return queued == 1; + } +} diff --git a/src/FluentCMS.Core/Jobs/JobErrorCodes.cs b/src/FluentCMS.Core/Jobs/JobErrorCodes.cs new file mode 100644 index 000000000..bfa7ae65b --- /dev/null +++ b/src/FluentCMS.Core/Jobs/JobErrorCodes.cs @@ -0,0 +1,12 @@ +namespace FluentCMS.Core.Jobs; + +internal static class JobErrorCodes +{ + public const string UnknownJobType = "FCMS2501"; + public const string BadPayload = "FCMS2502"; + public const string InvalidCron = "FCMS2503"; + public const string UnregisteredJob = "FCMS2504"; + public const string DuplicateJobType = "FCMS2505"; + public const string InvalidRunnerOptions = "FCMS2506"; + public const string BadActor = "FCMS2507"; +} diff --git a/src/FluentCMS.Core/Jobs/JobErrorText.cs b/src/FluentCMS.Core/Jobs/JobErrorText.cs new file mode 100644 index 000000000..ec42898f6 --- /dev/null +++ b/src/FluentCMS.Core/Jobs/JobErrorText.cs @@ -0,0 +1,109 @@ +using System.Text; +using System.Text.Json; +using System.Text.RegularExpressions; +using FluentCMS.Core.Storage.Entities; + +namespace FluentCMS.Core.Jobs; + +// An exception message can quote the job's input, so every string, number and property name of the payload is taken +// out of what is stored and logged. This cannot catch a secret the job has encoded or derived. +internal static class JobErrorText +{ + private const string Redacted = "***"; + private const string Truncated = "..."; + + // A shorter value is matched only as a whole word, since inside a word it is more likely chance than a quote. + private const int AnywhereFromLength = 4; + + public static string ForStorage(Exception exception, string payload) + { + if (exception is JobInvocationException) + { + return Cap(exception.Message); + } + + var values = ValuesOf(payload); + var text = new StringBuilder(); + for (var current = exception; current is not null; current = current.InnerException) + { + text.Append(text.Length == 0 ? "" : " ---> ").Append(current.GetType().FullName).Append(": ").Append(Scrub(current.Message, values)); + } + + return Cap(text.ToString()); + } + + public static string ForLog(Exception exception, string payload) => Scrub(exception.ToString(), ValuesOf(payload)); + + public static string Cap(string text) + { + if (text.Length <= Job.LastErrorMaxLength) + { + return text; + } + + var keep = Job.LastErrorMaxLength - Truncated.Length; + if (char.IsHighSurrogate(text[keep - 1])) + { + keep--; + } + + return string.Concat(text.AsSpan(0, keep), Truncated); + } + + private static string Scrub(string text, List values) + { + foreach (var value in values) + { + text = value.Length >= AnywhereFromLength + ? text.Replace(value, Redacted, StringComparison.Ordinal) + : Regex.Replace(text, $@"(? ValuesOf(string payload) + { + var values = new List(); + try + { + using var document = JsonDocument.Parse(payload); + Collect(document.RootElement, values); + } + catch (JsonException) + { + values.Add(payload); + } + + return [.. values.Where(value => value.Length > 0).Distinct(StringComparer.Ordinal).OrderByDescending(value => value.Length)]; + } + + private static void Collect(JsonElement element, List values) + { + switch (element.ValueKind) + { + case JsonValueKind.String: + values.Add(element.GetString()!); + break; + case JsonValueKind.Number: + values.Add(element.GetRawText()); + break; + case JsonValueKind.Array: + foreach (var item in element.EnumerateArray()) + { + Collect(item, values); + } + + break; + case JsonValueKind.Object: + foreach (var property in element.EnumerateObject()) + { + values.Add(property.Name); + Collect(property.Value, values); + } + + break; + } + } +} diff --git a/src/FluentCMS.Core/Jobs/JobEvents.cs b/src/FluentCMS.Core/Jobs/JobEvents.cs new file mode 100644 index 000000000..9f26ddd54 --- /dev/null +++ b/src/FluentCMS.Core/Jobs/JobEvents.cs @@ -0,0 +1,21 @@ +namespace FluentCMS.Core.Jobs; + +// The events of spec 4.11, until the event bus (#2697) publishes them. +internal sealed record JobFailedEvent(Guid JobId, string Type, string Queue, int Attempt, int MaxAttempts, string Error); + +internal sealed record JobDeadLetteredEvent(Guid JobId, string Type, string Queue, int Attempts, string Error); + +// The seam for #2697. A sink that throws is logged and changes nothing about the job. +internal interface IJobEventSink +{ + Task PublishAsync(JobFailedEvent failed); + + Task PublishAsync(JobDeadLetteredEvent deadLettered); +} + +internal sealed class NullJobEventSink : IJobEventSink +{ + public Task PublishAsync(JobFailedEvent failed) => Task.CompletedTask; + + public Task PublishAsync(JobDeadLetteredEvent deadLettered) => Task.CompletedTask; +} diff --git a/src/FluentCMS.Core/Jobs/JobExecutor.cs b/src/FluentCMS.Core/Jobs/JobExecutor.cs new file mode 100644 index 000000000..fca9d91e1 --- /dev/null +++ b/src/FluentCMS.Core/Jobs/JobExecutor.cs @@ -0,0 +1,228 @@ +using FluentCMS.Content; +using FluentCMS.Core.Sites; +using FluentCMS.Core.Storage; +using FluentCMS.Core.Storage.Entities; +using FluentCMS.Jobs; +using FluentCMS.Sites; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace FluentCMS.Core.Jobs; + +internal sealed class JobExecutor( + JobStore store, + JobRegistry registry, + IServiceScopeFactory scopes, + IOperationContextAccessor operation, + JobRunnerOptions options, + TimeProvider time, + ILogger logger) +{ + private static readonly TimeSpan FirstSettleRetry = TimeSpan.FromSeconds(1); + private static readonly TimeSpan LastSettleRetry = TimeSpan.FromSeconds(30); + + // False when the queue has nothing due. + public async Task RunNextAsync( + string queue, + string owner, + CancellationToken stopClaiming = default, + CancellationToken shutdown = default, + CancellationToken abandon = default) + { + if (await store.ClaimAsync(queue, owner, stopClaiming) is not { } job) + { + return false; + } + + await RunAsync(job, shutdown, abandon); + return true; + } + + public async Task RunAsync(ClaimedJob job, CancellationToken shutdown = default, CancellationToken abandon = default) + { + using var attempt = CancellationTokenSource.CreateLinkedTokenSource(shutdown); + using var stopHeartbeat = CancellationTokenSource.CreateLinkedTokenSource(abandon); + + var heartbeat = HeartbeatAsync(job, attempt, stopHeartbeat.Token); + try + { + await ExecuteAndSettleAsync(job, shutdown, attempt.Token, abandon); + } + finally + { + await stopHeartbeat.CancelAsync(); + await heartbeat; + } + } + + private async Task ExecuteAndSettleAsync(ClaimedJob job, CancellationToken shutdown, CancellationToken attempt, CancellationToken abandon) + { + Func> settle; + try + { + await InvokeAsync(job, attempt); + settle = () => store.SucceedAsync(job); + } + catch (OperationCanceledException) when (shutdown.IsCancellationRequested) + { + JobLog.Released(logger, job.Id, job.Type, job.Attempt); + settle = () => store.ReleaseAsync(job); + } + catch (OperationCanceledException) when (attempt.IsCancellationRequested) + { + JobLog.LeaseLost(logger, job.Id, job.Type, job.Attempt); + return; + } + catch (Exception exception) + { + JobLog.AttemptFailed(logger, job.Id, job.Type, job.Attempt, JobErrorText.ForLog(exception, job.Payload)); + var error = JobErrorText.ForStorage(exception, job.Payload); + settle = () => store.FailAsync(job, error); + } + + await SettleWithRetriesAsync(job, settle, abandon); + } + + // A result not written means the job runs again, or is dead-lettered as if it never finished, so a brief outage is + // waited out. Retrying past the real end of the lease is safe, because a write matches nothing once another worker + // has the job. It gives up after one lease, in case the failure is not an outage. + private async Task SettleWithRetriesAsync(ClaimedJob job, Func> settle, CancellationToken abandon) + { + var giveUpAt = time.GetUtcNow() + options.LeaseDuration; + var delay = FirstSettleRetry; + + while (true) + { + try + { + if (!await settle()) + { + JobLog.LeaseLost(logger, job.Id, job.Type, job.Attempt); + } + + return; + } + catch (Exception exception) when (time.GetUtcNow() + delay < giveUpAt) + { + JobLog.SettleFailed(logger, exception, job.Id); + } + + await Task.Delay(delay, time, abandon); + delay = TimeSpan.FromTicks(Math.Min(delay.Ticks * 2, LastSettleRetry.Ticks)); + } + } + + private async Task InvokeAsync(ClaimedJob job, CancellationToken attempt) + { + var registration = registry.Find(job.Type) + ?? throw new JobInvocationException($"{JobErrorCodes.UnknownJobType}: no job is registered for the type '{job.Type}'."); + + var payload = ReadPayload(registration, job); + + await using var scope = scopes.CreateAsyncScope(); + var context = new JobContext(job.Id, job.Type, job.Queue, job.Attempt, job.MaxAttempts, await OperationOfAsync(job, payload, scope.ServiceProvider), attempt); + + var previous = operation.Current; + operation.Current = context.Operation; + try + { + await registration.Invoke(scope.ServiceProvider, payload, context); + } + finally + { + operation.Current = previous; + } + } + + // Keeps nothing of the reader's exception: a converter can quote any value in it, and a path can name a dictionary key. + private static object ReadPayload(JobRegistration registration, ClaimedJob job) + { + try + { + return registration.Read(job.Payload) + ?? throw new JobInvocationException($"{JobErrorCodes.BadPayload}: the payload of '{job.Type}' is null."); + } + catch (Exception exception) when (exception is not JobInvocationException) + { + throw new JobInvocationException( + $"{JobErrorCodes.BadPayload}: the payload of '{job.Type}' could not be read as {registration.Payload.Name} ({exception.GetType().Name})."); + } + } + + private async Task OperationOfAsync(ClaimedJob job, object payload, IServiceProvider services) + { + var site = job.SiteId ?? DefaultSite.Id; + var locale = await services.GetRequiredService().Set() + .Where(row => row.Id == site) + .Select(row => row.DefaultLocale) + .FirstOrDefaultAsync(); + + return new OperationContext(Principal.System, Origin.Job, new SiteId(site), locale ?? DefaultSite.Locale, ClientId: null, job.Id.ToString(), time.GetUtcNow()) + { + Actor = ActorOf(job, payload), + }; + } + + // Built from the id alone, so that nothing in a payload can bring claims. It never replaces Principal.System. + private static Principal? ActorOf(ClaimedJob job, object payload) + { + if (payload is not IJobActorPayload { Actor: { } actor }) + { + return null; + } + + if (actor.Id == Guid.Empty) + { + throw new JobInvocationException($"{JobErrorCodes.BadActor}: the payload of '{job.Type}' names an actor with an empty id."); + } + + try + { + return new Principal(actor.Kind, actor.Id, actor.Name); + } + catch (ArgumentException) + { + throw new JobInvocationException($"{JobErrorCodes.BadActor}: the payload of '{job.Type}' names an actor of kind {actor.Kind}, which is not an identity a job can act for."); + } + } + + private async Task HeartbeatAsync(ClaimedJob job, CancellationTokenSource attempt, CancellationToken stop) + { + using var timer = new PeriodicTimer(options.HeartbeatInterval, time); + var leaseUntil = job.LeaseUntil; + try + { + while (await timer.WaitForNextTickAsync(stop)) + { + try + { + if (await store.RenewLeaseAsync(job) is not { } renewed) + { + await attempt.CancelAsync(); + return; + } + + leaseUntil = renewed; + } + catch (Exception exception) when (!stop.IsCancellationRequested) + { + JobLog.HeartbeatFailed(logger, exception, job.Id); + + // With no renewal before the lease ends, another worker can take the job, so stop first. + if (time.GetUtcNow() + options.HeartbeatInterval >= leaseUntil) + { + await attempt.CancelAsync(); + return; + } + } + } + } + catch (OperationCanceledException) when (stop.IsCancellationRequested) + { + } + } +} + +// The job could not start. The message is what last_error stores, and never holds a value of the payload. +internal sealed class JobInvocationException(string message) : Exception(message); diff --git a/src/FluentCMS.Core/Jobs/JobLog.cs b/src/FluentCMS.Core/Jobs/JobLog.cs new file mode 100644 index 000000000..78bd6856b --- /dev/null +++ b/src/FluentCMS.Core/Jobs/JobLog.cs @@ -0,0 +1,43 @@ +using Microsoft.Extensions.Logging; + +namespace FluentCMS.Core.Jobs; + +// What a job threw arrives as text already scrubbed of the payload, never as the exception itself. +internal static partial class JobLog +{ + [LoggerMessage(Level = LogLevel.Warning, Message = "Job {JobId} ({Type}) failed attempt {Attempt}: {Error}")] + public static partial void AttemptFailed(ILogger logger, Guid jobId, string type, int attempt, string error); + + [LoggerMessage(Level = LogLevel.Error, Message = "Job {JobId} ({Type}) was dead-lettered after {Attempts} attempts: {Error}")] + public static partial void DeadLettered(ILogger logger, Guid jobId, string type, int attempts, string error); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Job {JobId} ({Type}) lost its lease during attempt {Attempt}, so this worker leaves it alone.")] + public static partial void LeaseLost(ILogger logger, Guid jobId, string type, int attempt); + + [LoggerMessage(Level = LogLevel.Information, Message = "Job {JobId} ({Type}) was released by the shutdown during attempt {Attempt}.")] + public static partial void Released(ILogger logger, Guid jobId, string type, int attempt); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Could not renew the lease of job {JobId}.")] + public static partial void HeartbeatFailed(ILogger logger, Exception exception, Guid jobId); + + [LoggerMessage(Level = LogLevel.Warning, Message = "Could not write the outcome of job {JobId}; trying again.")] + public static partial void SettleFailed(ILogger logger, Exception exception, Guid jobId); + + [LoggerMessage(Level = LogLevel.Error, Message = "The worker of queue {Queue} hit an error and carries on.")] + public static partial void WorkerFailed(ILogger logger, Exception exception, string queue); + + [LoggerMessage(Level = LogLevel.Error, Message = "The maintenance of the job runner hit an error and carries on.")] + public static partial void MaintenanceFailed(ILogger logger, Exception exception); + + [LoggerMessage(Level = LogLevel.Error, Message = "A job event sink threw for job {JobId}.")] + public static partial void EventSinkFailed(ILogger logger, Exception exception, Guid jobId); + + [LoggerMessage(Level = LogLevel.Error, Message = "Recurring job {Name} has a cron expression that cannot fire, so it is skipped: {Reason}")] + public static partial void RecurringCronInvalid(ILogger logger, string name, string reason); + + [LoggerMessage(Level = LogLevel.Error, Message = "Recurring job {Name} could not fire; it tries again next time.")] + public static partial void RecurringFireFailed(ILogger logger, Exception exception, string name); + + [LoggerMessage(Level = LogLevel.Warning, Message = "The job runner stopped with jobs still running, whose leases will expire.")] + public static partial void StoppedWithRunningJobs(ILogger logger); +} diff --git a/src/FluentCMS.Core/Jobs/JobRegistry.cs b/src/FluentCMS.Core/Jobs/JobRegistry.cs new file mode 100644 index 000000000..b4b7472c6 --- /dev/null +++ b/src/FluentCMS.Core/Jobs/JobRegistry.cs @@ -0,0 +1,73 @@ +using System.Collections.Frozen; +using System.Text.Json; +using FluentCMS.Core.DependencyInjection; +using FluentCMS.Jobs; +using Microsoft.Extensions.DependencyInjection; + +namespace FluentCMS.Core.Jobs; + +internal sealed class JobRegistration +{ + private JobRegistration(string type, Type job, Type payload, Func read, Func invoke) + { + Type = type; + Job = job; + Payload = payload; + Read = read; + Invoke = invoke; + } + + public string Type { get; } + + public Type Job { get; } + + public Type Payload { get; } + + public Func Read { get; } + + public Func Invoke { get; } + + public static JobRegistration Create(string type) + where TJob : class, IJob => + new( + type, + typeof(TJob), + typeof(TPayload), + json => JsonSerializer.Deserialize(json, JobJson.Options), + (services, payload, context) => services.GetRequiredService().ExecuteAsync((TPayload)payload, context)); +} + +internal static class JobJson +{ + public static readonly JsonSerializerOptions Options = JsonSerializerOptions.Web; +} + +internal sealed class JobRegistry +{ + private readonly FrozenDictionary _byType; + private readonly FrozenDictionary<(Type Job, Type Payload), string> _byJob; + + public JobRegistry(IEnumerable registrations) + { + var all = registrations.ToList(); + var duplicate = all.GroupBy(registration => registration.Type, StringComparer.Ordinal).FirstOrDefault(group => group.Count() > 1); + if (duplicate is not null) + { + throw new FluentCmsConfigurationException(JobErrorCodes.DuplicateJobType, $"The job type '{duplicate.Key}' is registered {duplicate.Count()} times."); + } + + _byType = all.ToFrozenDictionary(registration => registration.Type, StringComparer.Ordinal); + _byJob = all + .GroupBy(registration => (registration.Job, registration.Payload)) + .ToFrozenDictionary(group => group.Key, group => group.First().Type); + } + + public JobRegistration? Find(string type) => _byType.GetValueOrDefault(type); + + public string TypeOf() => + _byJob.TryGetValue((typeof(TJob), typeof(TPayload)), out var type) + ? type + : throw new FluentCmsConfigurationException( + JobErrorCodes.UnregisteredJob, + $"{typeof(TJob).FullName} is not registered for the payload {typeof(TPayload).FullName}; call cms.Jobs.AddJob inside AddFluentCms."); +} diff --git a/src/FluentCMS.Core/Jobs/JobRunner.cs b/src/FluentCMS.Core/Jobs/JobRunner.cs new file mode 100644 index 000000000..05ddde28c --- /dev/null +++ b/src/FluentCMS.Core/Jobs/JobRunner.cs @@ -0,0 +1,176 @@ +using FluentCMS.Jobs; +using Microsoft.Extensions.Logging; + +namespace FluentCMS.Core.Jobs; + +internal sealed class JobRunner( + JobExecutor executor, + JobStore store, + RecurringJobs recurring, + JobRunnerOptions options, + TimeProvider time, + ILogger logger) : IJobRunner, IDisposable +{ + private static readonly TimeSpan DisposeWait = TimeSpan.FromSeconds(5); + + private readonly CancellationTokenSource _stopping = new(); + + private readonly CancellationTokenSource _shutdown = new(); + + // Stops the heartbeats of jobs that ignored the shutdown, so that their leases can expire. + private readonly CancellationTokenSource _abandon = new(); + + private readonly string _instance = Guid.NewGuid().ToString("N"); + + private Task? _loops; + + public Task StartAsync(CancellationToken cancellationToken = default) + { + options.Validate(); + + if (_loops is not null) + { + throw new InvalidOperationException("The job runner has already started."); + } + + var loops = new List { Task.Run(MaintainAsync, CancellationToken.None) }; + foreach (var (queue, workers) in options.Queues) + { + for (var worker = 0; worker < workers; worker++) + { + var owner = $"{_instance}/{queue}/{worker}"; + loops.Add(Task.Run(() => WorkAsync(queue, owner), CancellationToken.None)); + } + } + + _loops = Task.WhenAll(loops); + return Task.CompletedTask; + } + + public async Task StopAsync(CancellationToken cancellationToken = default) + { + if (_loops is null) + { + return; + } + + await _stopping.CancelAsync(); + + if (await FinishedWithinAsync(options.ShutdownGracePeriod, cancellationToken)) + { + return; + } + + await _shutdown.CancelAsync(); + + // The host may have stopped waiting, but a job that stops now still gets its release written. + if (!await FinishedWithinAsync(options.ReleaseTimeout, CancellationToken.None)) + { + JobLog.StoppedWithRunningJobs(logger); + await _abandon.CancelAsync(); + } + } + + // A host disposed without being stopped must not leave a loop that still opens the database. + public void Dispose() + { + _stopping.Cancel(); + _shutdown.Cancel(); + _abandon.Cancel(); + + try + { + if (_loops?.Wait(DisposeWait) == false) + { + return; + } + } + catch (AggregateException) + { + } + + _stopping.Dispose(); + _shutdown.Dispose(); + _abandon.Dispose(); + } + + private async Task FinishedWithinAsync(TimeSpan period, CancellationToken cancellationToken) + { + try + { + await _loops!.WaitAsync(period, time, cancellationToken); + return true; + } + catch (Exception exception) when (exception is TimeoutException or OperationCanceledException) + { + return false; + } + } + + private async Task WorkAsync(string queue, string owner) + { + while (!_stopping.IsCancellationRequested) + { + var worked = false; + try + { + worked = await executor.RunNextAsync(queue, owner, _stopping.Token, _shutdown.Token, _abandon.Token); + } + catch (OperationCanceledException) when (_stopping.IsCancellationRequested) + { + return; + } + catch (Exception exception) + { + JobLog.WorkerFailed(logger, exception, queue); + } + + if (!worked && !await WaitAsync(options.PollInterval)) + { + return; + } + } + } + + private async Task MaintainAsync() + { + while (!_stopping.IsCancellationRequested) + { + await MaintainAsync(() => recurring.FireDueAsync(_stopping.Token)); + await MaintainAsync(store.SweepAsync); + + if (!await WaitAsync(options.MaintenanceInterval)) + { + return; + } + } + } + + private async Task MaintainAsync(Func step) + { + try + { + await step(); + } + catch (OperationCanceledException) when (_stopping.IsCancellationRequested) + { + } + catch (Exception exception) + { + JobLog.MaintenanceFailed(logger, exception); + } + } + + private async Task WaitAsync(TimeSpan period) + { + try + { + await Task.Delay(period, time, _stopping.Token); + return true; + } + catch (OperationCanceledException) + { + return false; + } + } +} diff --git a/src/FluentCMS.Core/Jobs/JobRunnerHostedService.cs b/src/FluentCMS.Core/Jobs/JobRunnerHostedService.cs new file mode 100644 index 000000000..5ec3497de --- /dev/null +++ b/src/FluentCMS.Core/Jobs/JobRunnerHostedService.cs @@ -0,0 +1,11 @@ +using FluentCMS.Jobs; +using Microsoft.Extensions.Hosting; + +namespace FluentCMS.Core.Jobs; + +internal sealed class JobRunnerHostedService(IJobRunner runner) : IHostedService +{ + public Task StartAsync(CancellationToken cancellationToken) => runner.StartAsync(cancellationToken); + + public Task StopAsync(CancellationToken cancellationToken) => runner.StopAsync(cancellationToken); +} diff --git a/src/FluentCMS.Core/Jobs/JobRunnerOptions.cs b/src/FluentCMS.Core/Jobs/JobRunnerOptions.cs new file mode 100644 index 000000000..058ec8e48 --- /dev/null +++ b/src/FluentCMS.Core/Jobs/JobRunnerOptions.cs @@ -0,0 +1,65 @@ +using FluentCMS.Core.DependencyInjection; + +namespace FluentCMS.Core.Jobs; + +public sealed class JobRunnerOptions +{ + public const string DefaultQueue = "default"; + + private static readonly TimeSpan LongestWait = TimeSpan.FromDays(49); + + // Workers per queue on this instance, which is also the concurrency limit of the queue here. A queue left out has + // no worker on this instance. + public IDictionary Queues { get; } = new Dictionary(StringComparer.Ordinal) { [DefaultQueue] = 2 }; + + public int DefaultMaxAttempts { get; set; } = 5; + + public TimeSpan PollInterval { get; set; } = TimeSpan.FromSeconds(5); + + public TimeSpan LeaseDuration { get; set; } = TimeSpan.FromMinutes(5); + + public TimeSpan HeartbeatInterval { get; set; } = TimeSpan.FromMinutes(1); + + public TimeSpan MaintenanceInterval { get; set; } = TimeSpan.FromSeconds(15); + + public TimeSpan RetryBaseDelay { get; set; } = TimeSpan.FromSeconds(30); + + public TimeSpan RetryMaxDelay { get; set; } = TimeSpan.FromHours(1); + + // A retry waits up to this fraction longer, so that jobs failing together do not return together. + public double RetryJitter { get; set; } = 0.2; + + public TimeSpan ShutdownGracePeriod { get; set; } = TimeSpan.FromSeconds(20); + + public TimeSpan ReleaseTimeout { get; set; } = TimeSpan.FromSeconds(5); + + internal void Validate() + { + foreach (var (queue, workers) in Queues) + { + Require(!string.IsNullOrEmpty(queue) && queue.Length <= JobScheduler.QueueMaxLength, $"The queue name '{queue}' is not valid."); + Require(workers >= 0, $"Queue '{queue}' has {workers} workers."); + } + + Require(DefaultMaxAttempts >= 1, $"{nameof(DefaultMaxAttempts)} is {DefaultMaxAttempts}; it must be at least 1."); + Require(PollInterval > TimeSpan.Zero, $"{nameof(PollInterval)} must be positive."); + Require(MaintenanceInterval > TimeSpan.Zero, $"{nameof(MaintenanceInterval)} must be positive."); + Require(HeartbeatInterval > TimeSpan.Zero, $"{nameof(HeartbeatInterval)} must be positive."); + Require(LeaseDuration > HeartbeatInterval, $"{nameof(LeaseDuration)} must be longer than {nameof(HeartbeatInterval)}, or a live job loses its lease."); + Require(RetryBaseDelay >= TimeSpan.Zero && RetryMaxDelay >= RetryBaseDelay, $"{nameof(RetryMaxDelay)} must be at least {nameof(RetryBaseDelay)}, which must not be negative."); + Require(RetryJitter >= 0, $"{nameof(RetryJitter)} must not be negative."); + Require(ShutdownGracePeriod >= TimeSpan.Zero && ReleaseTimeout >= TimeSpan.Zero, "The shutdown periods must not be negative."); + + // Longer than a timer can wait, these would throw only when the loop first waits, or at shutdown. + TimeSpan[] waits = [PollInterval, MaintenanceInterval, HeartbeatInterval, ShutdownGracePeriod, ReleaseTimeout]; + Require(waits.All(wait => wait <= LongestWait), $"The intervals and shutdown periods must not be longer than {LongestWait.TotalDays} days."); + } + + private static void Require(bool condition, string message) + { + if (!condition) + { + throw new FluentCmsConfigurationException(JobErrorCodes.InvalidRunnerOptions, message); + } + } +} diff --git a/src/FluentCMS.Core/Jobs/JobScheduler.cs b/src/FluentCMS.Core/Jobs/JobScheduler.cs new file mode 100644 index 000000000..63f5b0cd5 --- /dev/null +++ b/src/FluentCMS.Core/Jobs/JobScheduler.cs @@ -0,0 +1,79 @@ +using System.Text.Json; +using FluentCMS.Content; +using FluentCMS.Core.Storage; +using FluentCMS.Core.Storage.Entities; +using FluentCMS.Jobs; +using Microsoft.Extensions.DependencyInjection; + +namespace FluentCMS.Core.Jobs; + +// Writes through a context of its own, so that enqueuing never saves what a caller has tracked on its own context. +internal sealed class JobScheduler( + IServiceScopeFactory scopes, + JobRegistry registry, + RecurringJobs recurring, + IOperationContextAccessor operation, + JobRunnerOptions options, + TimeProvider time) : IJobScheduler +{ + public const int TypeMaxLength = 128; + public const int QueueMaxLength = 64; + + public Task EnqueueAsync(TPayload payload, JobOptions? jobOptions = null, CancellationToken cancellationToken = default) + where TJob : class, IJob => + AddAsync(payload, time.GetUtcNow(), jobOptions, cancellationToken); + + public Task ScheduleAsync(TPayload payload, DateTimeOffset runAt, JobOptions? jobOptions = null, CancellationToken cancellationToken = default) + where TJob : class, IJob => + AddAsync(payload, runAt, jobOptions, cancellationToken); + + public Task AddRecurringAsync(string name, string cron, TPayload payload, string? queue = null, CancellationToken cancellationToken = default) + where TJob : class, IJob + { + ArgumentNullException.ThrowIfNull(payload); + + return recurring.AddAsync(name, cron, registry.TypeOf(), Serialize(payload), CheckedQueue(queue), cancellationToken); + } + + private async Task AddAsync(TPayload payload, DateTimeOffset runAt, JobOptions? jobOptions, CancellationToken cancellationToken) + where TJob : class, IJob + { + ArgumentNullException.ThrowIfNull(payload); + + var maxAttempts = jobOptions?.MaxAttempts ?? options.DefaultMaxAttempts; + ArgumentOutOfRangeException.ThrowIfLessThan(maxAttempts, 1, nameof(jobOptions)); + + var current = operation.Current; + var job = new Job + { + Type = registry.TypeOf(), + Payload = Serialize(payload), + Queue = CheckedQueue(jobOptions?.Queue), + SiteId = (jobOptions?.Site ?? current?.Site)?.Value, + RunAt = runAt, + Priority = jobOptions?.Priority ?? 0, + MaxAttempts = maxAttempts, + CreatedAt = time.GetUtcNow(), + CreatedBy = current?.AuditPrincipal.Id is { } author && author != Guid.Empty ? author : null, + Origin = current?.Origin ?? Origin.Job, + }; + + await using var scope = scopes.CreateAsyncScope(); + var context = scope.ServiceProvider.GetRequiredService(); + context.Add(job); + await context.SaveChangesAsync(cancellationToken); + + return job.Id; + } + + private static string Serialize(TPayload payload) => JsonSerializer.Serialize(payload, JobJson.Options); + + private static string CheckedQueue(string? queue) + { + queue ??= JobRunnerOptions.DefaultQueue; + ArgumentException.ThrowIfNullOrEmpty(queue); + ArgumentOutOfRangeException.ThrowIfGreaterThan(queue.Length, QueueMaxLength, nameof(queue)); + + return queue; + } +} diff --git a/src/FluentCMS.Core/Jobs/JobStatus.cs b/src/FluentCMS.Core/Jobs/JobStatus.cs new file mode 100644 index 000000000..b1f86a140 --- /dev/null +++ b/src/FluentCMS.Core/Jobs/JobStatus.cs @@ -0,0 +1,26 @@ +using FluentCMS.Core.Storage; + +namespace FluentCMS.Core.Jobs; + +public static class JobStatus +{ + public const string Pending = JobClaimStatus.Pending; + + public const string Running = JobClaimStatus.Running; + + public const string Succeeded = "succeeded"; + + public const string DeadLettered = "dead_lettered"; +} + +public static class JobAttemptOutcome +{ + public const string Succeeded = "succeeded"; + + public const string Failed = "failed"; + + public const string LeaseExpired = "lease_expired"; + + // A shutdown handed the job back; this does not use up an attempt. + public const string Released = "released"; +} diff --git a/src/FluentCMS.Core/Jobs/JobStore.cs b/src/FluentCMS.Core/Jobs/JobStore.cs new file mode 100644 index 000000000..26cc520cd --- /dev/null +++ b/src/FluentCMS.Core/Jobs/JobStore.cs @@ -0,0 +1,245 @@ +using FluentCMS.Core.Storage; +using FluentCMS.Core.Storage.Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Query; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace FluentCMS.Core.Jobs; + +internal sealed record ClaimedJob(Guid Id, string Type, string Payload, string Queue, Guid? SiteId, int Attempt, int MaxAttempts, string Owner, DateTimeOffset LeaseUntil); + +// A worker can have lost its job to another by the time it writes, so every write after the claim is conditional on +// still holding it, and matches no row otherwise. ExecuteUpdate, because SaveChanges uses UPDATE ... RETURNING on SQLite. +internal sealed class JobStore( + IServiceScopeFactory scopes, + IJobClaimDialect dialect, + IJobEventSink events, + JobRunnerOptions options, + TimeProvider time, + ILogger logger) +{ + private const int SweepBatch = 100; + private const string LeaseExpiredError = "The worker stopped without finishing this attempt, and its lease expired."; + private const string OutOfAttemptsError = "The worker of the last attempt stopped without finishing it, and no attempts are left."; + + public async Task ClaimAsync(string queue, string owner, CancellationToken cancellationToken = default) + { + await using var scope = scopes.CreateAsyncScope(); + var context = scope.ServiceProvider.GetRequiredService(); + + var now = time.GetUtcNow(); + + // The transaction takes the write lock on SQLite, which an idle poll must not do. + if (!await AnyDueAsync(context, queue, now, cancellationToken)) + { + return null; + } + + // One transaction, so that a failure here does not use up an attempt of a job that never ran. + await using var transaction = await context.Database.BeginTransactionAsync(cancellationToken); + + var leaseUntil = now + options.LeaseDuration; + if (await dialect.ClaimAsync(context.Database, new JobClaim(queue, owner, now, leaseUntil), cancellationToken) is not { } id) + { + return null; + } + + var job = await context.Set().AsNoTracking().SingleAsync(row => row.Id == id, cancellationToken); + + // A row still open from an earlier attempt means that attempt's worker died. + var expired = await context.Set() + .Where(attempt => attempt.JobId == id && attempt.Attempt < job.Attempts && attempt.Outcome == null) + .ExecuteUpdateAsync( + attempt => attempt + .SetProperty(row => row.Outcome, JobAttemptOutcome.LeaseExpired) + .SetProperty(row => row.FinishedAt, now) + .SetProperty(row => row.Error, LeaseExpiredError), + cancellationToken); + + context.Add(new JobAttempt { JobId = id, Attempt = job.Attempts, StartedAt = now }); + await context.SaveChangesAsync(cancellationToken); + + await transaction.CommitAsync(CancellationToken.None); + + if (expired > 0) + { + await PublishAsync(job.Id, new JobFailedEvent(job.Id, job.Type, job.Queue, job.Attempts - 1, job.MaxAttempts, LeaseExpiredError)); + } + + return new ClaimedJob(job.Id, job.Type, job.Payload, job.Queue, job.SiteId, job.Attempts, job.MaxAttempts, owner, leaseUntil); + } + + // The new end of the lease, or null when the worker no longer holds the job. + public async Task RenewLeaseAsync(ClaimedJob job) + { + await using var scope = scopes.CreateAsyncScope(); + var context = scope.ServiceProvider.GetRequiredService(); + + var until = time.GetUtcNow() + options.LeaseDuration; + var renewed = await Held(context, job).ExecuteUpdateAsync(row => row.SetProperty(column => column.LeaseUntil, until)); + + return renewed == 1 ? until : null; + } + + public Task SucceedAsync(ClaimedJob job) + { + var now = time.GetUtcNow(); + return SettleAsync(job, JobAttemptOutcome.Succeeded, error: null, now, row => row + .SetProperty(column => column.Status, JobStatus.Succeeded) + .SetProperty(column => column.CompletedAt, now) + .SetProperty(column => column.LastError, (string?)null) + .SetProperty(column => column.LeaseOwner, (string?)null) + .SetProperty(column => column.LeaseUntil, (DateTimeOffset?)null)); + } + + public async Task FailAsync(ClaimedJob job, string error) + { + var now = time.GetUtcNow(); + var deadLetter = job.Attempt >= job.MaxAttempts; + var runAt = now + RetryBackoff.After(job.Attempt, options); + + var held = await SettleAsync(job, JobAttemptOutcome.Failed, error, now, row => + { + row.SetProperty(column => column.LastError, error) + .SetProperty(column => column.LeaseOwner, (string?)null) + .SetProperty(column => column.LeaseUntil, (DateTimeOffset?)null); + + if (deadLetter) + { + row.SetProperty(column => column.Status, JobStatus.DeadLettered) + .SetProperty(column => column.CompletedAt, now); + } + else + { + row.SetProperty(column => column.Status, JobStatus.Pending) + .SetProperty(column => column.RunAt, runAt); + } + }); + + if (held) + { + await PublishAsync(job.Id, new JobFailedEvent(job.Id, job.Type, job.Queue, job.Attempt, job.MaxAttempts, error)); + if (deadLetter) + { + JobLog.DeadLettered(logger, job.Id, job.Type, job.Attempt, error); + await PublishAsync(job.Id, new JobDeadLetteredEvent(job.Id, job.Type, job.Queue, job.Attempt, error)); + } + } + + return held; + } + + // The attempt keeps its row, because attempts are numbered on, so max_attempts rises by one to give it back. + public Task ReleaseAsync(ClaimedJob job) + { + var now = time.GetUtcNow(); + return SettleAsync(job, JobAttemptOutcome.Released, error: null, now, row => row + .SetProperty(column => column.Status, JobStatus.Pending) + .SetProperty(column => column.MaxAttempts, column => column.MaxAttempts + 1) + .SetProperty(column => column.LeaseOwner, (string?)null) + .SetProperty(column => column.LeaseUntil, (DateTimeOffset?)null)); + } + + // A claim requires attempts < max_attempts, so a job whose worker died on its last attempt never comes back and + // would stay running without this. + public async Task SweepAsync() + { + await using var scope = scopes.CreateAsyncScope(); + var context = scope.ServiceProvider.GetRequiredService(); + + var now = time.GetUtcNow(); + var stuck = await Stuck(context, now).AsNoTracking().Take(SweepBatch).ToListAsync(); + + var swept = 0; + foreach (var job in stuck) + { + await using var transaction = await context.Database.BeginTransactionAsync(); + + var won = await Stuck(context, now).Where(row => row.Id == job.Id).ExecuteUpdateAsync(row => row + .SetProperty(column => column.Status, JobStatus.DeadLettered) + .SetProperty(column => column.CompletedAt, now) + .SetProperty(column => column.LastError, OutOfAttemptsError) + .SetProperty(column => column.LeaseOwner, (string?)null) + .SetProperty(column => column.LeaseUntil, (DateTimeOffset?)null)); + + if (won == 0) + { + continue; + } + + await context.Set() + .Where(attempt => attempt.JobId == job.Id && attempt.Outcome == null) + .ExecuteUpdateAsync(attempt => attempt + .SetProperty(row => row.Outcome, JobAttemptOutcome.LeaseExpired) + .SetProperty(row => row.FinishedAt, now) + .SetProperty(row => row.Error, OutOfAttemptsError)); + + await transaction.CommitAsync(); + + swept++; + JobLog.DeadLettered(logger, job.Id, job.Type, job.Attempts, OutOfAttemptsError); + await PublishAsync(job.Id, new JobFailedEvent(job.Id, job.Type, job.Queue, job.Attempts, job.MaxAttempts, OutOfAttemptsError)); + await PublishAsync(job.Id, new JobDeadLetteredEvent(job.Id, job.Type, job.Queue, job.Attempts, OutOfAttemptsError)); + } + + return swept; + } + + private static Task AnyDueAsync(SystemDbContext context, string queue, DateTimeOffset now, CancellationToken cancellationToken) => + context.Set().AnyAsync( + row => row.Queue == queue + && (row.Status == JobStatus.Pending || row.Status == JobStatus.Running) + && row.RunAt <= now + && row.Attempts < row.MaxAttempts + && (row.LeaseUntil == null || row.LeaseUntil < now), + cancellationToken); + + private static IQueryable Stuck(SystemDbContext context, DateTimeOffset now) => + context.Set().Where(row => + (row.Status == JobStatus.Pending || row.Status == JobStatus.Running) + && row.Attempts >= row.MaxAttempts + && (row.LeaseUntil == null || row.LeaseUntil < now)); + + private static IQueryable Held(SystemDbContext context, ClaimedJob job) => + context.Set().Where(row => + row.Id == job.Id && row.Status == JobStatus.Running && row.LeaseOwner == job.Owner && row.Attempts == job.Attempt); + + private async Task SettleAsync(ClaimedJob job, string outcome, string? error, DateTimeOffset now, Action> update) + { + await using var scope = scopes.CreateAsyncScope(); + var context = scope.ServiceProvider.GetRequiredService(); + await using var transaction = await context.Database.BeginTransactionAsync(); + + if (await Held(context, job).ExecuteUpdateAsync(update) == 0) + { + return false; + } + + await context.Set() + .Where(attempt => attempt.JobId == job.Id && attempt.Attempt == job.Attempt) + .ExecuteUpdateAsync(attempt => attempt + .SetProperty(row => row.Outcome, outcome) + .SetProperty(row => row.FinishedAt, now) + .SetProperty(row => row.Error, error)); + + await transaction.CommitAsync(); + return true; + } + + private Task PublishAsync(Guid jobId, JobFailedEvent failed) => GuardAsync(jobId, () => events.PublishAsync(failed)); + + private Task PublishAsync(Guid jobId, JobDeadLetteredEvent deadLettered) => GuardAsync(jobId, () => events.PublishAsync(deadLettered)); + + private async Task GuardAsync(Guid jobId, Func publish) + { + try + { + await publish(); + } + catch (Exception exception) + { + JobLog.EventSinkFailed(logger, exception, jobId); + } + } +} diff --git a/src/FluentCMS.Core/Jobs/JobsBuilder.cs b/src/FluentCMS.Core/Jobs/JobsBuilder.cs new file mode 100644 index 000000000..c6ce90443 --- /dev/null +++ b/src/FluentCMS.Core/Jobs/JobsBuilder.cs @@ -0,0 +1,28 @@ +using FluentCMS.Jobs; +using Microsoft.Extensions.DependencyInjection; + +namespace FluentCMS.Core.Jobs; + +public sealed class JobsBuilder +{ + private readonly IServiceCollection _services; + + internal JobsBuilder(IServiceCollection services) + { + _services = services; + } + + public JobRunnerOptions Options { get; } = new(); + + // The type is what cms_job stores, so it must outlive renames of the class. + public JobsBuilder AddJob(string type) + where TJob : class, IJob + { + ArgumentException.ThrowIfNullOrEmpty(type); + ArgumentOutOfRangeException.ThrowIfGreaterThan(type.Length, JobScheduler.TypeMaxLength, nameof(type)); + + _services.AddTransient(); + _services.AddSingleton(JobRegistration.Create(type)); + return this; + } +} diff --git a/src/FluentCMS.Core/Jobs/RecurringJobs.cs b/src/FluentCMS.Core/Jobs/RecurringJobs.cs new file mode 100644 index 000000000..8e76acbcd --- /dev/null +++ b/src/FluentCMS.Core/Jobs/RecurringJobs.cs @@ -0,0 +1,161 @@ +using Cronos; +using FluentCMS.Content; +using FluentCMS.Core.Storage; +using FluentCMS.Core.Storage.Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace FluentCMS.Core.Jobs; + +internal sealed class RecurringJobs(IServiceScopeFactory scopes, JobRunnerOptions options, TimeProvider time, ILogger logger) +{ + private const int NameMaxLength = 128; + + public async Task AddAsync(string name, string cron, string type, string payload, string queue, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrEmpty(name); + ArgumentOutOfRangeException.ThrowIfGreaterThan(name.Length, NameMaxLength, nameof(name)); + + var next = NextOccurrence(ParseCron(cron), time.GetUtcNow()) ?? throw new ArgumentException($"{JobErrorCodes.InvalidCron}: the cron expression '{cron}' never occurs.", nameof(cron)); + + await using var scope = scopes.CreateAsyncScope(); + var context = scope.ServiceProvider.GetRequiredService(); + + if (await UpdateAsync(context, name, cron, type, payload, queue, next, cancellationToken)) + { + return; + } + + context.Add(new RecurringJob { Name = name, Cron = cron, Type = type, Payload = payload, Queue = queue, NextRunAt = next, Enabled = true }); + try + { + await context.SaveChangesAsync(cancellationToken); + } + catch (DbUpdateException) + { + // Another instance added it at the same startup, so what is left is an update. + context.ChangeTracker.Clear(); + if (!await UpdateAsync(context, name, cron, type, payload, queue, next, cancellationToken)) + { + throw; + } + } + } + + public async Task FireDueAsync(CancellationToken cancellationToken = default) + { + List due; + var now = time.GetUtcNow(); + + await using (var scope = scopes.CreateAsyncScope()) + { + due = await scope.ServiceProvider.GetRequiredService().Set().AsNoTracking() + .Where(job => job.Enabled && job.NextRunAt <= now) + .ToListAsync(cancellationToken); + } + + foreach (var job in due) + { + if (Next(job, now) is not { } next) + { + continue; + } + + try + { + await FireAsync(job, next, now, cancellationToken); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + JobLog.RecurringFireFailed(logger, exception, job.Name); + } + } + } + + // Only the instance that moves next_run_at on enqueues the job, so an occurrence fires once however many try. + private async Task FireAsync(RecurringJob due, DateTimeOffset next, DateTimeOffset now, CancellationToken cancellationToken) + { + await using var scope = scopes.CreateAsyncScope(); + var context = scope.ServiceProvider.GetRequiredService(); + await using var transaction = await context.Database.BeginTransactionAsync(cancellationToken); + + var moved = await context.Set() + .Where(job => job.Name == due.Name && job.Enabled && job.NextRunAt == due.NextRunAt) + .ExecuteUpdateAsync(job => job.SetProperty(row => row.NextRunAt, next), cancellationToken); + + if (moved == 0) + { + return; + } + + context.Add(new Job + { + Type = due.Type, + Payload = due.Payload, + Queue = due.Queue, + RunAt = due.NextRunAt, + MaxAttempts = options.DefaultMaxAttempts, + CreatedAt = now, + Origin = Origin.Job, + }); + await context.SaveChangesAsync(cancellationToken); + await transaction.CommitAsync(cancellationToken); + } + + // A row can hold what AddAsync would refuse, if it was edited by hand. + private DateTimeOffset? Next(RecurringJob job, DateTimeOffset now) + { + try + { + if (NextOccurrence(ParseCron(job.Cron), now) is { } next) + { + return next; + } + + JobLog.RecurringCronInvalid(logger, job.Name, "it never occurs"); + } + catch (ArgumentException exception) + { + JobLog.RecurringCronInvalid(logger, job.Name, exception.Message); + } + + return null; + } + + private static async Task UpdateAsync(SystemDbContext context, string name, string cron, string type, string payload, string queue, DateTimeOffset next, CancellationToken cancellationToken) + { + var rows = context.Set().Where(job => job.Name == name); + + // Only a changed cron moves the next run, so that a restart does not skip an occurrence. + await rows.Where(job => job.Cron != cron) + .ExecuteUpdateAsync(job => job.SetProperty(row => row.NextRunAt, next), cancellationToken); + + var updated = await rows.ExecuteUpdateAsync( + job => job + .SetProperty(row => row.Cron, cron) + .SetProperty(row => row.Type, type) + .SetProperty(row => row.Payload, payload) + .SetProperty(row => row.Queue, queue), + cancellationToken); + + return updated == 1; + } + + private static DateTimeOffset? NextOccurrence(CronExpression expression, DateTimeOffset after) => + expression.GetNextOccurrence(after, TimeZoneInfo.Utc); + + private static CronExpression ParseCron(string cron) + { + ArgumentException.ThrowIfNullOrEmpty(cron); + + try + { + return CronExpression.Parse(cron); + } + catch (CronFormatException exception) + { + throw new ArgumentException($"{JobErrorCodes.InvalidCron}: the cron expression '{cron}' is not valid: {exception.Message}", nameof(cron), exception); + } + } +} diff --git a/src/FluentCMS.Core/Jobs/RetryBackoff.cs b/src/FluentCMS.Core/Jobs/RetryBackoff.cs new file mode 100644 index 000000000..f47001c8d --- /dev/null +++ b/src/FluentCMS.Core/Jobs/RetryBackoff.cs @@ -0,0 +1,13 @@ +namespace FluentCMS.Core.Jobs; + +internal static class RetryBackoff +{ + public static TimeSpan After(int attempt, JobRunnerOptions options) + { + var seconds = Math.Min( + options.RetryBaseDelay.TotalSeconds * Math.Pow(2, Math.Max(attempt - 1, 0)), + options.RetryMaxDelay.TotalSeconds); + + return TimeSpan.FromSeconds(seconds * (1 + (Random.Shared.NextDouble() * options.RetryJitter))); + } +} diff --git a/src/FluentCMS.Core/Storage/Entities/Job.cs b/src/FluentCMS.Core/Storage/Entities/Job.cs new file mode 100644 index 000000000..2e2daa862 --- /dev/null +++ b/src/FluentCMS.Core/Storage/Entities/Job.cs @@ -0,0 +1,44 @@ +using FluentCMS.Content; + +namespace FluentCMS.Core.Storage.Entities; + +// A row of cms_job. Write it through EF: on SQLite, everything after the claim binds the id as the uppercase text EF stores. +public sealed class Job +{ + public const int LastErrorMaxLength = 2000; + + public Guid Id { get; set; } + + public string Type { get; set; } = ""; + + public string Payload { get; set; } = ""; + + public string Queue { get; set; } = ""; + + public Guid? SiteId { get; set; } + + public DateTimeOffset RunAt { get; set; } + + public int Priority { get; set; } + + public string Status { get; set; } = JobClaimStatus.Pending; + + public int Attempts { get; set; } + + public int MaxAttempts { get; set; } + + public DateTimeOffset? LeaseUntil { get; set; } + + public string? LeaseOwner { get; set; } + + public string? LastError { get; set; } + + public DateTimeOffset CreatedAt { get; set; } + + public Guid? CreatedBy { get; set; } + + // Where the job was enqueued from, not how it runs: every job runs with Origin.Job. + public Origin Origin { get; set; } + + public DateTimeOffset? CompletedAt { get; set; } +} diff --git a/src/FluentCMS.Core/Storage/Entities/JobAttempt.cs b/src/FluentCMS.Core/Storage/Entities/JobAttempt.cs new file mode 100644 index 000000000..f71e51bbc --- /dev/null +++ b/src/FluentCMS.Core/Storage/Entities/JobAttempt.cs @@ -0,0 +1,18 @@ +namespace FluentCMS.Core.Storage.Entities; + +// A row of cms_job_attempt. Outcome stays null while the attempt runs, and after a crash until the job is next +// claimed or swept. +public sealed class JobAttempt +{ + public Guid JobId { get; set; } + + public int Attempt { get; set; } + + public DateTimeOffset StartedAt { get; set; } + + public DateTimeOffset? FinishedAt { get; set; } + + public string? Outcome { get; set; } + + public string? Error { get; set; } +} diff --git a/src/FluentCMS.Core/Storage/Entities/RecurringJob.cs b/src/FluentCMS.Core/Storage/Entities/RecurringJob.cs new file mode 100644 index 000000000..f1d7b4689 --- /dev/null +++ b/src/FluentCMS.Core/Storage/Entities/RecurringJob.cs @@ -0,0 +1,19 @@ +namespace FluentCMS.Core.Storage.Entities; + +public sealed class RecurringJob +{ + public string Name { get; set; } = ""; + + public string Cron { get; set; } = ""; + + // The name the job is registered under. + public string Type { get; set; } = ""; + + public string Payload { get; set; } = ""; + + public string Queue { get; set; } = ""; + + public DateTimeOffset NextRunAt { get; set; } + + public bool Enabled { get; set; } +} diff --git a/src/FluentCMS.Core/Storage/StorageNames.cs b/src/FluentCMS.Core/Storage/StorageNames.cs index 88a10c79c..c773b784e 100644 --- a/src/FluentCMS.Core/Storage/StorageNames.cs +++ b/src/FluentCMS.Core/Storage/StorageNames.cs @@ -15,6 +15,9 @@ internal static class StorageNames public const string Version = "cms_version"; public const string Role = "cms_role"; public const string RolePermission = "cms_role_permission"; + public const string Job = "cms_job"; + public const string JobAttempt = "cms_job_attempt"; + public const string RecurringJob = "cms_recurring_job"; public const string Id = "id"; public const string DocumentId = "document_id"; diff --git a/src/FluentCMS.Core/Storage/SystemModelBuilder.cs b/src/FluentCMS.Core/Storage/SystemModelBuilder.cs index d9afb8cb2..4a8248f7b 100644 --- a/src/FluentCMS.Core/Storage/SystemModelBuilder.cs +++ b/src/FluentCMS.Core/Storage/SystemModelBuilder.cs @@ -14,9 +14,17 @@ internal sealed class SystemModelBuilder(IStorageConventions conventions) private const int PermissionLength = 128; private const int SiteNameLength = 128; private const int SlugLength = 64; + private const int JobTypeLength = 128; + private const int QueueLength = 64; + private const int LeaseOwnerLength = 128; + private const int RecurringNameLength = 128; + private const int CronLength = 128; public void Apply(ModelBuilder modelBuilder) { + modelBuilder.Entity(MapJob); + modelBuilder.Entity(MapJobAttempt); + modelBuilder.Entity(MapRecurringJob); modelBuilder.Entity(MapSite); modelBuilder.Entity(MapSiteLocale); modelBuilder.Entity(MapDocument); @@ -146,6 +154,53 @@ private void MapRoleGrant(EntityTypeBuilder entity) .HasDatabaseName("ix_cms_role_permission_role_permission"); } + private void MapJob(EntityTypeBuilder entity) + { + entity.ToTable(StorageNames.Job); + MapColumnNames(entity); + entity.Property(job => job.Id).HasValueGenerator(); + entity.Property(job => job.Type).HasMaxLength(JobTypeLength); + entity.Property(job => job.Queue).HasMaxLength(QueueLength); + entity.Property(job => job.Status).HasMaxLength(StatusLength); + entity.Property(job => job.LeaseOwner).HasMaxLength(LeaseOwnerLength); + entity.Property(job => job.LastError).HasMaxLength(Job.LastErrorMaxLength); + entity.Property(job => job.Origin).HasConversion().HasMaxLength(StatusLength); + conventions.ConfigureJsonColumn(entity, entity.Property(job => job.Payload)); + + // In the order of the claim's ORDER BY, so that it seeks and does not sort, and over live jobs only, so that + // finished ones are never walked. The claim must repeat this filter as literals for a database to use it. + entity.HasIndex(job => new { job.Queue, job.Priority, job.RunAt, job.Id }) + .IsDescending(false, true, false, false) + .HasFilter("status IN ('pending', 'running')") + .HasDatabaseName("ix_cms_job_claim"); + + entity.HasIndex(job => new { job.Status, job.LeaseUntil }) + .HasDatabaseName("ix_cms_job_status_lease"); + } + + private static void MapJobAttempt(EntityTypeBuilder entity) + { + entity.ToTable(StorageNames.JobAttempt); + MapColumnNames(entity); + entity.HasKey(attempt => new { attempt.JobId, attempt.Attempt }); + entity.Property(attempt => attempt.Outcome).HasMaxLength(StatusLength); + entity.Property(attempt => attempt.Error).HasMaxLength(Job.LastErrorMaxLength); + + entity.HasOne().WithMany().HasForeignKey(attempt => attempt.JobId).OnDelete(DeleteBehavior.Cascade); + } + + private void MapRecurringJob(EntityTypeBuilder entity) + { + entity.ToTable(StorageNames.RecurringJob); + MapColumnNames(entity); + entity.HasKey(job => job.Name); + entity.Property(job => job.Name).HasMaxLength(RecurringNameLength); + entity.Property(job => job.Cron).HasMaxLength(CronLength); + entity.Property(job => job.Type).HasMaxLength(JobTypeLength); + entity.Property(job => job.Queue).HasMaxLength(QueueLength); + conventions.ConfigureJsonColumn(entity, entity.Property(job => job.Payload)); + } + private static void MapColumnNames(EntityTypeBuilder entity) where T : class { diff --git a/src/FluentCMS.Data.PostgreSql/Migrations/20261102202439_AddJobs.Designer.cs b/src/FluentCMS.Data.PostgreSql/Migrations/20261102202439_AddJobs.Designer.cs new file mode 100644 index 000000000..21c467968 --- /dev/null +++ b/src/FluentCMS.Data.PostgreSql/Migrations/20261102202439_AddJobs.Designer.cs @@ -0,0 +1,753 @@ +// +using System; +using FluentCMS.Core.Storage; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace FluentCMS.Data.PostgreSql.Migrations +{ + [DbContext(typeof(SystemDbContext))] + [Migration("20261102202439_AddJobs")] + partial class AddJobs + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.14") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.ContentVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("AuthorId") + .HasColumnType("uuid") + .HasColumnName("author_id"); + + b.Property("ClientId") + .HasColumnType("text") + .HasColumnName("client_id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DocumentId") + .HasColumnType("uuid") + .HasColumnName("document_id"); + + b.Property("Locale") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("locale"); + + b.Property("Message") + .HasColumnType("text") + .HasColumnName("message"); + + b.Property("Number") + .HasColumnType("integer") + .HasColumnName("number"); + + b.Property("Origin") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("origin"); + + b.Property("Snapshot") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("snapshot"); + + b.Property("StatusAfter") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("status_after"); + + b.HasKey("Id"); + + b.HasIndex("DocumentId", "Locale", "Number") + .IsUnique() + .HasDatabaseName("uq_cms_version_document_locale_number"); + + b.ToTable("cms_version", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Document", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CreatedBy") + .HasColumnType("uuid") + .HasColumnName("created_by"); + + b.Property("DeletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("deleted_at"); + + b.Property("DeletedBy") + .HasColumnType("uuid") + .HasColumnName("deleted_by"); + + b.Property("DocumentId") + .HasColumnType("uuid") + .HasColumnName("document_id"); + + b.Property("DraftVersion") + .HasColumnType("integer") + .HasColumnName("draft_version"); + + b.Property("FirstPublishedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("first_published_at"); + + b.Property("Locale") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("locale"); + + b.Property("PublishedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("published_at"); + + b.Property("PublishedVersion") + .HasColumnType("integer") + .HasColumnName("published_version"); + + b.Property("SchedulePublishAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("schedule_publish_at"); + + b.Property("ScheduleUnpublishAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("schedule_unpublish_at"); + + b.Property("SiteId") + .HasColumnType("uuid") + .HasColumnName("site_id"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("status"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("type"); + + b.Property("UpdatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("updated_at"); + + b.Property("UpdatedBy") + .HasColumnType("uuid") + .HasColumnName("updated_by"); + + b.HasKey("Id"); + + b.HasIndex("DocumentId", "Locale") + .IsUnique() + .HasDatabaseName("uq_cms_document_document_locale"); + + b.ToTable("cms_document", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Job", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Attempts") + .HasColumnType("integer") + .HasColumnName("attempts"); + + b.Property("CompletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("completed_at"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CreatedBy") + .HasColumnType("uuid") + .HasColumnName("created_by"); + + b.Property("LastError") + .HasMaxLength(2000) + .HasColumnType("character varying(2000)") + .HasColumnName("last_error"); + + b.Property("LeaseOwner") + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("lease_owner"); + + b.Property("LeaseUntil") + .HasColumnType("timestamp with time zone") + .HasColumnName("lease_until"); + + b.Property("MaxAttempts") + .HasColumnType("integer") + .HasColumnName("max_attempts"); + + b.Property("Origin") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("origin"); + + b.Property("Payload") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("payload"); + + b.Property("Priority") + .HasColumnType("integer") + .HasColumnName("priority"); + + b.Property("Queue") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("queue"); + + b.Property("RunAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("run_at"); + + b.Property("SiteId") + .HasColumnType("uuid") + .HasColumnName("site_id"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("status"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("type"); + + b.HasKey("Id"); + + b.HasIndex("Status", "LeaseUntil") + .HasDatabaseName("ix_cms_job_status_lease"); + + b.HasIndex("Queue", "Priority", "RunAt", "Id") + .IsDescending(false, true, false, false) + .HasDatabaseName("ix_cms_job_claim") + .HasFilter("status IN ('pending', 'running')"); + + b.ToTable("cms_job", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.JobAttempt", b => + { + b.Property("JobId") + .HasColumnType("uuid") + .HasColumnName("job_id"); + + b.Property("Attempt") + .HasColumnType("integer") + .HasColumnName("attempt"); + + b.Property("Error") + .HasMaxLength(2000) + .HasColumnType("character varying(2000)") + .HasColumnName("error"); + + b.Property("FinishedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("finished_at"); + + b.Property("Outcome") + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("outcome"); + + b.Property("StartedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("started_at"); + + b.HasKey("JobId", "Attempt"); + + b.ToTable("cms_job_attempt", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.RecurringJob", b => + { + b.Property("Name") + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("name"); + + b.Property("Cron") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("cron"); + + b.Property("Enabled") + .HasColumnType("boolean") + .HasColumnName("enabled"); + + b.Property("NextRunAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("next_run_at"); + + b.Property("Payload") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("payload"); + + b.Property("Queue") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("queue"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("type"); + + b.HasKey("Name"); + + b.ToTable("cms_recurring_job", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Reference", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Field") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("field"); + + b.Property("Locale") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("locale"); + + b.Property("Position") + .HasColumnType("integer") + .HasColumnName("position"); + + b.Property("SourceDocumentId") + .HasColumnType("uuid") + .HasColumnName("source_document_id"); + + b.Property("SourceType") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("source_type"); + + b.Property("State") + .HasColumnType("smallint") + .HasColumnName("state"); + + b.Property("TargetId") + .HasColumnType("uuid") + .HasColumnName("target_id"); + + b.Property("TargetType") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("target_type"); + + b.HasKey("Id"); + + b.HasIndex("TargetType", "TargetId") + .HasDatabaseName("ix_cms_reference_target"); + + b.HasIndex("SourceType", "SourceDocumentId", "State", "Locale", "Field", "Position") + .IsUnique() + .HasDatabaseName("uq_cms_reference_source"); + + b.ToTable("cms_reference", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Role", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("BuiltIn") + .HasColumnType("boolean") + .HasColumnName("built_in"); + + b.Property("Description") + .HasMaxLength(512) + .HasColumnType("character varying(512)") + .HasColumnName("description"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("name"); + + b.Property("PermissionsVersion") + .IsConcurrencyToken() + .HasColumnType("bigint") + .HasColumnName("permissions_version"); + + b.Property("SiteId") + .HasColumnType("uuid") + .HasColumnName("site_id"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique() + .HasDatabaseName("uq_cms_role_name") + .HasFilter("site_id IS NULL"); + + b.HasIndex("SiteId", "Name") + .IsUnique() + .HasDatabaseName("uq_cms_role_site_name") + .HasFilter("site_id IS NOT NULL"); + + b.ToTable("cms_role", (string)null); + + b.HasData( + new + { + Id = new Guid("00000000-0000-7000-8000-000000000001"), + BuiltIn = true, + Description = "Holds every permission.", + Name = "Super Admin", + PermissionsVersion = 1L + }, + new + { + Id = new Guid("00000000-0000-7000-8000-000000000002"), + BuiltIn = true, + Description = "Creates, edits, publishes and removes any content.", + Name = "Editor", + PermissionsVersion = 1L + }, + new + { + Id = new Guid("00000000-0000-7000-8000-000000000003"), + BuiltIn = true, + Description = "Creates content, and reads, edits or removes only their own entries.", + Name = "Author", + PermissionsVersion = 1L + }, + new + { + Id = new Guid("00000000-0000-7000-8000-000000000004"), + BuiltIn = true, + Description = "The anonymous visitor. Grants here open public Delivery access.", + Name = "Public", + PermissionsVersion = 1L + }, + new + { + Id = new Guid("00000000-0000-7000-8000-000000000005"), + BuiltIn = true, + Description = "The default role of a registered member.", + Name = "Member", + PermissionsVersion = 1L + }, + new + { + Id = new Guid("00000000-0000-7000-8000-000000000006"), + BuiltIn = true, + Description = "The anonymous visitor. Grants here open public Delivery access.", + Name = "Public", + PermissionsVersion = 1L, + SiteId = new Guid("00000000-0000-7000-8002-000000000001") + }); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.RoleGrant", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Constraints") + .HasColumnType("jsonb") + .HasColumnName("constraints"); + + b.Property("Permission") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("permission"); + + b.Property("RoleId") + .HasColumnType("uuid") + .HasColumnName("role_id"); + + b.HasKey("Id"); + + b.HasIndex("RoleId", "Permission") + .HasDatabaseName("ix_cms_role_permission_role_permission"); + + b.ToTable("cms_role_permission", (string)null); + + b.HasData( + new + { + Id = new Guid("00000000-0000-7000-8001-000000000001"), + Permission = "*", + RoleId = new Guid("00000000-0000-7000-8000-000000000001") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000002"), + Permission = "content.*.create", + RoleId = new Guid("00000000-0000-7000-8000-000000000002") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000003"), + Permission = "content.*.read", + RoleId = new Guid("00000000-0000-7000-8000-000000000002") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000004"), + Permission = "content.*.update", + RoleId = new Guid("00000000-0000-7000-8000-000000000002") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000005"), + Permission = "content.*.delete", + RoleId = new Guid("00000000-0000-7000-8000-000000000002") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000006"), + Permission = "content.*.publish", + RoleId = new Guid("00000000-0000-7000-8000-000000000002") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000007"), + Permission = "content.*.unpublish", + RoleId = new Guid("00000000-0000-7000-8000-000000000002") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000008"), + Permission = "content.*.restore", + RoleId = new Guid("00000000-0000-7000-8000-000000000002") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000009"), + Permission = "schema.read", + RoleId = new Guid("00000000-0000-7000-8000-000000000002") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000010"), + Permission = "content.*.create", + RoleId = new Guid("00000000-0000-7000-8000-000000000003") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000011"), + Constraints = "{\"rowRule\":\"own-entries\"}", + Permission = "content.*.read", + RoleId = new Guid("00000000-0000-7000-8000-000000000003") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000012"), + Constraints = "{\"rowRule\":\"own-entries\"}", + Permission = "content.*.update", + RoleId = new Guid("00000000-0000-7000-8000-000000000003") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000013"), + Constraints = "{\"rowRule\":\"own-entries\"}", + Permission = "content.*.delete", + RoleId = new Guid("00000000-0000-7000-8000-000000000003") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000014"), + Permission = "schema.read", + RoleId = new Guid("00000000-0000-7000-8000-000000000003") + }); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Site", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("DefaultLocale") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("default_locale"); + + b.Property("IsDefault") + .HasColumnType("boolean") + .HasColumnName("is_default"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("name"); + + b.Property("Slug") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("slug"); + + b.HasKey("Id"); + + b.HasIndex("Slug") + .IsUnique() + .HasDatabaseName("uq_cms_site_slug"); + + b.ToTable("cms_site", (string)null); + + b.HasData( + new + { + Id = new Guid("00000000-0000-7000-8002-000000000001"), + CreatedAt = new DateTimeOffset(new DateTime(2026, 11, 2, 0, 0, 0, 0, DateTimeKind.Unspecified), new TimeSpan(0, 0, 0, 0, 0)), + DefaultLocale = "en", + IsDefault = true, + Name = "Default", + Slug = "default" + }); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.SiteLocale", b => + { + b.Property("SiteId") + .HasColumnType("uuid") + .HasColumnName("site_id"); + + b.Property("Code") + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("code"); + + b.Property("Enabled") + .HasColumnType("boolean") + .HasColumnName("enabled"); + + b.Property("Fallbacks") + .HasColumnType("jsonb") + .HasColumnName("fallbacks"); + + b.Property("Position") + .HasColumnType("integer") + .HasColumnName("position"); + + b.HasKey("SiteId", "Code"); + + b.ToTable("cms_site_locale", (string)null); + + b.HasData( + new + { + SiteId = new Guid("00000000-0000-7000-8002-000000000001"), + Code = "en", + Enabled = true, + Position = 0 + }); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.JobAttempt", b => + { + b.HasOne("FluentCMS.Core.Storage.Entities.Job", null) + .WithMany() + .HasForeignKey("JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Role", b => + { + b.HasOne("FluentCMS.Core.Storage.Entities.Site", null) + .WithMany() + .HasForeignKey("SiteId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.RoleGrant", b => + { + b.HasOne("FluentCMS.Core.Storage.Entities.Role", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.SiteLocale", b => + { + b.HasOne("FluentCMS.Core.Storage.Entities.Site", null) + .WithMany() + .HasForeignKey("SiteId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/FluentCMS.Data.PostgreSql/Migrations/20261102202439_AddJobs.cs b/src/FluentCMS.Data.PostgreSql/Migrations/20261102202439_AddJobs.cs new file mode 100644 index 000000000..86ec8a8fb --- /dev/null +++ b/src/FluentCMS.Data.PostgreSql/Migrations/20261102202439_AddJobs.cs @@ -0,0 +1,107 @@ +// +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace FluentCMS.Data.PostgreSql.Migrations +{ + /// + public partial class AddJobs : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "cms_job", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + type = table.Column(type: "character varying(128)", maxLength: 128, nullable: false), + payload = table.Column(type: "jsonb", nullable: false), + queue = table.Column(type: "character varying(64)", maxLength: 64, nullable: false), + site_id = table.Column(type: "uuid", nullable: true), + run_at = table.Column(type: "timestamp with time zone", nullable: false), + priority = table.Column(type: "integer", nullable: false), + status = table.Column(type: "character varying(16)", maxLength: 16, nullable: false), + attempts = table.Column(type: "integer", nullable: false), + max_attempts = table.Column(type: "integer", nullable: false), + lease_until = table.Column(type: "timestamp with time zone", nullable: true), + lease_owner = table.Column(type: "character varying(128)", maxLength: 128, nullable: true), + last_error = table.Column(type: "character varying(2000)", maxLength: 2000, nullable: true), + created_at = table.Column(type: "timestamp with time zone", nullable: false), + created_by = table.Column(type: "uuid", nullable: true), + origin = table.Column(type: "character varying(16)", maxLength: 16, nullable: false), + completed_at = table.Column(type: "timestamp with time zone", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_cms_job", x => x.id); + }); + + migrationBuilder.CreateTable( + name: "cms_recurring_job", + columns: table => new + { + name = table.Column(type: "character varying(128)", maxLength: 128, nullable: false), + cron = table.Column(type: "character varying(128)", maxLength: 128, nullable: false), + type = table.Column(type: "character varying(128)", maxLength: 128, nullable: false), + payload = table.Column(type: "jsonb", nullable: false), + queue = table.Column(type: "character varying(64)", maxLength: 64, nullable: false), + next_run_at = table.Column(type: "timestamp with time zone", nullable: false), + enabled = table.Column(type: "boolean", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_cms_recurring_job", x => x.name); + }); + + migrationBuilder.CreateTable( + name: "cms_job_attempt", + columns: table => new + { + job_id = table.Column(type: "uuid", nullable: false), + attempt = table.Column(type: "integer", nullable: false), + started_at = table.Column(type: "timestamp with time zone", nullable: false), + finished_at = table.Column(type: "timestamp with time zone", nullable: true), + outcome = table.Column(type: "character varying(16)", maxLength: 16, nullable: true), + error = table.Column(type: "character varying(2000)", maxLength: 2000, nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_cms_job_attempt", x => new { x.job_id, x.attempt }); + table.ForeignKey( + name: "FK_cms_job_attempt_cms_job_job_id", + column: x => x.job_id, + principalTable: "cms_job", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "ix_cms_job_claim", + table: "cms_job", + columns: new[] { "queue", "priority", "run_at", "id" }, + descending: new[] { false, true, false, false }, + filter: "status IN ('pending', 'running')"); + + migrationBuilder.CreateIndex( + name: "ix_cms_job_status_lease", + table: "cms_job", + columns: new[] { "status", "lease_until" }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "cms_job_attempt"); + + migrationBuilder.DropTable( + name: "cms_recurring_job"); + + migrationBuilder.DropTable( + name: "cms_job"); + } + } +} diff --git a/src/FluentCMS.Data.PostgreSql/Migrations/SystemDbContextModelSnapshot.cs b/src/FluentCMS.Data.PostgreSql/Migrations/SystemDbContextModelSnapshot.cs index 1f3e4cbaa..ea9a86439 100644 --- a/src/FluentCMS.Data.PostgreSql/Migrations/SystemDbContextModelSnapshot.cs +++ b/src/FluentCMS.Data.PostgreSql/Migrations/SystemDbContextModelSnapshot.cs @@ -175,6 +175,177 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("cms_document", (string)null); }); + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Job", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Attempts") + .HasColumnType("integer") + .HasColumnName("attempts"); + + b.Property("CompletedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("completed_at"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_at"); + + b.Property("CreatedBy") + .HasColumnType("uuid") + .HasColumnName("created_by"); + + b.Property("LastError") + .HasMaxLength(2000) + .HasColumnType("character varying(2000)") + .HasColumnName("last_error"); + + b.Property("LeaseOwner") + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("lease_owner"); + + b.Property("LeaseUntil") + .HasColumnType("timestamp with time zone") + .HasColumnName("lease_until"); + + b.Property("MaxAttempts") + .HasColumnType("integer") + .HasColumnName("max_attempts"); + + b.Property("Origin") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("origin"); + + b.Property("Payload") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("payload"); + + b.Property("Priority") + .HasColumnType("integer") + .HasColumnName("priority"); + + b.Property("Queue") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("queue"); + + b.Property("RunAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("run_at"); + + b.Property("SiteId") + .HasColumnType("uuid") + .HasColumnName("site_id"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("status"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("type"); + + b.HasKey("Id"); + + b.HasIndex("Status", "LeaseUntil") + .HasDatabaseName("ix_cms_job_status_lease"); + + b.HasIndex("Queue", "Priority", "RunAt", "Id") + .IsDescending(false, true, false, false) + .HasDatabaseName("ix_cms_job_claim") + .HasFilter("status IN ('pending', 'running')"); + + b.ToTable("cms_job", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.JobAttempt", b => + { + b.Property("JobId") + .HasColumnType("uuid") + .HasColumnName("job_id"); + + b.Property("Attempt") + .HasColumnType("integer") + .HasColumnName("attempt"); + + b.Property("Error") + .HasMaxLength(2000) + .HasColumnType("character varying(2000)") + .HasColumnName("error"); + + b.Property("FinishedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("finished_at"); + + b.Property("Outcome") + .HasMaxLength(16) + .HasColumnType("character varying(16)") + .HasColumnName("outcome"); + + b.Property("StartedAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("started_at"); + + b.HasKey("JobId", "Attempt"); + + b.ToTable("cms_job_attempt", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.RecurringJob", b => + { + b.Property("Name") + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("name"); + + b.Property("Cron") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("cron"); + + b.Property("Enabled") + .HasColumnType("boolean") + .HasColumnName("enabled"); + + b.Property("NextRunAt") + .HasColumnType("timestamp with time zone") + .HasColumnName("next_run_at"); + + b.Property("Payload") + .IsRequired() + .HasColumnType("jsonb") + .HasColumnName("payload"); + + b.Property("Queue") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("character varying(64)") + .HasColumnName("queue"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("character varying(128)") + .HasColumnName("type"); + + b.HasKey("Name"); + + b.ToTable("cms_recurring_job", (string)null); + }); + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Reference", b => { b.Property("Id") @@ -539,6 +710,15 @@ protected override void BuildModel(ModelBuilder modelBuilder) }); }); + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.JobAttempt", b => + { + b.HasOne("FluentCMS.Core.Storage.Entities.Job", null) + .WithMany() + .HasForeignKey("JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Role", b => { b.HasOne("FluentCMS.Core.Storage.Entities.Site", null) diff --git a/src/FluentCMS.Data.PostgreSql/PostgreSqlJobClaimDialect.cs b/src/FluentCMS.Data.PostgreSql/PostgreSqlJobClaimDialect.cs index 8825b3c40..ed21f437a 100644 --- a/src/FluentCMS.Data.PostgreSql/PostgreSqlJobClaimDialect.cs +++ b/src/FluentCMS.Data.PostgreSql/PostgreSqlJobClaimDialect.cs @@ -20,7 +20,7 @@ UPDATE cms_job SELECT id FROM cms_job WHERE queue = {claim.Queue} - AND status IN ({JobClaimStatus.Pending}, {JobClaimStatus.Running}) + AND status IN ('pending', 'running') AND run_at <= {now} AND attempts < max_attempts AND (lease_until IS NULL OR lease_until < {now}) diff --git a/src/FluentCMS.Data.SqlServer/Migrations/20261102202445_AddJobs.Designer.cs b/src/FluentCMS.Data.SqlServer/Migrations/20261102202445_AddJobs.Designer.cs new file mode 100644 index 000000000..8b71c541e --- /dev/null +++ b/src/FluentCMS.Data.SqlServer/Migrations/20261102202445_AddJobs.Designer.cs @@ -0,0 +1,768 @@ +// +using System; +using FluentCMS.Core.Storage; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace FluentCMS.Data.SqlServer.Migrations +{ + [DbContext(typeof(SystemDbContext))] + [Migration("20261102202445_AddJobs")] + partial class AddJobs + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.14") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.ContentVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasColumnName("id"); + + b.Property("AuthorId") + .HasColumnType("uniqueidentifier") + .HasColumnName("author_id"); + + b.Property("ClientId") + .HasColumnType("nvarchar(max)") + .HasColumnName("client_id"); + + b.Property("CreatedAt") + .HasColumnType("datetimeoffset") + .HasColumnName("created_at"); + + b.Property("DocumentId") + .HasColumnType("uniqueidentifier") + .HasColumnName("document_id"); + + b.Property("Locale") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)") + .HasColumnName("locale"); + + b.Property("Message") + .HasColumnType("nvarchar(max)") + .HasColumnName("message"); + + b.Property("Number") + .HasColumnType("int") + .HasColumnName("number"); + + b.Property("Origin") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)") + .HasColumnName("origin"); + + b.Property("Snapshot") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("snapshot"); + + b.Property("StatusAfter") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)") + .HasColumnName("status_after"); + + b.HasKey("Id"); + + b.HasIndex("DocumentId", "Locale", "Number") + .IsUnique() + .HasDatabaseName("uq_cms_version_document_locale_number"); + + b.ToTable("cms_version", null, t => + { + t.HasCheckConstraint("ck_cms_version_snapshot_json", "ISJSON([snapshot]) = 1"); + }); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Document", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasColumnName("id"); + + b.Property("CreatedAt") + .HasColumnType("datetimeoffset") + .HasColumnName("created_at"); + + b.Property("CreatedBy") + .HasColumnType("uniqueidentifier") + .HasColumnName("created_by"); + + b.Property("DeletedAt") + .HasColumnType("datetimeoffset") + .HasColumnName("deleted_at"); + + b.Property("DeletedBy") + .HasColumnType("uniqueidentifier") + .HasColumnName("deleted_by"); + + b.Property("DocumentId") + .HasColumnType("uniqueidentifier") + .HasColumnName("document_id"); + + b.Property("DraftVersion") + .HasColumnType("int") + .HasColumnName("draft_version"); + + b.Property("FirstPublishedAt") + .HasColumnType("datetimeoffset") + .HasColumnName("first_published_at"); + + b.Property("Locale") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)") + .HasColumnName("locale"); + + b.Property("PublishedAt") + .HasColumnType("datetimeoffset") + .HasColumnName("published_at"); + + b.Property("PublishedVersion") + .HasColumnType("int") + .HasColumnName("published_version"); + + b.Property("SchedulePublishAt") + .HasColumnType("datetimeoffset") + .HasColumnName("schedule_publish_at"); + + b.Property("ScheduleUnpublishAt") + .HasColumnType("datetimeoffset") + .HasColumnName("schedule_unpublish_at"); + + b.Property("SiteId") + .HasColumnType("uniqueidentifier") + .HasColumnName("site_id"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)") + .HasColumnName("status"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)") + .HasColumnName("type"); + + b.Property("UpdatedAt") + .HasColumnType("datetimeoffset") + .HasColumnName("updated_at"); + + b.Property("UpdatedBy") + .HasColumnType("uniqueidentifier") + .HasColumnName("updated_by"); + + b.HasKey("Id"); + + b.HasIndex("DocumentId", "Locale") + .IsUnique() + .HasDatabaseName("uq_cms_document_document_locale"); + + b.ToTable("cms_document", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Job", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasColumnName("id"); + + b.Property("Attempts") + .HasColumnType("int") + .HasColumnName("attempts"); + + b.Property("CompletedAt") + .HasColumnType("datetimeoffset") + .HasColumnName("completed_at"); + + b.Property("CreatedAt") + .HasColumnType("datetimeoffset") + .HasColumnName("created_at"); + + b.Property("CreatedBy") + .HasColumnType("uniqueidentifier") + .HasColumnName("created_by"); + + b.Property("LastError") + .HasMaxLength(2000) + .HasColumnType("nvarchar(2000)") + .HasColumnName("last_error"); + + b.Property("LeaseOwner") + .HasMaxLength(128) + .HasColumnType("nvarchar(128)") + .HasColumnName("lease_owner"); + + b.Property("LeaseUntil") + .HasColumnType("datetimeoffset") + .HasColumnName("lease_until"); + + b.Property("MaxAttempts") + .HasColumnType("int") + .HasColumnName("max_attempts"); + + b.Property("Origin") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)") + .HasColumnName("origin"); + + b.Property("Payload") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("payload"); + + b.Property("Priority") + .HasColumnType("int") + .HasColumnName("priority"); + + b.Property("Queue") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)") + .HasColumnName("queue"); + + b.Property("RunAt") + .HasColumnType("datetimeoffset") + .HasColumnName("run_at"); + + b.Property("SiteId") + .HasColumnType("uniqueidentifier") + .HasColumnName("site_id"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)") + .HasColumnName("status"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)") + .HasColumnName("type"); + + b.HasKey("Id"); + + b.HasIndex("Status", "LeaseUntil") + .HasDatabaseName("ix_cms_job_status_lease"); + + b.HasIndex("Queue", "Priority", "RunAt", "Id") + .IsDescending(false, true, false, false) + .HasDatabaseName("ix_cms_job_claim") + .HasFilter("status IN ('pending', 'running')"); + + b.ToTable("cms_job", null, t => + { + t.HasCheckConstraint("ck_cms_job_payload_json", "ISJSON([payload]) = 1"); + }); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.JobAttempt", b => + { + b.Property("JobId") + .HasColumnType("uniqueidentifier") + .HasColumnName("job_id"); + + b.Property("Attempt") + .HasColumnType("int") + .HasColumnName("attempt"); + + b.Property("Error") + .HasMaxLength(2000) + .HasColumnType("nvarchar(2000)") + .HasColumnName("error"); + + b.Property("FinishedAt") + .HasColumnType("datetimeoffset") + .HasColumnName("finished_at"); + + b.Property("Outcome") + .HasMaxLength(16) + .HasColumnType("nvarchar(16)") + .HasColumnName("outcome"); + + b.Property("StartedAt") + .HasColumnType("datetimeoffset") + .HasColumnName("started_at"); + + b.HasKey("JobId", "Attempt"); + + b.ToTable("cms_job_attempt", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.RecurringJob", b => + { + b.Property("Name") + .HasMaxLength(128) + .HasColumnType("nvarchar(128)") + .HasColumnName("name"); + + b.Property("Cron") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)") + .HasColumnName("cron"); + + b.Property("Enabled") + .HasColumnType("bit") + .HasColumnName("enabled"); + + b.Property("NextRunAt") + .HasColumnType("datetimeoffset") + .HasColumnName("next_run_at"); + + b.Property("Payload") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("payload"); + + b.Property("Queue") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)") + .HasColumnName("queue"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)") + .HasColumnName("type"); + + b.HasKey("Name"); + + b.ToTable("cms_recurring_job", null, t => + { + t.HasCheckConstraint("ck_cms_recurring_job_payload_json", "ISJSON([payload]) = 1"); + }); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Reference", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasColumnName("id"); + + b.Property("Field") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)") + .HasColumnName("field"); + + b.Property("Locale") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)") + .HasColumnName("locale"); + + b.Property("Position") + .HasColumnType("int") + .HasColumnName("position"); + + b.Property("SourceDocumentId") + .HasColumnType("uniqueidentifier") + .HasColumnName("source_document_id"); + + b.Property("SourceType") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)") + .HasColumnName("source_type"); + + b.Property("State") + .HasColumnType("smallint") + .HasColumnName("state"); + + b.Property("TargetId") + .HasColumnType("uniqueidentifier") + .HasColumnName("target_id"); + + b.Property("TargetType") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)") + .HasColumnName("target_type"); + + b.HasKey("Id"); + + b.HasIndex("TargetType", "TargetId") + .HasDatabaseName("ix_cms_reference_target"); + + b.HasIndex("SourceType", "SourceDocumentId", "State", "Locale", "Field", "Position") + .IsUnique() + .HasDatabaseName("uq_cms_reference_source"); + + b.ToTable("cms_reference", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Role", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasColumnName("id"); + + b.Property("BuiltIn") + .HasColumnType("bit") + .HasColumnName("built_in"); + + b.Property("Description") + .HasMaxLength(512) + .HasColumnType("nvarchar(512)") + .HasColumnName("description"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)") + .HasColumnName("name"); + + b.Property("PermissionsVersion") + .IsConcurrencyToken() + .HasColumnType("bigint") + .HasColumnName("permissions_version"); + + b.Property("SiteId") + .HasColumnType("uniqueidentifier") + .HasColumnName("site_id"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique() + .HasDatabaseName("uq_cms_role_name") + .HasFilter("site_id IS NULL"); + + b.HasIndex("SiteId", "Name") + .IsUnique() + .HasDatabaseName("uq_cms_role_site_name") + .HasFilter("site_id IS NOT NULL"); + + b.ToTable("cms_role", (string)null); + + b.HasData( + new + { + Id = new Guid("00000000-0000-7000-8000-000000000001"), + BuiltIn = true, + Description = "Holds every permission.", + Name = "Super Admin", + PermissionsVersion = 1L + }, + new + { + Id = new Guid("00000000-0000-7000-8000-000000000002"), + BuiltIn = true, + Description = "Creates, edits, publishes and removes any content.", + Name = "Editor", + PermissionsVersion = 1L + }, + new + { + Id = new Guid("00000000-0000-7000-8000-000000000003"), + BuiltIn = true, + Description = "Creates content, and reads, edits or removes only their own entries.", + Name = "Author", + PermissionsVersion = 1L + }, + new + { + Id = new Guid("00000000-0000-7000-8000-000000000004"), + BuiltIn = true, + Description = "The anonymous visitor. Grants here open public Delivery access.", + Name = "Public", + PermissionsVersion = 1L + }, + new + { + Id = new Guid("00000000-0000-7000-8000-000000000005"), + BuiltIn = true, + Description = "The default role of a registered member.", + Name = "Member", + PermissionsVersion = 1L + }, + new + { + Id = new Guid("00000000-0000-7000-8000-000000000006"), + BuiltIn = true, + Description = "The anonymous visitor. Grants here open public Delivery access.", + Name = "Public", + PermissionsVersion = 1L, + SiteId = new Guid("00000000-0000-7000-8002-000000000001") + }); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.RoleGrant", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasColumnName("id"); + + b.Property("Constraints") + .HasColumnType("nvarchar(max)") + .HasColumnName("constraints"); + + b.Property("Permission") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)") + .HasColumnName("permission"); + + b.Property("RoleId") + .HasColumnType("uniqueidentifier") + .HasColumnName("role_id"); + + b.HasKey("Id"); + + b.HasIndex("RoleId", "Permission") + .HasDatabaseName("ix_cms_role_permission_role_permission"); + + b.ToTable("cms_role_permission", null, t => + { + t.HasCheckConstraint("ck_cms_role_permission_constraints_json", "ISJSON([constraints]) = 1"); + }); + + b.HasData( + new + { + Id = new Guid("00000000-0000-7000-8001-000000000001"), + Permission = "*", + RoleId = new Guid("00000000-0000-7000-8000-000000000001") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000002"), + Permission = "content.*.create", + RoleId = new Guid("00000000-0000-7000-8000-000000000002") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000003"), + Permission = "content.*.read", + RoleId = new Guid("00000000-0000-7000-8000-000000000002") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000004"), + Permission = "content.*.update", + RoleId = new Guid("00000000-0000-7000-8000-000000000002") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000005"), + Permission = "content.*.delete", + RoleId = new Guid("00000000-0000-7000-8000-000000000002") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000006"), + Permission = "content.*.publish", + RoleId = new Guid("00000000-0000-7000-8000-000000000002") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000007"), + Permission = "content.*.unpublish", + RoleId = new Guid("00000000-0000-7000-8000-000000000002") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000008"), + Permission = "content.*.restore", + RoleId = new Guid("00000000-0000-7000-8000-000000000002") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000009"), + Permission = "schema.read", + RoleId = new Guid("00000000-0000-7000-8000-000000000002") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000010"), + Permission = "content.*.create", + RoleId = new Guid("00000000-0000-7000-8000-000000000003") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000011"), + Constraints = "{\"rowRule\":\"own-entries\"}", + Permission = "content.*.read", + RoleId = new Guid("00000000-0000-7000-8000-000000000003") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000012"), + Constraints = "{\"rowRule\":\"own-entries\"}", + Permission = "content.*.update", + RoleId = new Guid("00000000-0000-7000-8000-000000000003") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000013"), + Constraints = "{\"rowRule\":\"own-entries\"}", + Permission = "content.*.delete", + RoleId = new Guid("00000000-0000-7000-8000-000000000003") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000014"), + Permission = "schema.read", + RoleId = new Guid("00000000-0000-7000-8000-000000000003") + }); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Site", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasColumnName("id"); + + b.Property("CreatedAt") + .HasColumnType("datetimeoffset") + .HasColumnName("created_at"); + + b.Property("DefaultLocale") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)") + .HasColumnName("default_locale"); + + b.Property("IsDefault") + .HasColumnType("bit") + .HasColumnName("is_default"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)") + .HasColumnName("name"); + + b.Property("Slug") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)") + .HasColumnName("slug"); + + b.HasKey("Id"); + + b.HasIndex("Slug") + .IsUnique() + .HasDatabaseName("uq_cms_site_slug"); + + b.ToTable("cms_site", (string)null); + + b.HasData( + new + { + Id = new Guid("00000000-0000-7000-8002-000000000001"), + CreatedAt = new DateTimeOffset(new DateTime(2026, 11, 2, 0, 0, 0, 0, DateTimeKind.Unspecified), new TimeSpan(0, 0, 0, 0, 0)), + DefaultLocale = "en", + IsDefault = true, + Name = "Default", + Slug = "default" + }); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.SiteLocale", b => + { + b.Property("SiteId") + .HasColumnType("uniqueidentifier") + .HasColumnName("site_id"); + + b.Property("Code") + .HasMaxLength(16) + .HasColumnType("nvarchar(16)") + .HasColumnName("code"); + + b.Property("Enabled") + .HasColumnType("bit") + .HasColumnName("enabled"); + + b.Property("Fallbacks") + .HasColumnType("nvarchar(max)") + .HasColumnName("fallbacks"); + + b.Property("Position") + .HasColumnType("int") + .HasColumnName("position"); + + b.HasKey("SiteId", "Code"); + + b.ToTable("cms_site_locale", null, t => + { + t.HasCheckConstraint("ck_cms_site_locale_fallbacks_json", "ISJSON([fallbacks]) = 1"); + }); + + b.HasData( + new + { + SiteId = new Guid("00000000-0000-7000-8002-000000000001"), + Code = "en", + Enabled = true, + Position = 0 + }); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.JobAttempt", b => + { + b.HasOne("FluentCMS.Core.Storage.Entities.Job", null) + .WithMany() + .HasForeignKey("JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Role", b => + { + b.HasOne("FluentCMS.Core.Storage.Entities.Site", null) + .WithMany() + .HasForeignKey("SiteId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.RoleGrant", b => + { + b.HasOne("FluentCMS.Core.Storage.Entities.Role", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.SiteLocale", b => + { + b.HasOne("FluentCMS.Core.Storage.Entities.Site", null) + .WithMany() + .HasForeignKey("SiteId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/FluentCMS.Data.SqlServer/Migrations/20261102202445_AddJobs.cs b/src/FluentCMS.Data.SqlServer/Migrations/20261102202445_AddJobs.cs new file mode 100644 index 000000000..63e65f680 --- /dev/null +++ b/src/FluentCMS.Data.SqlServer/Migrations/20261102202445_AddJobs.cs @@ -0,0 +1,109 @@ +// +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace FluentCMS.Data.SqlServer.Migrations +{ + /// + public partial class AddJobs : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "cms_job", + columns: table => new + { + id = table.Column(type: "uniqueidentifier", nullable: false), + type = table.Column(type: "nvarchar(128)", maxLength: 128, nullable: false), + payload = table.Column(type: "nvarchar(max)", nullable: false), + queue = table.Column(type: "nvarchar(64)", maxLength: 64, nullable: false), + site_id = table.Column(type: "uniqueidentifier", nullable: true), + run_at = table.Column(type: "datetimeoffset", nullable: false), + priority = table.Column(type: "int", nullable: false), + status = table.Column(type: "nvarchar(16)", maxLength: 16, nullable: false), + attempts = table.Column(type: "int", nullable: false), + max_attempts = table.Column(type: "int", nullable: false), + lease_until = table.Column(type: "datetimeoffset", nullable: true), + lease_owner = table.Column(type: "nvarchar(128)", maxLength: 128, nullable: true), + last_error = table.Column(type: "nvarchar(2000)", maxLength: 2000, nullable: true), + created_at = table.Column(type: "datetimeoffset", nullable: false), + created_by = table.Column(type: "uniqueidentifier", nullable: true), + origin = table.Column(type: "nvarchar(16)", maxLength: 16, nullable: false), + completed_at = table.Column(type: "datetimeoffset", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_cms_job", x => x.id); + table.CheckConstraint("ck_cms_job_payload_json", "ISJSON([payload]) = 1"); + }); + + migrationBuilder.CreateTable( + name: "cms_recurring_job", + columns: table => new + { + name = table.Column(type: "nvarchar(128)", maxLength: 128, nullable: false), + cron = table.Column(type: "nvarchar(128)", maxLength: 128, nullable: false), + type = table.Column(type: "nvarchar(128)", maxLength: 128, nullable: false), + payload = table.Column(type: "nvarchar(max)", nullable: false), + queue = table.Column(type: "nvarchar(64)", maxLength: 64, nullable: false), + next_run_at = table.Column(type: "datetimeoffset", nullable: false), + enabled = table.Column(type: "bit", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_cms_recurring_job", x => x.name); + table.CheckConstraint("ck_cms_recurring_job_payload_json", "ISJSON([payload]) = 1"); + }); + + migrationBuilder.CreateTable( + name: "cms_job_attempt", + columns: table => new + { + job_id = table.Column(type: "uniqueidentifier", nullable: false), + attempt = table.Column(type: "int", nullable: false), + started_at = table.Column(type: "datetimeoffset", nullable: false), + finished_at = table.Column(type: "datetimeoffset", nullable: true), + outcome = table.Column(type: "nvarchar(16)", maxLength: 16, nullable: true), + error = table.Column(type: "nvarchar(2000)", maxLength: 2000, nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_cms_job_attempt", x => new { x.job_id, x.attempt }); + table.ForeignKey( + name: "FK_cms_job_attempt_cms_job_job_id", + column: x => x.job_id, + principalTable: "cms_job", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "ix_cms_job_claim", + table: "cms_job", + columns: new[] { "queue", "priority", "run_at", "id" }, + descending: new[] { false, true, false, false }, + filter: "status IN ('pending', 'running')"); + + migrationBuilder.CreateIndex( + name: "ix_cms_job_status_lease", + table: "cms_job", + columns: new[] { "status", "lease_until" }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "cms_job_attempt"); + + migrationBuilder.DropTable( + name: "cms_recurring_job"); + + migrationBuilder.DropTable( + name: "cms_job"); + } + } +} diff --git a/src/FluentCMS.Data.SqlServer/Migrations/SystemDbContextModelSnapshot.cs b/src/FluentCMS.Data.SqlServer/Migrations/SystemDbContextModelSnapshot.cs index 81201afa6..7f8437fcb 100644 --- a/src/FluentCMS.Data.SqlServer/Migrations/SystemDbContextModelSnapshot.cs +++ b/src/FluentCMS.Data.SqlServer/Migrations/SystemDbContextModelSnapshot.cs @@ -178,6 +178,183 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("cms_document", (string)null); }); + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Job", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier") + .HasColumnName("id"); + + b.Property("Attempts") + .HasColumnType("int") + .HasColumnName("attempts"); + + b.Property("CompletedAt") + .HasColumnType("datetimeoffset") + .HasColumnName("completed_at"); + + b.Property("CreatedAt") + .HasColumnType("datetimeoffset") + .HasColumnName("created_at"); + + b.Property("CreatedBy") + .HasColumnType("uniqueidentifier") + .HasColumnName("created_by"); + + b.Property("LastError") + .HasMaxLength(2000) + .HasColumnType("nvarchar(2000)") + .HasColumnName("last_error"); + + b.Property("LeaseOwner") + .HasMaxLength(128) + .HasColumnType("nvarchar(128)") + .HasColumnName("lease_owner"); + + b.Property("LeaseUntil") + .HasColumnType("datetimeoffset") + .HasColumnName("lease_until"); + + b.Property("MaxAttempts") + .HasColumnType("int") + .HasColumnName("max_attempts"); + + b.Property("Origin") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)") + .HasColumnName("origin"); + + b.Property("Payload") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("payload"); + + b.Property("Priority") + .HasColumnType("int") + .HasColumnName("priority"); + + b.Property("Queue") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)") + .HasColumnName("queue"); + + b.Property("RunAt") + .HasColumnType("datetimeoffset") + .HasColumnName("run_at"); + + b.Property("SiteId") + .HasColumnType("uniqueidentifier") + .HasColumnName("site_id"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("nvarchar(16)") + .HasColumnName("status"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)") + .HasColumnName("type"); + + b.HasKey("Id"); + + b.HasIndex("Status", "LeaseUntil") + .HasDatabaseName("ix_cms_job_status_lease"); + + b.HasIndex("Queue", "Priority", "RunAt", "Id") + .IsDescending(false, true, false, false) + .HasDatabaseName("ix_cms_job_claim") + .HasFilter("status IN ('pending', 'running')"); + + b.ToTable("cms_job", null, t => + { + t.HasCheckConstraint("ck_cms_job_payload_json", "ISJSON([payload]) = 1"); + }); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.JobAttempt", b => + { + b.Property("JobId") + .HasColumnType("uniqueidentifier") + .HasColumnName("job_id"); + + b.Property("Attempt") + .HasColumnType("int") + .HasColumnName("attempt"); + + b.Property("Error") + .HasMaxLength(2000) + .HasColumnType("nvarchar(2000)") + .HasColumnName("error"); + + b.Property("FinishedAt") + .HasColumnType("datetimeoffset") + .HasColumnName("finished_at"); + + b.Property("Outcome") + .HasMaxLength(16) + .HasColumnType("nvarchar(16)") + .HasColumnName("outcome"); + + b.Property("StartedAt") + .HasColumnType("datetimeoffset") + .HasColumnName("started_at"); + + b.HasKey("JobId", "Attempt"); + + b.ToTable("cms_job_attempt", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.RecurringJob", b => + { + b.Property("Name") + .HasMaxLength(128) + .HasColumnType("nvarchar(128)") + .HasColumnName("name"); + + b.Property("Cron") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)") + .HasColumnName("cron"); + + b.Property("Enabled") + .HasColumnType("bit") + .HasColumnName("enabled"); + + b.Property("NextRunAt") + .HasColumnType("datetimeoffset") + .HasColumnName("next_run_at"); + + b.Property("Payload") + .IsRequired() + .HasColumnType("nvarchar(max)") + .HasColumnName("payload"); + + b.Property("Queue") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("nvarchar(64)") + .HasColumnName("queue"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("nvarchar(128)") + .HasColumnName("type"); + + b.HasKey("Name"); + + b.ToTable("cms_recurring_job", null, t => + { + t.HasCheckConstraint("ck_cms_recurring_job_payload_json", "ISJSON([payload]) = 1"); + }); + }); + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Reference", b => { b.Property("Id") @@ -548,6 +725,15 @@ protected override void BuildModel(ModelBuilder modelBuilder) }); }); + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.JobAttempt", b => + { + b.HasOne("FluentCMS.Core.Storage.Entities.Job", null) + .WithMany() + .HasForeignKey("JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Role", b => { b.HasOne("FluentCMS.Core.Storage.Entities.Site", null) diff --git a/src/FluentCMS.Data.SqlServer/SqlServerJobClaimDialect.cs b/src/FluentCMS.Data.SqlServer/SqlServerJobClaimDialect.cs index 9bd1efbfd..daf3ea3e5 100644 --- a/src/FluentCMS.Data.SqlServer/SqlServerJobClaimDialect.cs +++ b/src/FluentCMS.Data.SqlServer/SqlServerJobClaimDialect.cs @@ -17,7 +17,7 @@ WITH next_job AS ( SELECT TOP (1) status, lease_owner, lease_until, attempts, id FROM cms_job WITH (UPDLOCK, READPAST, ROWLOCK) WHERE queue = {claim.Queue} - AND status IN ({JobClaimStatus.Pending}, {JobClaimStatus.Running}) + AND status IN ('pending', 'running') AND run_at <= {claim.Now} AND attempts < max_attempts AND (lease_until IS NULL OR lease_until < {claim.Now}) diff --git a/src/FluentCMS.Data.Sqlite/Migrations/20261102202434_AddJobs.Designer.cs b/src/FluentCMS.Data.Sqlite/Migrations/20261102202434_AddJobs.Designer.cs new file mode 100644 index 000000000..7feffaded --- /dev/null +++ b/src/FluentCMS.Data.Sqlite/Migrations/20261102202434_AddJobs.Designer.cs @@ -0,0 +1,763 @@ +// +using System; +using FluentCMS.Core.Storage; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace FluentCMS.Data.Sqlite.Migrations +{ + [DbContext(typeof(SystemDbContext))] + [Migration("20261102202434_AddJobs")] + partial class AddJobs + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "10.0.14"); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.ContentVersion", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT") + .HasColumnName("id"); + + b.Property("AuthorId") + .HasColumnType("TEXT") + .HasColumnName("author_id"); + + b.Property("ClientId") + .HasColumnType("TEXT") + .HasColumnName("client_id"); + + b.Property("CreatedAt") + .HasColumnType("INTEGER") + .HasColumnName("created_at"); + + b.Property("DocumentId") + .HasColumnType("TEXT") + .HasColumnName("document_id"); + + b.Property("Locale") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("TEXT") + .HasColumnName("locale"); + + b.Property("Message") + .HasColumnType("TEXT") + .HasColumnName("message"); + + b.Property("Number") + .HasColumnType("INTEGER") + .HasColumnName("number"); + + b.Property("Origin") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("TEXT") + .HasColumnName("origin"); + + b.Property("Snapshot") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("snapshot"); + + b.Property("StatusAfter") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("TEXT") + .HasColumnName("status_after"); + + b.HasKey("Id"); + + b.HasIndex("DocumentId", "Locale", "Number") + .IsUnique() + .HasDatabaseName("uq_cms_version_document_locale_number"); + + b.ToTable("cms_version", null, t => + { + t.HasCheckConstraint("ck_cms_version_snapshot_json", "json_valid(\"snapshot\")"); + }); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Document", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT") + .HasColumnName("id"); + + b.Property("CreatedAt") + .HasColumnType("INTEGER") + .HasColumnName("created_at"); + + b.Property("CreatedBy") + .HasColumnType("TEXT") + .HasColumnName("created_by"); + + b.Property("DeletedAt") + .HasColumnType("INTEGER") + .HasColumnName("deleted_at"); + + b.Property("DeletedBy") + .HasColumnType("TEXT") + .HasColumnName("deleted_by"); + + b.Property("DocumentId") + .HasColumnType("TEXT") + .HasColumnName("document_id"); + + b.Property("DraftVersion") + .HasColumnType("INTEGER") + .HasColumnName("draft_version"); + + b.Property("FirstPublishedAt") + .HasColumnType("INTEGER") + .HasColumnName("first_published_at"); + + b.Property("Locale") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("TEXT") + .HasColumnName("locale"); + + b.Property("PublishedAt") + .HasColumnType("INTEGER") + .HasColumnName("published_at"); + + b.Property("PublishedVersion") + .HasColumnType("INTEGER") + .HasColumnName("published_version"); + + b.Property("SchedulePublishAt") + .HasColumnType("INTEGER") + .HasColumnName("schedule_publish_at"); + + b.Property("ScheduleUnpublishAt") + .HasColumnType("INTEGER") + .HasColumnName("schedule_unpublish_at"); + + b.Property("SiteId") + .HasColumnType("TEXT") + .HasColumnName("site_id"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("TEXT") + .HasColumnName("status"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT") + .HasColumnName("type"); + + b.Property("UpdatedAt") + .HasColumnType("INTEGER") + .HasColumnName("updated_at"); + + b.Property("UpdatedBy") + .HasColumnType("TEXT") + .HasColumnName("updated_by"); + + b.HasKey("Id"); + + b.HasIndex("DocumentId", "Locale") + .IsUnique() + .HasDatabaseName("uq_cms_document_document_locale"); + + b.ToTable("cms_document", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Job", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT") + .HasColumnName("id"); + + b.Property("Attempts") + .HasColumnType("INTEGER") + .HasColumnName("attempts"); + + b.Property("CompletedAt") + .HasColumnType("INTEGER") + .HasColumnName("completed_at"); + + b.Property("CreatedAt") + .HasColumnType("INTEGER") + .HasColumnName("created_at"); + + b.Property("CreatedBy") + .HasColumnType("TEXT") + .HasColumnName("created_by"); + + b.Property("LastError") + .HasMaxLength(2000) + .HasColumnType("TEXT") + .HasColumnName("last_error"); + + b.Property("LeaseOwner") + .HasMaxLength(128) + .HasColumnType("TEXT") + .HasColumnName("lease_owner"); + + b.Property("LeaseUntil") + .HasColumnType("INTEGER") + .HasColumnName("lease_until"); + + b.Property("MaxAttempts") + .HasColumnType("INTEGER") + .HasColumnName("max_attempts"); + + b.Property("Origin") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("TEXT") + .HasColumnName("origin"); + + b.Property("Payload") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("payload"); + + b.Property("Priority") + .HasColumnType("INTEGER") + .HasColumnName("priority"); + + b.Property("Queue") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT") + .HasColumnName("queue"); + + b.Property("RunAt") + .HasColumnType("INTEGER") + .HasColumnName("run_at"); + + b.Property("SiteId") + .HasColumnType("TEXT") + .HasColumnName("site_id"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("TEXT") + .HasColumnName("status"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT") + .HasColumnName("type"); + + b.HasKey("Id"); + + b.HasIndex("Status", "LeaseUntil") + .HasDatabaseName("ix_cms_job_status_lease"); + + b.HasIndex("Queue", "Priority", "RunAt", "Id") + .IsDescending(false, true, false, false) + .HasDatabaseName("ix_cms_job_claim") + .HasFilter("status IN ('pending', 'running')"); + + b.ToTable("cms_job", null, t => + { + t.HasCheckConstraint("ck_cms_job_payload_json", "json_valid(\"payload\")"); + }); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.JobAttempt", b => + { + b.Property("JobId") + .HasColumnType("TEXT") + .HasColumnName("job_id"); + + b.Property("Attempt") + .HasColumnType("INTEGER") + .HasColumnName("attempt"); + + b.Property("Error") + .HasMaxLength(2000) + .HasColumnType("TEXT") + .HasColumnName("error"); + + b.Property("FinishedAt") + .HasColumnType("INTEGER") + .HasColumnName("finished_at"); + + b.Property("Outcome") + .HasMaxLength(16) + .HasColumnType("TEXT") + .HasColumnName("outcome"); + + b.Property("StartedAt") + .HasColumnType("INTEGER") + .HasColumnName("started_at"); + + b.HasKey("JobId", "Attempt"); + + b.ToTable("cms_job_attempt", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.RecurringJob", b => + { + b.Property("Name") + .HasMaxLength(128) + .HasColumnType("TEXT") + .HasColumnName("name"); + + b.Property("Cron") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT") + .HasColumnName("cron"); + + b.Property("Enabled") + .HasColumnType("INTEGER") + .HasColumnName("enabled"); + + b.Property("NextRunAt") + .HasColumnType("INTEGER") + .HasColumnName("next_run_at"); + + b.Property("Payload") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("payload"); + + b.Property("Queue") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT") + .HasColumnName("queue"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT") + .HasColumnName("type"); + + b.HasKey("Name"); + + b.ToTable("cms_recurring_job", null, t => + { + t.HasCheckConstraint("ck_cms_recurring_job_payload_json", "json_valid(\"payload\")"); + }); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Reference", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT") + .HasColumnName("id"); + + b.Property("Field") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT") + .HasColumnName("field"); + + b.Property("Locale") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("TEXT") + .HasColumnName("locale"); + + b.Property("Position") + .HasColumnType("INTEGER") + .HasColumnName("position"); + + b.Property("SourceDocumentId") + .HasColumnType("TEXT") + .HasColumnName("source_document_id"); + + b.Property("SourceType") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT") + .HasColumnName("source_type"); + + b.Property("State") + .HasColumnType("INTEGER") + .HasColumnName("state"); + + b.Property("TargetId") + .HasColumnType("TEXT") + .HasColumnName("target_id"); + + b.Property("TargetType") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT") + .HasColumnName("target_type"); + + b.HasKey("Id"); + + b.HasIndex("TargetType", "TargetId") + .HasDatabaseName("ix_cms_reference_target"); + + b.HasIndex("SourceType", "SourceDocumentId", "State", "Locale", "Field", "Position") + .IsUnique() + .HasDatabaseName("uq_cms_reference_source"); + + b.ToTable("cms_reference", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Role", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT") + .HasColumnName("id"); + + b.Property("BuiltIn") + .HasColumnType("INTEGER") + .HasColumnName("built_in"); + + b.Property("Description") + .HasMaxLength(512) + .HasColumnType("TEXT") + .HasColumnName("description"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT") + .HasColumnName("name"); + + b.Property("PermissionsVersion") + .IsConcurrencyToken() + .HasColumnType("INTEGER") + .HasColumnName("permissions_version"); + + b.Property("SiteId") + .HasColumnType("TEXT") + .HasColumnName("site_id"); + + b.HasKey("Id"); + + b.HasIndex("Name") + .IsUnique() + .HasDatabaseName("uq_cms_role_name") + .HasFilter("site_id IS NULL"); + + b.HasIndex("SiteId", "Name") + .IsUnique() + .HasDatabaseName("uq_cms_role_site_name") + .HasFilter("site_id IS NOT NULL"); + + b.ToTable("cms_role", (string)null); + + b.HasData( + new + { + Id = new Guid("00000000-0000-7000-8000-000000000001"), + BuiltIn = true, + Description = "Holds every permission.", + Name = "Super Admin", + PermissionsVersion = 1L + }, + new + { + Id = new Guid("00000000-0000-7000-8000-000000000002"), + BuiltIn = true, + Description = "Creates, edits, publishes and removes any content.", + Name = "Editor", + PermissionsVersion = 1L + }, + new + { + Id = new Guid("00000000-0000-7000-8000-000000000003"), + BuiltIn = true, + Description = "Creates content, and reads, edits or removes only their own entries.", + Name = "Author", + PermissionsVersion = 1L + }, + new + { + Id = new Guid("00000000-0000-7000-8000-000000000004"), + BuiltIn = true, + Description = "The anonymous visitor. Grants here open public Delivery access.", + Name = "Public", + PermissionsVersion = 1L + }, + new + { + Id = new Guid("00000000-0000-7000-8000-000000000005"), + BuiltIn = true, + Description = "The default role of a registered member.", + Name = "Member", + PermissionsVersion = 1L + }, + new + { + Id = new Guid("00000000-0000-7000-8000-000000000006"), + BuiltIn = true, + Description = "The anonymous visitor. Grants here open public Delivery access.", + Name = "Public", + PermissionsVersion = 1L, + SiteId = new Guid("00000000-0000-7000-8002-000000000001") + }); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.RoleGrant", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT") + .HasColumnName("id"); + + b.Property("Constraints") + .HasColumnType("TEXT") + .HasColumnName("constraints"); + + b.Property("Permission") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT") + .HasColumnName("permission"); + + b.Property("RoleId") + .HasColumnType("TEXT") + .HasColumnName("role_id"); + + b.HasKey("Id"); + + b.HasIndex("RoleId", "Permission") + .HasDatabaseName("ix_cms_role_permission_role_permission"); + + b.ToTable("cms_role_permission", null, t => + { + t.HasCheckConstraint("ck_cms_role_permission_constraints_json", "json_valid(\"constraints\")"); + }); + + b.HasData( + new + { + Id = new Guid("00000000-0000-7000-8001-000000000001"), + Permission = "*", + RoleId = new Guid("00000000-0000-7000-8000-000000000001") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000002"), + Permission = "content.*.create", + RoleId = new Guid("00000000-0000-7000-8000-000000000002") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000003"), + Permission = "content.*.read", + RoleId = new Guid("00000000-0000-7000-8000-000000000002") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000004"), + Permission = "content.*.update", + RoleId = new Guid("00000000-0000-7000-8000-000000000002") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000005"), + Permission = "content.*.delete", + RoleId = new Guid("00000000-0000-7000-8000-000000000002") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000006"), + Permission = "content.*.publish", + RoleId = new Guid("00000000-0000-7000-8000-000000000002") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000007"), + Permission = "content.*.unpublish", + RoleId = new Guid("00000000-0000-7000-8000-000000000002") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000008"), + Permission = "content.*.restore", + RoleId = new Guid("00000000-0000-7000-8000-000000000002") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000009"), + Permission = "schema.read", + RoleId = new Guid("00000000-0000-7000-8000-000000000002") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000010"), + Permission = "content.*.create", + RoleId = new Guid("00000000-0000-7000-8000-000000000003") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000011"), + Constraints = "{\"rowRule\":\"own-entries\"}", + Permission = "content.*.read", + RoleId = new Guid("00000000-0000-7000-8000-000000000003") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000012"), + Constraints = "{\"rowRule\":\"own-entries\"}", + Permission = "content.*.update", + RoleId = new Guid("00000000-0000-7000-8000-000000000003") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000013"), + Constraints = "{\"rowRule\":\"own-entries\"}", + Permission = "content.*.delete", + RoleId = new Guid("00000000-0000-7000-8000-000000000003") + }, + new + { + Id = new Guid("00000000-0000-7000-8001-000000000014"), + Permission = "schema.read", + RoleId = new Guid("00000000-0000-7000-8000-000000000003") + }); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Site", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT") + .HasColumnName("id"); + + b.Property("CreatedAt") + .HasColumnType("INTEGER") + .HasColumnName("created_at"); + + b.Property("DefaultLocale") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("TEXT") + .HasColumnName("default_locale"); + + b.Property("IsDefault") + .HasColumnType("INTEGER") + .HasColumnName("is_default"); + + b.Property("Name") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT") + .HasColumnName("name"); + + b.Property("Slug") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT") + .HasColumnName("slug"); + + b.HasKey("Id"); + + b.HasIndex("Slug") + .IsUnique() + .HasDatabaseName("uq_cms_site_slug"); + + b.ToTable("cms_site", (string)null); + + b.HasData( + new + { + Id = new Guid("00000000-0000-7000-8002-000000000001"), + CreatedAt = 639291744000000000L, + DefaultLocale = "en", + IsDefault = true, + Name = "Default", + Slug = "default" + }); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.SiteLocale", b => + { + b.Property("SiteId") + .HasColumnType("TEXT") + .HasColumnName("site_id"); + + b.Property("Code") + .HasMaxLength(16) + .HasColumnType("TEXT") + .HasColumnName("code"); + + b.Property("Enabled") + .HasColumnType("INTEGER") + .HasColumnName("enabled"); + + b.Property("Fallbacks") + .HasColumnType("TEXT") + .HasColumnName("fallbacks"); + + b.Property("Position") + .HasColumnType("INTEGER") + .HasColumnName("position"); + + b.HasKey("SiteId", "Code"); + + b.ToTable("cms_site_locale", null, t => + { + t.HasCheckConstraint("ck_cms_site_locale_fallbacks_json", "json_valid(\"fallbacks\")"); + }); + + b.HasData( + new + { + SiteId = new Guid("00000000-0000-7000-8002-000000000001"), + Code = "en", + Enabled = true, + Position = 0 + }); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.JobAttempt", b => + { + b.HasOne("FluentCMS.Core.Storage.Entities.Job", null) + .WithMany() + .HasForeignKey("JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Role", b => + { + b.HasOne("FluentCMS.Core.Storage.Entities.Site", null) + .WithMany() + .HasForeignKey("SiteId") + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.RoleGrant", b => + { + b.HasOne("FluentCMS.Core.Storage.Entities.Role", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.SiteLocale", b => + { + b.HasOne("FluentCMS.Core.Storage.Entities.Site", null) + .WithMany() + .HasForeignKey("SiteId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/FluentCMS.Data.Sqlite/Migrations/20261102202434_AddJobs.cs b/src/FluentCMS.Data.Sqlite/Migrations/20261102202434_AddJobs.cs new file mode 100644 index 000000000..1e0423c1b --- /dev/null +++ b/src/FluentCMS.Data.Sqlite/Migrations/20261102202434_AddJobs.cs @@ -0,0 +1,109 @@ +// +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace FluentCMS.Data.Sqlite.Migrations +{ + /// + public partial class AddJobs : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "cms_job", + columns: table => new + { + id = table.Column(type: "TEXT", nullable: false), + type = table.Column(type: "TEXT", maxLength: 128, nullable: false), + payload = table.Column(type: "TEXT", nullable: false), + queue = table.Column(type: "TEXT", maxLength: 64, nullable: false), + site_id = table.Column(type: "TEXT", nullable: true), + run_at = table.Column(type: "INTEGER", nullable: false), + priority = table.Column(type: "INTEGER", nullable: false), + status = table.Column(type: "TEXT", maxLength: 16, nullable: false), + attempts = table.Column(type: "INTEGER", nullable: false), + max_attempts = table.Column(type: "INTEGER", nullable: false), + lease_until = table.Column(type: "INTEGER", nullable: true), + lease_owner = table.Column(type: "TEXT", maxLength: 128, nullable: true), + last_error = table.Column(type: "TEXT", maxLength: 2000, nullable: true), + created_at = table.Column(type: "INTEGER", nullable: false), + created_by = table.Column(type: "TEXT", nullable: true), + origin = table.Column(type: "TEXT", maxLength: 16, nullable: false), + completed_at = table.Column(type: "INTEGER", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_cms_job", x => x.id); + table.CheckConstraint("ck_cms_job_payload_json", "json_valid(\"payload\")"); + }); + + migrationBuilder.CreateTable( + name: "cms_recurring_job", + columns: table => new + { + name = table.Column(type: "TEXT", maxLength: 128, nullable: false), + cron = table.Column(type: "TEXT", maxLength: 128, nullable: false), + type = table.Column(type: "TEXT", maxLength: 128, nullable: false), + payload = table.Column(type: "TEXT", nullable: false), + queue = table.Column(type: "TEXT", maxLength: 64, nullable: false), + next_run_at = table.Column(type: "INTEGER", nullable: false), + enabled = table.Column(type: "INTEGER", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_cms_recurring_job", x => x.name); + table.CheckConstraint("ck_cms_recurring_job_payload_json", "json_valid(\"payload\")"); + }); + + migrationBuilder.CreateTable( + name: "cms_job_attempt", + columns: table => new + { + job_id = table.Column(type: "TEXT", nullable: false), + attempt = table.Column(type: "INTEGER", nullable: false), + started_at = table.Column(type: "INTEGER", nullable: false), + finished_at = table.Column(type: "INTEGER", nullable: true), + outcome = table.Column(type: "TEXT", maxLength: 16, nullable: true), + error = table.Column(type: "TEXT", maxLength: 2000, nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_cms_job_attempt", x => new { x.job_id, x.attempt }); + table.ForeignKey( + name: "FK_cms_job_attempt_cms_job_job_id", + column: x => x.job_id, + principalTable: "cms_job", + principalColumn: "id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "ix_cms_job_claim", + table: "cms_job", + columns: new[] { "queue", "priority", "run_at", "id" }, + descending: new[] { false, true, false, false }, + filter: "status IN ('pending', 'running')"); + + migrationBuilder.CreateIndex( + name: "ix_cms_job_status_lease", + table: "cms_job", + columns: new[] { "status", "lease_until" }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "cms_job_attempt"); + + migrationBuilder.DropTable( + name: "cms_recurring_job"); + + migrationBuilder.DropTable( + name: "cms_job"); + } + } +} diff --git a/src/FluentCMS.Data.Sqlite/Migrations/SystemDbContextModelSnapshot.cs b/src/FluentCMS.Data.Sqlite/Migrations/SystemDbContextModelSnapshot.cs index cc16994ac..496c16940 100644 --- a/src/FluentCMS.Data.Sqlite/Migrations/SystemDbContextModelSnapshot.cs +++ b/src/FluentCMS.Data.Sqlite/Migrations/SystemDbContextModelSnapshot.cs @@ -173,6 +173,183 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("cms_document", (string)null); }); + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Job", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT") + .HasColumnName("id"); + + b.Property("Attempts") + .HasColumnType("INTEGER") + .HasColumnName("attempts"); + + b.Property("CompletedAt") + .HasColumnType("INTEGER") + .HasColumnName("completed_at"); + + b.Property("CreatedAt") + .HasColumnType("INTEGER") + .HasColumnName("created_at"); + + b.Property("CreatedBy") + .HasColumnType("TEXT") + .HasColumnName("created_by"); + + b.Property("LastError") + .HasMaxLength(2000) + .HasColumnType("TEXT") + .HasColumnName("last_error"); + + b.Property("LeaseOwner") + .HasMaxLength(128) + .HasColumnType("TEXT") + .HasColumnName("lease_owner"); + + b.Property("LeaseUntil") + .HasColumnType("INTEGER") + .HasColumnName("lease_until"); + + b.Property("MaxAttempts") + .HasColumnType("INTEGER") + .HasColumnName("max_attempts"); + + b.Property("Origin") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("TEXT") + .HasColumnName("origin"); + + b.Property("Payload") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("payload"); + + b.Property("Priority") + .HasColumnType("INTEGER") + .HasColumnName("priority"); + + b.Property("Queue") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT") + .HasColumnName("queue"); + + b.Property("RunAt") + .HasColumnType("INTEGER") + .HasColumnName("run_at"); + + b.Property("SiteId") + .HasColumnType("TEXT") + .HasColumnName("site_id"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(16) + .HasColumnType("TEXT") + .HasColumnName("status"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT") + .HasColumnName("type"); + + b.HasKey("Id"); + + b.HasIndex("Status", "LeaseUntil") + .HasDatabaseName("ix_cms_job_status_lease"); + + b.HasIndex("Queue", "Priority", "RunAt", "Id") + .IsDescending(false, true, false, false) + .HasDatabaseName("ix_cms_job_claim") + .HasFilter("status IN ('pending', 'running')"); + + b.ToTable("cms_job", null, t => + { + t.HasCheckConstraint("ck_cms_job_payload_json", "json_valid(\"payload\")"); + }); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.JobAttempt", b => + { + b.Property("JobId") + .HasColumnType("TEXT") + .HasColumnName("job_id"); + + b.Property("Attempt") + .HasColumnType("INTEGER") + .HasColumnName("attempt"); + + b.Property("Error") + .HasMaxLength(2000) + .HasColumnType("TEXT") + .HasColumnName("error"); + + b.Property("FinishedAt") + .HasColumnType("INTEGER") + .HasColumnName("finished_at"); + + b.Property("Outcome") + .HasMaxLength(16) + .HasColumnType("TEXT") + .HasColumnName("outcome"); + + b.Property("StartedAt") + .HasColumnType("INTEGER") + .HasColumnName("started_at"); + + b.HasKey("JobId", "Attempt"); + + b.ToTable("cms_job_attempt", (string)null); + }); + + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.RecurringJob", b => + { + b.Property("Name") + .HasMaxLength(128) + .HasColumnType("TEXT") + .HasColumnName("name"); + + b.Property("Cron") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT") + .HasColumnName("cron"); + + b.Property("Enabled") + .HasColumnType("INTEGER") + .HasColumnName("enabled"); + + b.Property("NextRunAt") + .HasColumnType("INTEGER") + .HasColumnName("next_run_at"); + + b.Property("Payload") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("payload"); + + b.Property("Queue") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT") + .HasColumnName("queue"); + + b.Property("Type") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT") + .HasColumnName("type"); + + b.HasKey("Name"); + + b.ToTable("cms_recurring_job", null, t => + { + t.HasCheckConstraint("ck_cms_recurring_job_payload_json", "json_valid(\"payload\")"); + }); + }); + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Reference", b => { b.Property("Id") @@ -543,6 +720,15 @@ protected override void BuildModel(ModelBuilder modelBuilder) }); }); + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.JobAttempt", b => + { + b.HasOne("FluentCMS.Core.Storage.Entities.Job", null) + .WithMany() + .HasForeignKey("JobId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + modelBuilder.Entity("FluentCMS.Core.Storage.Entities.Role", b => { b.HasOne("FluentCMS.Core.Storage.Entities.Site", null) diff --git a/src/FluentCMS.Data.Sqlite/SqliteJobClaimDialect.cs b/src/FluentCMS.Data.Sqlite/SqliteJobClaimDialect.cs index c44a98a0e..6d4611b82 100644 --- a/src/FluentCMS.Data.Sqlite/SqliteJobClaimDialect.cs +++ b/src/FluentCMS.Data.Sqlite/SqliteJobClaimDialect.cs @@ -4,31 +4,33 @@ namespace FluentCMS.Data.Sqlite; -// Optimistic: read the next job, then update it only if its lease is still free. A worker that loses the race -// updates nothing and reads again. The two steps are separate because UPDATE ... RETURNING commits on a later step -// than the one Microsoft.Data.Sqlite retries while the database is busy, so under load it throws 'database is -// locked'. Dates are UTC ticks, as SqliteStorageConventions stores them. +// Optimistic: read the next job, then update it only if it can still be claimed. A worker that loses the race updates +// nothing and reads again. The two steps are separate because UPDATE ... RETURNING commits on a later step than the +// one Microsoft.Data.Sqlite retries while the database is busy, so under load it throws 'database is locked'. public sealed class SqliteJobClaimDialect : IJobClaimDialect { + private const int MaxTries = 32; + public async Task ClaimAsync(DatabaseFacade database, JobClaim claim, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(database); ArgumentNullException.ThrowIfNull(claim); var now = claim.Now.UtcTicks; + var unreadable = 0; - while (true) + for (var tries = 0; tries < MaxTries; tries++) { - var candidates = await database.SqlQuery($""" + var candidates = await database.SqlQuery($""" SELECT id AS "Value" FROM cms_job WHERE queue = {claim.Queue} - AND status IN ({JobClaimStatus.Pending}, {JobClaimStatus.Running}) + AND status IN ('pending', 'running') AND run_at <= {now} AND attempts < max_attempts AND (lease_until IS NULL OR lease_until < {now}) ORDER BY priority DESC, run_at, id - LIMIT 1 + LIMIT 1 OFFSET {unreadable} """).ToListAsync(cancellationToken); if (candidates.Count == 0) @@ -36,16 +38,32 @@ LIMIT 1 return null; } + // A row not written by EF must not stop the queue, so it is skipped if it holds no id at all. + if (!Guid.TryParse(candidates[0], out var id)) + { + unreadable++; + continue; + } + + // Matches the text as stored, and stores it as EF would, because everything after the claim binds the + // id as uppercase text. Every condition is checked again, not only the lease: between the two steps + // another worker can have finished the job or failed it to a later run_at, and both clear the lease. var claimed = await database.ExecuteSqlAsync($""" UPDATE cms_job - SET status = {JobClaimStatus.Running}, lease_owner = {claim.Owner}, lease_until = {claim.LeaseUntil.UtcTicks}, attempts = attempts + 1 - WHERE id = {candidates[0]} AND (lease_until IS NULL OR lease_until < {now}) + SET id = {id.ToString().ToUpperInvariant()}, status = {JobClaimStatus.Running}, lease_owner = {claim.Owner}, lease_until = {claim.LeaseUntil.UtcTicks}, attempts = attempts + 1 + WHERE id = {candidates[0]} + AND status IN ('pending', 'running') + AND run_at <= {now} + AND attempts < max_attempts + AND (lease_until IS NULL OR lease_until < {now}) """, cancellationToken); if (claimed == 1) { - return candidates[0]; + return id; } } + + return null; } } diff --git a/src/FluentCMS.Sdk/Content/OperationContext.cs b/src/FluentCMS.Sdk/Content/OperationContext.cs index 695b81bd8..4b5fbbea6 100644 --- a/src/FluentCMS.Sdk/Content/OperationContext.cs +++ b/src/FluentCMS.Sdk/Content/OperationContext.cs @@ -20,4 +20,14 @@ public sealed record OperationContext( string Locale, string? ClientId, string CorrelationId, - DateTimeOffset Timestamp); + DateTimeOffset Timestamp) +{ + /// + /// Who this operation acts for, when does the work on someone's behalf. It is for the + /// audit trail only: authorization reads and never this. + /// + public Principal? Actor { get; init; } + + /// The identity to record on what this operation writes: if there is one, otherwise . + public Principal AuditPrincipal => Actor ?? Principal; +} diff --git a/src/FluentCMS.Sdk/Jobs/IJob.cs b/src/FluentCMS.Sdk/Jobs/IJob.cs new file mode 100644 index 000000000..2ca8520a7 --- /dev/null +++ b/src/FluentCMS.Sdk/Jobs/IJob.cs @@ -0,0 +1,15 @@ +namespace FluentCMS.Jobs; + +/// +/// A unit of background work. An attempt that throws is retried with backoff, so a job should be safe to run again: +/// a worker that dies can leave a job half done. +/// +/// The job's input, stored as JSON when the job is enqueued. +public interface IJob +{ + /// Runs one attempt of the job. + /// The input given when the job was enqueued. + /// The attempt, its operation context and its cancellation token. + /// A task that completes when the attempt has succeeded, and faults when it has failed. + Task ExecuteAsync(TPayload payload, JobContext context); +} diff --git a/src/FluentCMS.Sdk/Jobs/IJobDeadLetters.cs b/src/FluentCMS.Sdk/Jobs/IJobDeadLetters.cs new file mode 100644 index 000000000..3a4dbba35 --- /dev/null +++ b/src/FluentCMS.Sdk/Jobs/IJobDeadLetters.cs @@ -0,0 +1,27 @@ +namespace FluentCMS.Jobs; + +/// A job that ran out of attempts. Its payload is left out, because it can hold secrets. +/// The id of the job. +/// The name the job is registered under. +/// The queue the job is on. +/// The attempts the job has had. +/// The error of the last attempt. +/// When the job was dead-lettered. +public sealed record DeadLetteredJob(Guid Id, string Type, string Queue, int Attempts, string? LastError, DateTimeOffset? DeadLetteredAt); + +/// Lists the jobs that ran out of attempts and gives them another chance. +public interface IJobDeadLetters +{ + /// Lists dead-lettered jobs, the most recent first. + /// The number of jobs to skip. + /// The most jobs to return. + /// Cancels the read. + /// The jobs. + Task> ListAsync(int offset = 0, int limit = 50, CancellationToken cancellationToken = default); + + /// Queues a dead-lettered job to run now, with the runner's default number of further attempts. + /// The id of the job. + /// Cancels the write. + /// Whether a dead-lettered job with this id was queued. + Task RetryAsync(Guid id, CancellationToken cancellationToken = default); +} diff --git a/src/FluentCMS.Sdk/Jobs/IJobRunner.cs b/src/FluentCMS.Sdk/Jobs/IJobRunner.cs new file mode 100644 index 000000000..32ac6a835 --- /dev/null +++ b/src/FluentCMS.Sdk/Jobs/IJobRunner.cs @@ -0,0 +1,18 @@ +namespace FluentCMS.Jobs; + +/// +/// Runs the queued jobs while the host is up. Register another implementation before AddFluentCms to hand the +/// work to something else, such as Hangfire, together with an that feeds it. +/// +public interface IJobRunner +{ + /// Starts running jobs and returns without waiting for them. + /// Cancels the start. + /// A task that completes when the runner has started. + Task StartAsync(CancellationToken cancellationToken = default); + + /// Stops taking jobs, lets the running ones finish or releases them, and returns. + /// Cancelled when the host no longer waits for the running jobs. + /// A task that completes when the runner has stopped. + Task StopAsync(CancellationToken cancellationToken = default); +} diff --git a/src/FluentCMS.Sdk/Jobs/IJobScheduler.cs b/src/FluentCMS.Sdk/Jobs/IJobScheduler.cs new file mode 100644 index 000000000..29fdea4be --- /dev/null +++ b/src/FluentCMS.Sdk/Jobs/IJobScheduler.cs @@ -0,0 +1,45 @@ +namespace FluentCMS.Jobs; + +/// +/// Adds jobs to the queue. A job is written on its own, outside any transaction of the caller, so it can start before +/// the caller's transaction commits. +/// +public interface IJobScheduler +{ + /// Enqueues a job to run as soon as a worker is free. + /// The job, which must be registered. + /// The job's input. + /// The input, which must not be . + /// How to run the job, or for the defaults. + /// Cancels the write. + /// The id of the job. + Task EnqueueAsync(TPayload payload, JobOptions? options = null, CancellationToken cancellationToken = default) + where TJob : class, IJob; + + /// Enqueues a job that runs no earlier than a given time. + /// The job, which must be registered. + /// The job's input. + /// The input, which must not be . + /// The earliest time to run the job. + /// How to run the job, or for the defaults. + /// Cancels the write. + /// The id of the job. + Task ScheduleAsync(TPayload payload, DateTimeOffset runAt, JobOptions? options = null, CancellationToken cancellationToken = default) + where TJob : class, IJob; + + /// + /// Adds a job that fires on a schedule, once across all instances, or updates the one of that name. Keeping the + /// same cron keeps the next run, so calling this at every startup does not move the schedule. + /// + /// The job, which must be registered. + /// The job's input. + /// The name of the schedule. + /// A five-field cron expression, read as UTC. + /// The input of every run, which must not be . + /// The queue, or for the default queue. + /// Cancels the write. + /// A task that completes when the schedule is stored. + /// The cron expression is not valid, or never occurs. + Task AddRecurringAsync(string name, string cron, TPayload payload, string? queue = null, CancellationToken cancellationToken = default) + where TJob : class, IJob; +} diff --git a/src/FluentCMS.Sdk/Jobs/JobActor.cs b/src/FluentCMS.Sdk/Jobs/JobActor.cs new file mode 100644 index 000000000..b5f8f327d --- /dev/null +++ b/src/FluentCMS.Sdk/Jobs/JobActor.cs @@ -0,0 +1,19 @@ +using FluentCMS.Content; + +namespace FluentCMS.Jobs; + +/// Who a job acts for, to record on what it writes. It says nothing about what the job may do. +/// The kind of identity. and are refused. +/// The id of the identity. +/// A display name for logs and the activity log. +public sealed record JobActor(PrincipalKind Kind, Guid Id, string Name); + +/// +/// A job payload that names an actor. The job still runs as : the actor reaches +/// with no claims and is never used for authorization. +/// +public interface IJobActorPayload +{ + /// The actor to record, or for none. + JobActor? Actor { get; } +} diff --git a/src/FluentCMS.Sdk/Jobs/JobContext.cs b/src/FluentCMS.Sdk/Jobs/JobContext.cs new file mode 100644 index 000000000..688f7fd0e --- /dev/null +++ b/src/FluentCMS.Sdk/Jobs/JobContext.cs @@ -0,0 +1,54 @@ +using FluentCMS.Content; + +namespace FluentCMS.Jobs; + +/// What a job knows about the attempt it is running. +public sealed class JobContext +{ + /// Creates the context of an attempt. + /// The id of the job. + /// The name the job is registered under. + /// The queue the job is on. + /// The number of this attempt, starting at one. + /// The attempts the job may have before it is dead-lettered. + /// The operation context the attempt runs under. + /// Cancelled when the attempt should stop. + public JobContext(Guid jobId, string type, string queue, int attempt, int maxAttempts, OperationContext operation, CancellationToken cancellationToken) + { + JobId = jobId; + Type = type; + Queue = queue; + Attempt = attempt; + MaxAttempts = maxAttempts; + Operation = operation; + CancellationToken = cancellationToken; + } + + /// The id of the job. + public Guid JobId { get; } + + /// The name the job is registered under. + public string Type { get; } + + /// The queue the job is on. + public string Queue { get; } + + /// The number of this attempt, starting at one. + public int Attempt { get; } + + /// The attempts the job may have before it is dead-lettered. + public int MaxAttempts { get; } + + /// + /// The operation context the attempt runs under, which is also . + /// Its principal is and its origin . + /// + public OperationContext Operation { get; } + + /// + /// Cancelled when the host is shutting down and its grace period has passed, when another worker has taken over + /// the job, or when this worker cannot renew its lease in time. A job that throws + /// because of this has not failed. + /// + public CancellationToken CancellationToken { get; } +} diff --git a/src/FluentCMS.Sdk/Jobs/JobOptions.cs b/src/FluentCMS.Sdk/Jobs/JobOptions.cs new file mode 100644 index 000000000..fc26f72e5 --- /dev/null +++ b/src/FluentCMS.Sdk/Jobs/JobOptions.cs @@ -0,0 +1,19 @@ +using FluentCMS.Sites; + +namespace FluentCMS.Jobs; + +/// How to run a job that is being enqueued. +public sealed record JobOptions +{ + /// The queue, or for the default queue. + public string? Queue { get; init; } + + /// The priority. A higher one runs first. + public int Priority { get; init; } + + /// The attempts before the job is dead-lettered, or for the runner's default. + public int? MaxAttempts { get; init; } + + /// The site the job is for, or for the site of the current operation. + public SiteId? Site { get; init; } +} diff --git a/src/FluentCMS.Sdk/PublicAPI.Unshipped.txt b/src/FluentCMS.Sdk/PublicAPI.Unshipped.txt index e61804ef0..65e00d1c1 100644 --- a/src/FluentCMS.Sdk/PublicAPI.Unshipped.txt +++ b/src/FluentCMS.Sdk/PublicAPI.Unshipped.txt @@ -100,6 +100,9 @@ FluentCMS.Content.Not.Filter.init -> void FluentCMS.Content.Not.Not(FluentCMS.Content.ContentFilter! Filter) -> void FluentCMS.Content.OperationContext FluentCMS.Content.OperationContext.$() -> FluentCMS.Content.OperationContext! +FluentCMS.Content.OperationContext.Actor.get -> FluentCMS.Content.Principal? +FluentCMS.Content.OperationContext.Actor.init -> void +FluentCMS.Content.OperationContext.AuditPrincipal.get -> FluentCMS.Content.Principal! FluentCMS.Content.OperationContext.ClientId.get -> string? FluentCMS.Content.OperationContext.ClientId.init -> void FluentCMS.Content.OperationContext.CorrelationId.get -> string! @@ -248,6 +251,69 @@ FluentCMS.Hooks.IBeforeUpdateHook FluentCMS.Hooks.IBeforeUpdateHook.BeforeUpdateAsync(FluentCMS.Hooks.HookContext! context, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.ValueTask FluentCMS.Hooks.IBeforeValidateHook FluentCMS.Hooks.IBeforeValidateHook.BeforeValidateAsync(FluentCMS.Hooks.HookContext! context, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.ValueTask +FluentCMS.Jobs.DeadLetteredJob +FluentCMS.Jobs.DeadLetteredJob.$() -> FluentCMS.Jobs.DeadLetteredJob! +FluentCMS.Jobs.DeadLetteredJob.Attempts.get -> int +FluentCMS.Jobs.DeadLetteredJob.Attempts.init -> void +FluentCMS.Jobs.DeadLetteredJob.DeadLetteredAt.get -> System.DateTimeOffset? +FluentCMS.Jobs.DeadLetteredJob.DeadLetteredAt.init -> void +FluentCMS.Jobs.DeadLetteredJob.DeadLetteredJob(System.Guid Id, string! Type, string! Queue, int Attempts, string? LastError, System.DateTimeOffset? DeadLetteredAt) -> void +FluentCMS.Jobs.DeadLetteredJob.Deconstruct(out System.Guid Id, out string! Type, out string! Queue, out int Attempts, out string? LastError, out System.DateTimeOffset? DeadLetteredAt) -> void +FluentCMS.Jobs.DeadLetteredJob.Equals(FluentCMS.Jobs.DeadLetteredJob? other) -> bool +FluentCMS.Jobs.DeadLetteredJob.Id.get -> System.Guid +FluentCMS.Jobs.DeadLetteredJob.Id.init -> void +FluentCMS.Jobs.DeadLetteredJob.LastError.get -> string? +FluentCMS.Jobs.DeadLetteredJob.LastError.init -> void +FluentCMS.Jobs.DeadLetteredJob.Queue.get -> string! +FluentCMS.Jobs.DeadLetteredJob.Queue.init -> void +FluentCMS.Jobs.DeadLetteredJob.Type.get -> string! +FluentCMS.Jobs.DeadLetteredJob.Type.init -> void +FluentCMS.Jobs.IJob +FluentCMS.Jobs.IJob.ExecuteAsync(TPayload payload, FluentCMS.Jobs.JobContext! context) -> System.Threading.Tasks.Task! +FluentCMS.Jobs.IJobActorPayload +FluentCMS.Jobs.IJobActorPayload.Actor.get -> FluentCMS.Jobs.JobActor? +FluentCMS.Jobs.IJobDeadLetters +FluentCMS.Jobs.IJobDeadLetters.ListAsync(int offset = 0, int limit = 50, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! +FluentCMS.Jobs.IJobDeadLetters.RetryAsync(System.Guid id, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +FluentCMS.Jobs.IJobRunner +FluentCMS.Jobs.IJobRunner.StartAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +FluentCMS.Jobs.IJobRunner.StopAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +FluentCMS.Jobs.IJobScheduler +FluentCMS.Jobs.IJobScheduler.AddRecurringAsync(string! name, string! cron, TPayload payload, string? queue = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +FluentCMS.Jobs.IJobScheduler.EnqueueAsync(TPayload payload, FluentCMS.Jobs.JobOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +FluentCMS.Jobs.IJobScheduler.ScheduleAsync(TPayload payload, System.DateTimeOffset runAt, FluentCMS.Jobs.JobOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +FluentCMS.Jobs.JobActor +FluentCMS.Jobs.JobActor.$() -> FluentCMS.Jobs.JobActor! +FluentCMS.Jobs.JobActor.Deconstruct(out FluentCMS.Content.PrincipalKind Kind, out System.Guid Id, out string! Name) -> void +FluentCMS.Jobs.JobActor.Equals(FluentCMS.Jobs.JobActor? other) -> bool +FluentCMS.Jobs.JobActor.Id.get -> System.Guid +FluentCMS.Jobs.JobActor.Id.init -> void +FluentCMS.Jobs.JobActor.JobActor(FluentCMS.Content.PrincipalKind Kind, System.Guid Id, string! Name) -> void +FluentCMS.Jobs.JobActor.Kind.get -> FluentCMS.Content.PrincipalKind +FluentCMS.Jobs.JobActor.Kind.init -> void +FluentCMS.Jobs.JobActor.Name.get -> string! +FluentCMS.Jobs.JobActor.Name.init -> void +FluentCMS.Jobs.JobContext +FluentCMS.Jobs.JobContext.Attempt.get -> int +FluentCMS.Jobs.JobContext.CancellationToken.get -> System.Threading.CancellationToken +FluentCMS.Jobs.JobContext.JobContext(System.Guid jobId, string! type, string! queue, int attempt, int maxAttempts, FluentCMS.Content.OperationContext! operation, System.Threading.CancellationToken cancellationToken) -> void +FluentCMS.Jobs.JobContext.JobId.get -> System.Guid +FluentCMS.Jobs.JobContext.MaxAttempts.get -> int +FluentCMS.Jobs.JobContext.Operation.get -> FluentCMS.Content.OperationContext! +FluentCMS.Jobs.JobContext.Queue.get -> string! +FluentCMS.Jobs.JobContext.Type.get -> string! +FluentCMS.Jobs.JobOptions +FluentCMS.Jobs.JobOptions.$() -> FluentCMS.Jobs.JobOptions! +FluentCMS.Jobs.JobOptions.Equals(FluentCMS.Jobs.JobOptions? other) -> bool +FluentCMS.Jobs.JobOptions.JobOptions() -> void +FluentCMS.Jobs.JobOptions.MaxAttempts.get -> int? +FluentCMS.Jobs.JobOptions.MaxAttempts.init -> void +FluentCMS.Jobs.JobOptions.Priority.get -> int +FluentCMS.Jobs.JobOptions.Priority.init -> void +FluentCMS.Jobs.JobOptions.Queue.get -> string? +FluentCMS.Jobs.JobOptions.Queue.init -> void +FluentCMS.Jobs.JobOptions.Site.get -> FluentCMS.Sites.SiteId? +FluentCMS.Jobs.JobOptions.Site.init -> void FluentCMS.Localization.FallbackChain FluentCMS.Localization.FallbackChain.FallbackChain(System.Collections.Generic.IEnumerable! locales) -> void FluentCMS.Localization.FallbackChain.Fallbacks.get -> System.Collections.Generic.IEnumerable! @@ -711,6 +777,15 @@ override FluentCMS.Content.SortField.ToString() -> string! override FluentCMS.Content.ValidationError.Equals(object? obj) -> bool override FluentCMS.Content.ValidationError.GetHashCode() -> int override FluentCMS.Content.ValidationError.ToString() -> string! +override FluentCMS.Jobs.DeadLetteredJob.Equals(object? obj) -> bool +override FluentCMS.Jobs.DeadLetteredJob.GetHashCode() -> int +override FluentCMS.Jobs.DeadLetteredJob.ToString() -> string! +override FluentCMS.Jobs.JobActor.Equals(object? obj) -> bool +override FluentCMS.Jobs.JobActor.GetHashCode() -> int +override FluentCMS.Jobs.JobActor.ToString() -> string! +override FluentCMS.Jobs.JobOptions.Equals(object? obj) -> bool +override FluentCMS.Jobs.JobOptions.GetHashCode() -> int +override FluentCMS.Jobs.JobOptions.ToString() -> string! override FluentCMS.Localization.LocaleRequest.Equals(object? obj) -> bool override FluentCMS.Localization.LocaleRequest.GetHashCode() -> int override FluentCMS.Localization.LocaleRequest.ToString() -> string! @@ -767,6 +842,12 @@ static FluentCMS.Content.SortField.operator ==(FluentCMS.Content.SortField? left static FluentCMS.Content.ValidationError.operator !=(FluentCMS.Content.ValidationError? left, FluentCMS.Content.ValidationError? right) -> bool static FluentCMS.Content.ValidationError.operator ==(FluentCMS.Content.ValidationError? left, FluentCMS.Content.ValidationError? right) -> bool static FluentCMS.Content.ValidationResult.Success.get -> FluentCMS.Content.ValidationResult! +static FluentCMS.Jobs.DeadLetteredJob.operator !=(FluentCMS.Jobs.DeadLetteredJob? left, FluentCMS.Jobs.DeadLetteredJob? right) -> bool +static FluentCMS.Jobs.DeadLetteredJob.operator ==(FluentCMS.Jobs.DeadLetteredJob? left, FluentCMS.Jobs.DeadLetteredJob? right) -> bool +static FluentCMS.Jobs.JobActor.operator !=(FluentCMS.Jobs.JobActor? left, FluentCMS.Jobs.JobActor? right) -> bool +static FluentCMS.Jobs.JobActor.operator ==(FluentCMS.Jobs.JobActor? left, FluentCMS.Jobs.JobActor? right) -> bool +static FluentCMS.Jobs.JobOptions.operator !=(FluentCMS.Jobs.JobOptions? left, FluentCMS.Jobs.JobOptions? right) -> bool +static FluentCMS.Jobs.JobOptions.operator ==(FluentCMS.Jobs.JobOptions? left, FluentCMS.Jobs.JobOptions? right) -> bool static FluentCMS.Localization.FallbackChain.Only(string! locale) -> FluentCMS.Localization.FallbackChain! static FluentCMS.Localization.LocaleRequest.operator !=(FluentCMS.Localization.LocaleRequest? left, FluentCMS.Localization.LocaleRequest? right) -> bool static FluentCMS.Localization.LocaleRequest.operator ==(FluentCMS.Localization.LocaleRequest? left, FluentCMS.Localization.LocaleRequest? right) -> bool diff --git a/tests/FluentCMS.Data.Tests/Jobs/JobClaimPlanTests.cs b/tests/FluentCMS.Data.Tests/Jobs/JobClaimPlanTests.cs new file mode 100644 index 000000000..a5b368dc4 --- /dev/null +++ b/tests/FluentCMS.Data.Tests/Jobs/JobClaimPlanTests.cs @@ -0,0 +1,63 @@ +using System.Data.Common; +using FluentCMS.Core.Jobs; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.DependencyInjection; +using Shouldly; +using Xunit; + +namespace FluentCMS.Data.Tests.Jobs; + +public sealed class JobClaimPlanTests : IDisposable +{ + private readonly JobFixture _fixture = new(); + + public void Dispose() => _fixture.Dispose(); + + [Fact] + public async Task TheClaimSeeksItsIndexWithoutASort() + { + var faults = new JobFaults(); + var store = _fixture.NewInstance(faults: faults).GetRequiredService(); + await _fixture.EnqueueAsync(); + + // Asks the database about the statement the dialect really sends, with what it really binds. + string? sql = null; + List parameters = []; + faults.Fail(command => + { + if (sql is null && command.CommandText.Contains("ORDER BY priority DESC", StringComparison.Ordinal)) + { + sql = command.CommandText; + parameters = [.. command.Parameters.Cast().Select(parameter => new SqliteParameter(parameter.ParameterName, parameter.Value))]; + } + + return false; + }); + + await store.ClaimAsync(JobRunnerOptions.DefaultQueue, "A", TestContext.Current.CancellationToken); + + var plan = ExplainQueryPlan(sql.ShouldNotBeNull(), parameters); + plan.ShouldContain(step => step.Contains("ix_cms_job_claim", StringComparison.Ordinal)); + plan.ShouldNotContain(step => step.Contains("TEMP B-TREE", StringComparison.Ordinal)); + } + + private List ExplainQueryPlan(string sql, List parameters) + { + using var connection = new SqliteConnection(_fixture.Database.ConnectionString); + connection.Open(); + using var command = connection.CreateCommand(); +#pragma warning disable CA2100 // The statement of the dialect under test. + command.CommandText = "EXPLAIN QUERY PLAN " + sql; +#pragma warning restore CA2100 + command.Parameters.AddRange(parameters); + + using var reader = command.ExecuteReader(); + var plan = new List(); + while (reader.Read()) + { + plan.Add(reader.GetString(3)); + } + + return plan; + } +} diff --git a/tests/FluentCMS.Data.Tests/Jobs/JobExecutionTests.cs b/tests/FluentCMS.Data.Tests/Jobs/JobExecutionTests.cs new file mode 100644 index 000000000..353063bab --- /dev/null +++ b/tests/FluentCMS.Data.Tests/Jobs/JobExecutionTests.cs @@ -0,0 +1,611 @@ +using FluentCMS.Content; +using FluentCMS.Core.DependencyInjection; +using FluentCMS.Core.Jobs; +using FluentCMS.Core.Sites; +using FluentCMS.Core.Storage; +using FluentCMS.Core.Storage.Entities; +using FluentCMS.Data.Tests.Sqlite; +using FluentCMS.Jobs; +using FluentCMS.Sites; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Shouldly; +using Xunit; + +namespace FluentCMS.Data.Tests.Jobs; + +// One step at a time on a manual clock, so that what happens is what the test does and nothing else is running. +public sealed class JobExecutionTests : IDisposable +{ + private const string Queue = JobRunnerOptions.DefaultQueue; + private const string Owner = "worker"; + private const string OtherOwner = "other-worker"; + + private static readonly TimeSpan Lease = new JobRunnerOptions().LeaseDuration; + + private readonly JobFixture _fixture = new(); + + public void Dispose() => _fixture.Dispose(); + + private Task RunNextAsync(string owner = Owner) => _fixture.Executor.RunNextAsync(Queue, owner, TestContext.Current.CancellationToken); + + private void AlwaysFail() => + _fixture.Probe.Behavior = (_, _) => throw new InvalidOperationException("It broke."); + + [Fact] + public async Task AJobRunsOnceAndSucceeds() + { + var id = await _fixture.EnqueueAsync(); + + (await RunNextAsync()).ShouldBeTrue(); + (await RunNextAsync()).ShouldBeFalse(); + + _fixture.Probe.Runs.Count.ShouldBe(1); + var job = _fixture.Job(id); + job.Status.ShouldBe(JobStatus.Succeeded); + job.Attempts.ShouldBe(1); + job.CompletedAt.ShouldBe(JobFixture.Start); + job.LeaseOwner.ShouldBeNull(); + job.LeaseUntil.ShouldBeNull(); + _fixture.ShouldHaveRecorded(id, (1, JobAttemptOutcome.Succeeded)); + _fixture.Events.Failed.ShouldBeEmpty(); + } + + [Fact] + public async Task RetriesBackOffThenDeadLetterWithEveryAttemptRecorded() + { + AlwaysFail(); + var id = await _fixture.EnqueueAsync(options: new JobOptions { MaxAttempts = 3 }); + + (await RunNextAsync()).ShouldBeTrue(); + var first = _fixture.Job(id); + first.Status.ShouldBe(JobStatus.Pending); + first.LastError.ShouldNotBeNull().ShouldContain("It broke."); + first.RunAt.ShouldBeInRange(JobFixture.Start.AddSeconds(30), JobFixture.Start.AddSeconds(36)); + + (await RunNextAsync()).ShouldBeFalse("the retry is not due yet"); + + _fixture.Clock.Advance(first.RunAt - _fixture.Clock.GetUtcNow()); + (await RunNextAsync()).ShouldBeTrue(); + var second = _fixture.Job(id); + second.RunAt.ShouldBeInRange(first.RunAt.AddSeconds(60), first.RunAt.AddSeconds(72)); + + _fixture.Clock.Advance(second.RunAt - _fixture.Clock.GetUtcNow()); + (await RunNextAsync()).ShouldBeTrue(); + + var last = _fixture.Job(id); + last.Status.ShouldBe(JobStatus.DeadLettered); + last.CompletedAt.ShouldBe(second.RunAt); + last.LeaseOwner.ShouldBeNull(); + _fixture.Probe.Runs.Count.ShouldBe(3); + _fixture.ShouldHaveRecorded(id, (1, JobAttemptOutcome.Failed), (2, JobAttemptOutcome.Failed), (3, JobAttemptOutcome.Failed)); + _fixture.Attempts(id).ShouldAllBe(attempt => attempt.FinishedAt != null && attempt.Error!.Contains("It broke.")); + + _fixture.Events.Failed.Select(failed => failed.Attempt).ShouldBe([1, 2, 3]); + _fixture.Events.DeadLettered.ShouldHaveSingleItem().JobId.ShouldBe(id); + + _fixture.Clock.Advance(TimeSpan.FromDays(1)); + (await RunNextAsync()).ShouldBeFalse("a dead-lettered job does not come back by itself"); + } + + [Fact] + public async Task EachRetryFromDeadLetterGivesTheDefaultAttempts() + { + AlwaysFail(); + var further = new JobRunnerOptions().DefaultMaxAttempts; + var letters = _fixture.Services.GetRequiredService(); + var id = await _fixture.EnqueueAsync(options: new JobOptions { MaxAttempts = 2 }); + + var deadAt = 2; + for (var round = 0; round < 3; round++) + { + while (_fixture.Job(id).Status != JobStatus.DeadLettered) + { + _fixture.Clock.Advance(TimeSpan.FromHours(2)); + await RunNextAsync(); + } + + _fixture.Job(id).Attempts.ShouldBe(deadAt); + + (await letters.RetryAsync(id, TestContext.Current.CancellationToken)).ShouldBeTrue(); + + deadAt += further; + _fixture.Job(id).MaxAttempts.ShouldBe(deadAt); + } + } + + [Fact] + public void BackoffIsCappedHoweverManyAttemptsThereHaveBeen() + { + var options = new JobRunnerOptions(); + + RetryBackoff.After(int.MaxValue, options).ShouldBeLessThanOrEqualTo(options.RetryMaxDelay * (1 + options.RetryJitter)); + RetryBackoff.After(1, options).ShouldBeGreaterThanOrEqualTo(options.RetryBaseDelay); + } + + [Fact] + public async Task ADeadLetteredJobIsListedAndCanBeRetried() + { + AlwaysFail(); + var id = await _fixture.EnqueueAsync(options: new JobOptions { MaxAttempts = 1 }, secret: "payload-stays-out"); + await RunNextAsync(); + + var letters = _fixture.Services.GetRequiredService(); + var token = TestContext.Current.CancellationToken; + + var listed = (await letters.ListAsync(cancellationToken: token)).ShouldHaveSingleItem(); + listed.Id.ShouldBe(id); + listed.Type.ShouldBe(JobFixture.ProbeType); + listed.Attempts.ShouldBe(1); + listed.LastError.ShouldNotBeNull().ShouldContain("It broke."); + listed.DeadLetteredAt.ShouldBe(JobFixture.Start); + (await letters.ListAsync(offset: 1, cancellationToken: token)).ShouldBeEmpty(); + + (await letters.RetryAsync(Guid.CreateVersion7(), token)).ShouldBeFalse(); + (await letters.RetryAsync(id, token)).ShouldBeTrue(); + (await letters.RetryAsync(id, token)).ShouldBeFalse("it is no longer dead-lettered"); + (await letters.ListAsync(cancellationToken: token)).ShouldBeEmpty(); + + _fixture.Probe.Behavior = (_, _) => Task.CompletedTask; + (await RunNextAsync()).ShouldBeTrue(); + + var job = _fixture.Job(id); + job.Status.ShouldBe(JobStatus.Succeeded); + job.Attempts.ShouldBe(2); + job.CompletedAt.ShouldNotBeNull(); + job.LastError.ShouldBeNull(); + _fixture.ShouldHaveRecorded(id, (1, JobAttemptOutcome.Failed), (2, JobAttemptOutcome.Succeeded)); + } + + [Fact] + public async Task AKilledWorkersJobReturnsAfterItsLeaseExpires() + { + var id = await _fixture.EnqueueAsync(); + + // The worker claims the job and is never heard from again. + var claimed = (await _fixture.Store.ClaimAsync(Queue, Owner, TestContext.Current.CancellationToken)).ShouldNotBeNull(); + + (await RunNextAsync(OtherOwner)).ShouldBeFalse("the lease still holds"); + + _fixture.Clock.Advance(Lease + TimeSpan.FromTicks(1)); + (await RunNextAsync(OtherOwner)).ShouldBeTrue(); + + var job = _fixture.Job(id); + job.Status.ShouldBe(JobStatus.Succeeded); + job.Attempts.ShouldBe(2); + _fixture.Probe.Runs.Count.ShouldBe(1); + _fixture.ShouldHaveRecorded(id, (1, JobAttemptOutcome.LeaseExpired), (2, JobAttemptOutcome.Succeeded)); + _fixture.Events.Failed.ShouldHaveSingleItem().Attempt.ShouldBe(1); + + // If that worker was only stalled, what it writes on waking changes nothing. + (await _fixture.Store.FailAsync(claimed, "too late")).ShouldBeFalse(); + (await _fixture.Store.SucceedAsync(claimed)).ShouldBeFalse(); + (await _fixture.Store.RenewLeaseAsync(claimed)).ShouldBeNull(); + _fixture.Job(id).LastError.ShouldBeNull(); + _fixture.Attempts(id)[0].Outcome.ShouldBe(JobAttemptOutcome.LeaseExpired); + } + + [Fact] + public async Task AWorkerThatDiedOnTheLastAttemptIsDeadLetteredByTheSweep() + { + var id = await _fixture.EnqueueAsync(options: new JobOptions { MaxAttempts = 1 }); + (await _fixture.Store.ClaimAsync(Queue, Owner, TestContext.Current.CancellationToken)).ShouldNotBeNull(); + + (await _fixture.Store.SweepAsync()).ShouldBe(0, "the lease still holds"); + + _fixture.Clock.Advance(Lease + TimeSpan.FromTicks(1)); + (await RunNextAsync(OtherOwner)).ShouldBeFalse("no claim takes a job with no attempts left"); + _fixture.Job(id).Status.ShouldBe(JobStatus.Running); + + (await _fixture.Store.SweepAsync()).ShouldBe(1); + + var job = _fixture.Job(id); + job.Status.ShouldBe(JobStatus.DeadLettered); + job.LeaseOwner.ShouldBeNull(); + job.LeaseUntil.ShouldBeNull(); + job.CompletedAt.ShouldBe(_fixture.Clock.GetUtcNow()); + job.LastError.ShouldNotBeNull(); + _fixture.ShouldHaveRecorded(id, (1, JobAttemptOutcome.LeaseExpired)); + _fixture.Events.DeadLettered.ShouldHaveSingleItem().JobId.ShouldBe(id); + + (await _fixture.Store.SweepAsync()).ShouldBe(0); + _fixture.Events.DeadLettered.Count.ShouldBe(1); + } + + [Fact] + public async Task TheSweepLeavesAJobThatStillHasAttempts() + { + var id = await _fixture.EnqueueAsync(options: new JobOptions { MaxAttempts = 2 }); + await _fixture.Store.ClaimAsync(Queue, Owner, TestContext.Current.CancellationToken); + + _fixture.Clock.Advance(Lease + TimeSpan.FromTicks(1)); + + (await _fixture.Store.SweepAsync()).ShouldBe(0); + _fixture.Job(id).Status.ShouldBe(JobStatus.Running); + } + + [Fact] + public async Task ConcurrentSweepsDeadLetterAJobOnce() + { + var id = await _fixture.EnqueueAsync(options: new JobOptions { MaxAttempts = 1 }); + await _fixture.Store.ClaimAsync(Queue, Owner, TestContext.Current.CancellationToken); + _fixture.Clock.Advance(Lease + TimeSpan.FromTicks(1)); + + var stores = Enumerable.Range(0, 4).Select(_ => _fixture.NewInstance().GetRequiredService()).ToList(); + var swept = await Task.WhenAll(stores.Select(store => Task.Run(store.SweepAsync, TestContext.Current.CancellationToken))); + + swept.Sum().ShouldBe(1); + _fixture.Job(id).Status.ShouldBe(JobStatus.DeadLettered); + _fixture.Events.DeadLettered.ShouldHaveSingleItem(); + } + + [Fact] + public async Task AScheduledJobWaitsForItsRunAt() + { + var runAt = JobFixture.Start.AddHours(1); + var id = await _fixture.Scheduler.ScheduleAsync(new ProbePayload("later"), runAt, cancellationToken: TestContext.Current.CancellationToken); + + (await RunNextAsync()).ShouldBeFalse(); + + _fixture.Clock.Advance(TimeSpan.FromHours(1) - TimeSpan.FromTicks(1)); + (await RunNextAsync()).ShouldBeFalse(); + + _fixture.Clock.Advance(TimeSpan.FromTicks(1)); + (await RunNextAsync()).ShouldBeTrue(); + + _fixture.Job(id).Status.ShouldBe(JobStatus.Succeeded); + } + + [Fact] + public async Task HigherPriorityRunsFirst() + { + await _fixture.EnqueueAsync("normal"); + await _fixture.EnqueueAsync("low", new JobOptions { Priority = -1 }); + await _fixture.EnqueueAsync("high", new JobOptions { Priority = 5 }); + + while (await RunNextAsync()) + { + } + + _fixture.Probe.Runs.Select(run => run.Payload.Key).ShouldBe(["high", "normal", "low"]); + } + + [Fact] + public async Task AWorkerTakesOnlyItsOwnQueue() + { + var id = await _fixture.EnqueueAsync(options: new JobOptions { Queue = "mail" }); + + (await RunNextAsync()).ShouldBeFalse(); + (await _fixture.Executor.RunNextAsync("mail", Owner, TestContext.Current.CancellationToken)).ShouldBeTrue(); + + _fixture.Job(id).Status.ShouldBe(JobStatus.Succeeded); + } + + [Fact] + public async Task AnUnknownTypeFailsTheAttemptWithAClearError() + { + var id = _fixture.AddRow("gone-with-a-plugin", "{}", maxAttempts: 1); + + (await RunNextAsync()).ShouldBeTrue(); + + var job = _fixture.Job(id); + job.Status.ShouldBe(JobStatus.DeadLettered); + job.LastError.ShouldBe("FCMS2501: no job is registered for the type 'gone-with-a-plugin'."); + _fixture.ShouldHaveRecorded(id, (1, JobAttemptOutcome.Failed)); + } + + [Theory] + [InlineData("""{"key":5,"secret":"secret-in-a-bad-payload"}""")] + [InlineData("""{"key":"a","actor":{"kind":"secret-in-a-bad-payload"}}""")] + [InlineData("""["secret-in-a-bad-payload"]""")] + [InlineData("null")] + public async Task APayloadThatDoesNotDeserializeFailsTheAttemptWithAClearError(string payload) + { + var id = _fixture.AddRow(JobFixture.ProbeType, payload, maxAttempts: 1); + + (await RunNextAsync()).ShouldBeTrue(); + + var job = _fixture.Job(id); + job.Status.ShouldBe(JobStatus.DeadLettered); + job.LastError.ShouldNotBeNull().ShouldStartWith("FCMS2502"); + job.LastError.ShouldNotContain("secret-in-a-bad-payload"); + _fixture.Probe.Runs.ShouldBeEmpty(); + _fixture.Attempts(id).ShouldHaveSingleItem().Error.ShouldBe(job.LastError); + } + + [Fact] + public async Task AJobRunsAsTheSystemPrincipalWithOriginJob() + { + var id = await _fixture.EnqueueAsync(); + + await RunNextAsync(); + + var run = _fixture.Probe.Runs.ShouldHaveSingleItem(); + run.Ambient.ShouldBeSameAs(run.Context.Operation); + run.Context.JobId.ShouldBe(id); + run.Context.Type.ShouldBe(JobFixture.ProbeType); + run.Context.Queue.ShouldBe(Queue); + run.Context.Attempt.ShouldBe(1); + run.Context.MaxAttempts.ShouldBe(new JobRunnerOptions().DefaultMaxAttempts); + + var operation = run.Context.Operation; + operation.Principal.ShouldBeSameAs(Principal.System); + operation.Origin.ShouldBe(Origin.Job); + operation.Actor.ShouldBeNull(); + operation.AuditPrincipal.ShouldBeSameAs(Principal.System); + operation.Site.ShouldBe(new SiteId(DefaultSite.Id)); + operation.Locale.ShouldBe(DefaultSite.Locale); + operation.CorrelationId.ShouldBe(id.ToString()); + operation.Timestamp.ShouldBe(JobFixture.Start); + + _fixture.Services.GetRequiredService().Current.ShouldBeNull(); + } + + [Fact] + public async Task AnActorInThePayloadIsForAuditAndGivesTheJobNothing() + { + var actor = new JobActor(PrincipalKind.AdminUser, Guid.CreateVersion7(), "Scheduler"); + await _fixture.EnqueueAsync(actor: actor); + + await RunNextAsync(); + + var operation = _fixture.Probe.Runs.ShouldHaveSingleItem().Context.Operation; + operation.Principal.ShouldBeSameAs(Principal.System); + operation.Origin.ShouldBe(Origin.Job); + var audit = operation.AuditPrincipal; + audit.ShouldBeSameAs(operation.Actor); + audit.Kind.ShouldBe(PrincipalKind.AdminUser); + audit.Id.ShouldBe(actor.Id); + audit.Name.ShouldBe("Scheduler"); + } + + // The payload has no way to bring claims, however it is written, and claims are where roles and scopes come from. + [Fact] + public async Task AnActorCannotCarryClaims() + { + var id = Guid.CreateVersion7(); + _fixture.AddRow(JobFixture.ProbeType, """{"actor":{"kind":1,"id":"ID","name":"x","claims":{"role":"Super Admin"}}}""".Replace("ID", id.ToString(), StringComparison.Ordinal)); + + await RunNextAsync(); + + var actor = _fixture.Probe.Runs.ShouldHaveSingleItem().Context.Operation.Actor.ShouldNotBeNull(); + actor.Id.ShouldBe(id); + actor.Claims.ShouldBeEmpty(); + } + + [Theory] + [InlineData(PrincipalKind.System)] + [InlineData(PrincipalKind.Anonymous)] + public async Task AnActorMayNotBeAReservedPrincipal(PrincipalKind kind) + { + var id = await _fixture.EnqueueAsync(options: new JobOptions { MaxAttempts = 1 }, actor: new JobActor(kind, Guid.CreateVersion7(), "forged")); + + await RunNextAsync(); + + _fixture.Probe.Runs.ShouldBeEmpty(); + _fixture.Job(id).LastError.ShouldNotBeNull().ShouldStartWith("FCMS2507"); + } + + [Fact] + public async Task ASiteJobRunsInThatSiteWithItsDefaultLocale() + { + var site = new Site { Name = "Fr", Slug = "fr", DefaultLocale = "fr", CreatedAt = JobFixture.Start }; + using (var scope = _fixture.Services.CreateScope()) + { + var context = scope.ServiceProvider.GetRequiredService(); + context.Add(site); + await context.SaveChangesAsync(TestContext.Current.CancellationToken); + } + + await _fixture.EnqueueAsync(options: new JobOptions { Site = new SiteId(site.Id) }); + await RunNextAsync(); + + var operation = _fixture.Probe.Runs.ShouldHaveSingleItem().Context.Operation; + operation.Site.ShouldBe(new SiteId(site.Id)); + operation.Locale.ShouldBe("fr"); + } + + [Fact] + public async Task EnqueuingRecordsWhoAndWhereFromTheCurrentOperation() + { + var user = new Principal(PrincipalKind.AdminUser, Guid.CreateVersion7(), "Ann"); + var site = new SiteId(Guid.CreateVersion7()); + var accessor = _fixture.Services.GetRequiredService(); + accessor.Current = new OperationContext(user, Origin.Admin, site, "en", null, "c", JobFixture.Start); + + var id = await _fixture.EnqueueAsync(); + accessor.Current = null; + + var job = _fixture.Job(id); + job.CreatedBy.ShouldBe(user.Id); + job.Origin.ShouldBe(Origin.Admin); + job.SiteId.ShouldBe(site.Value); + job.CreatedAt.ShouldBe(JobFixture.Start); + } + + [Fact] + public async Task EnqueuingOutsideAnOperationRecordsNoOne() + { + var job = _fixture.Job(await _fixture.EnqueueAsync()); + + job.CreatedBy.ShouldBeNull(); + job.Origin.ShouldBe(Origin.Job); + job.SiteId.ShouldBeNull(); + job.Queue.ShouldBe(Queue); + } + + [Fact] + public async Task AnUnregisteredJobIsRefusedWhenEnqueued() + { + var failure = await Should.ThrowAsync(() => + _fixture.Scheduler.EnqueueAsync(new ProbePayload(), cancellationToken: TestContext.Current.CancellationToken)); + + failure.Code.ShouldBe("FCMS2504"); + } + + [Fact] + public async Task ANullPayloadAndNoAttemptsAreRefusedWhenEnqueued() + { + var token = TestContext.Current.CancellationToken; + + await Should.ThrowAsync(() => _fixture.Scheduler.EnqueueAsync(null!, cancellationToken: token)); + await Should.ThrowAsync(() => _fixture.EnqueueAsync(options: new JobOptions { MaxAttempts = 0 })); + } + + [Fact] + public void TwoJobsCannotShareAType() + { + using var database = new SqliteFileDatabase(); + using var services = database.CreateServices(configure: cms => cms.Jobs + .AddJob("same") + .AddJob("same")); + + Should.Throw(services.GetRequiredService).Code.ShouldBe("FCMS2505"); + } + + [Fact] + public async Task AFailureThatQuotesThePayloadIsNotStoredWithIt() + { + const string Secret = "api-key-1234567890"; + _fixture.Probe.Behavior = (payload, _) => + throw new InvalidOperationException($"POST failed with key {payload.Secret}", new FormatException($"bad {payload.Secret}")); + var id = await _fixture.EnqueueAsync(secret: Secret); + + await RunNextAsync(); + + var job = _fixture.Job(id); + job.LastError.ShouldBe("System.InvalidOperationException: POST failed with *** *** ---> System.FormatException: bad ***"); + _fixture.Attempts(id).ShouldHaveSingleItem().Error.ShouldBe(job.LastError); + _fixture.Events.Failed.ShouldHaveSingleItem().Error.ShouldBe(job.LastError); + } + + [Fact] + public void ScrubbingTakesOutNumbersShortStringsAndPropertyNames() + { + const string Payload = """{"card":4111111111111111,"pin":"123","tokens":{"sk_live_abcdef":1.50},"secret":"hunter2hunter2"}"""; + var failure = new InvalidOperationException("number 4111111111111111 code 123 token sk_live_abcdef rate 1.50 secret hunter2hunter2 rejected"); + + var text = JobErrorText.ForStorage(failure, Payload); + + text.ShouldBe("System.InvalidOperationException: number *** code *** token *** rate *** *** *** rejected"); + JobErrorText.ForLog(failure, Payload).ShouldNotContain("4111111111111111"); + } + + [Fact] + public void AShortValueIsTakenOutOnlyAsAWholeWord() + { + var text = JobErrorText.ForStorage(new InvalidOperationException("stored 12 in 123 at 12:30"), """{"to":12}"""); + + text.ShouldBe("System.InvalidOperationException: stored *** in 123 at ***:30"); + } + + [Fact] + public async Task ABadPayloadDoesNotStoreADictionaryKey() + { + var instance = _fixture.NewInstance(cms: cms => cms.Jobs.AddJob>("dictionary")); + var id = _fixture.AddRow("dictionary", """{"sk_live_abcdef":"not a number"}""", maxAttempts: 1); + + (await instance.GetRequiredService().RunNextAsync(Queue, Owner, TestContext.Current.CancellationToken)).ShouldBeTrue(); + + var error = _fixture.Job(id).LastError.ShouldNotBeNull(); + error.ShouldStartWith("FCMS2502"); + error.ShouldNotContain("sk_live_abcdef"); + } + + [Fact] + public async Task AnActorMustHaveAnId() + { + var id = await _fixture.EnqueueAsync(options: new JobOptions { MaxAttempts = 1 }, actor: new JobActor(PrincipalKind.AdminUser, Guid.Empty, "no one")); + + await RunNextAsync(); + + _fixture.Probe.Runs.ShouldBeEmpty(); + _fixture.Job(id).LastError.ShouldNotBeNull().ShouldStartWith("FCMS2507"); + } + + [Fact] + public async Task AnUnboundedErrorIsCapped() + { + _fixture.Probe.Behavior = (_, _) => throw new InvalidOperationException(new string('x', 100_000)); + var id = await _fixture.EnqueueAsync(); + + await RunNextAsync(); + + var error = _fixture.Job(id).LastError.ShouldNotBeNull(); + error.Length.ShouldBe(Job.LastErrorMaxLength); + error.ShouldEndWith("..."); + _fixture.Attempts(id).ShouldHaveSingleItem().Error!.Length.ShouldBe(Job.LastErrorMaxLength); + } + + [Fact] + public void ACapDoesNotSplitACharacter() + { + var text = new string('x', Job.LastErrorMaxLength - 4) + "\U0001F600" + new string('x', 10); + + var capped = JobErrorText.Cap(text); + + capped.ShouldEndWith("x..."); + capped.Length.ShouldBeLessThanOrEqualTo(Job.LastErrorMaxLength); + } + + [Fact] + public async Task ASinkThatThrowsChangesNothingAboutTheJob() + { + _fixture.Events.Throws = true; + AlwaysFail(); + var id = await _fixture.EnqueueAsync(options: new JobOptions { MaxAttempts = 1 }); + + (await RunNextAsync()).ShouldBeTrue(); + + _fixture.Job(id).Status.ShouldBe(JobStatus.DeadLettered); + _fixture.Events.DeadLettered.ShouldHaveSingleItem(); + } + + [Fact] + public async Task AJobThatLosesItsLeaseIsCancelledAndLeavesTheRowAlone() + { + var cancelled = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _fixture.Probe.Behavior = async (_, context) => + { + try + { + await Task.Delay(Timeout.Infinite, context.CancellationToken); + } + finally + { + cancelled.SetResult(); + } + }; + + var id = await _fixture.EnqueueAsync(); + var running = Task.Run(() => RunNextAsync(), TestContext.Current.CancellationToken); + await Eventually.HoldsAsync(() => !_fixture.Probe.Runs.IsEmpty); + + // Another worker has taken the job, as it would after a stall longer than the lease. + using (var scope = _fixture.Services.CreateScope()) + { + await scope.ServiceProvider.GetRequiredService().Set() + .Where(job => job.Id == id) + .ExecuteUpdateAsync(job => job.SetProperty(row => row.LeaseOwner, OtherOwner), TestContext.Current.CancellationToken); + } + + await Eventually.CompletesAsync(running, () => _fixture.Clock.Advance(new JobRunnerOptions().HeartbeatInterval)); + + cancelled.Task.IsCompleted.ShouldBeTrue(); + var row = _fixture.Job(id); + row.Status.ShouldBe(JobStatus.Running); + row.LeaseOwner.ShouldBe(OtherOwner); + row.LastError.ShouldBeNull(); + _fixture.Events.Failed.ShouldBeEmpty(); + } + + private sealed class DictionaryJob : IJob> + { + public Task ExecuteAsync(Dictionary payload, JobContext context) => Task.CompletedTask; + } + + private sealed class UnregisteredJob : IJob + { + public Task ExecuteAsync(ProbePayload payload, JobContext context) => Task.CompletedTask; + } +} diff --git a/tests/FluentCMS.Data.Tests/Jobs/JobFailureTests.cs b/tests/FluentCMS.Data.Tests/Jobs/JobFailureTests.cs new file mode 100644 index 000000000..52420c58f --- /dev/null +++ b/tests/FluentCMS.Data.Tests/Jobs/JobFailureTests.cs @@ -0,0 +1,483 @@ +using FluentCMS.Core.Jobs; +using FluentCMS.Core.Storage; +using FluentCMS.Jobs; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.DependencyInjection; +using Shouldly; +using Xunit; + +namespace FluentCMS.Data.Tests.Jobs; + +// What happens when an instance loses the database, or a job's row is not what it should be. +public sealed class JobFailureTests : IDisposable +{ + private const string Queue = JobRunnerOptions.DefaultQueue; + + private static readonly JobRunnerOptions Defaults = new(); + private static readonly TimeSpan Beat = Defaults.HeartbeatInterval; + private static readonly TimeSpan Lease = Defaults.LeaseDuration; + + private readonly JobFixture _fixture = new(); + private readonly JobFaults _faults = new(); + + private static CancellationToken Token => TestContext.Current.CancellationToken; + + public void Dispose() => _fixture.Dispose(); + + private JobExecutor FaultyExecutor() => _fixture.NewInstance(faults: _faults).GetRequiredService(); + + private void RunUntilCancelled(Action? started = null) => + _fixture.Probe.Behavior = async (_, context) => + { + started?.Invoke(context.CancellationToken); + await Task.Delay(Timeout.Infinite, context.CancellationToken); + }; + + private async Task BeatAsync(int beats) + { + var before = _fixture.Logs.Count(nameof(JobLog.HeartbeatFailed)); + for (var beat = 1; beat <= beats; beat++) + { + _fixture.Clock.Advance(Beat); + await Eventually.HoldsAsync(() => _fixture.Logs.Count(nameof(JobLog.HeartbeatFailed)) == before + beat); + } + } + + [Fact] + public async Task AWorkerThatCannotRenewStopsBeforeItsLeaseEnds() + { + var attempt = CancellationToken.None; + RunUntilCancelled(token => attempt = token); + var id = await _fixture.EnqueueAsync(); + + var running = Task.Run(() => FaultyExecutor().RunNextAsync(Queue, "A", Token), Token); + await Eventually.HoldsAsync(() => !_fixture.Probe.Runs.IsEmpty); + _faults.FailEverything(); + + // A renewal is still possible in time up to here. + await BeatAsync(3); + attempt.IsCancellationRequested.ShouldBeFalse(); + + // The next would come only as the lease ends. + await BeatAsync(1); + (await running).ShouldBeTrue(); + attempt.IsCancellationRequested.ShouldBeTrue(); + (_fixture.Job(id).LeaseUntil > _fixture.Clock.GetUtcNow()).ShouldBeTrue("it stopped while nobody else could have the job"); + _fixture.Job(id).Status.ShouldBe(JobStatus.Running); + + (await _fixture.Executor.RunNextAsync(Queue, "B", Token)).ShouldBeFalse(); + + _fixture.Probe.Behavior = (_, _) => Task.CompletedTask; + _fixture.Clock.Advance(Beat + TimeSpan.FromTicks(1)); + (await _fixture.Executor.RunNextAsync(Queue, "B", Token)).ShouldBeTrue(); + + _fixture.Job(id).Status.ShouldBe(JobStatus.Succeeded); + _fixture.ShouldHaveRecorded(id, (1, JobAttemptOutcome.LeaseExpired), (2, JobAttemptOutcome.Succeeded)); + } + + [Fact] + public async Task OnTheLastAttemptTheSweepOnlyTakesAJobAlreadyCancelled() + { + var attempt = CancellationToken.None; + RunUntilCancelled(token => attempt = token); + var id = await _fixture.EnqueueAsync(options: new JobOptions { MaxAttempts = 1 }); + + var running = Task.Run(() => FaultyExecutor().RunNextAsync(Queue, "A", Token), Token); + await Eventually.HoldsAsync(() => !_fixture.Probe.Runs.IsEmpty); + _faults.FailEverything(); + + await BeatAsync(4); + await running; + attempt.IsCancellationRequested.ShouldBeTrue(); + + _fixture.Clock.Advance(Beat + TimeSpan.FromTicks(1)); + (await _fixture.Store.SweepAsync()).ShouldBe(1); + + _fixture.Job(id).Status.ShouldBe(JobStatus.DeadLettered); + } + + [Fact] + public async Task AHeartbeatThatRecoversInTimeKeepsTheJob() + { + var attempt = CancellationToken.None; + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _fixture.Probe.Behavior = async (_, context) => + { + attempt = context.CancellationToken; + await gate.Task; + }; + var id = await _fixture.EnqueueAsync(); + + var running = Task.Run(() => FaultyExecutor().RunNextAsync(Queue, "A", Token), Token); + await Eventually.HoldsAsync(() => !_fixture.Probe.Runs.IsEmpty); + + _faults.FailEverything(); + await BeatAsync(3); + _faults.Recover(); + + // Enough beats that only the renewals after the outage can have kept the lease. + for (var beat = 0; beat < 10; beat++) + { + _fixture.Clock.Advance(Beat); + var expected = _fixture.Clock.GetUtcNow() + Lease; + await Eventually.HoldsAsync(() => _fixture.Job(id).LeaseUntil == expected); + } + + attempt.IsCancellationRequested.ShouldBeFalse(); + gate.SetResult(); + (await running).ShouldBeTrue(); + + _fixture.Job(id).Status.ShouldBe(JobStatus.Succeeded); + _fixture.Job(id).Attempts.ShouldBe(1); + } + + [Fact] + public async Task ARenewalThatIsCancelledDoesNotEndTheHeartbeat() + { + var attempt = CancellationToken.None; + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _fixture.Probe.Behavior = async (_, context) => + { + attempt = context.CancellationToken; + await gate.Task; + }; + var id = await _fixture.EnqueueAsync(); + + var running = Task.Run(() => FaultyExecutor().RunNextAsync(Queue, "A", Token), Token); + await Eventually.HoldsAsync(() => !_fixture.Probe.Runs.IsEmpty); + + _faults.Fail(command => + { + if (!command.CommandText.Contains("\"lease_until\"", StringComparison.Ordinal)) + { + return false; + } + + _faults.Recover(); + throw new OperationCanceledException(); + }); + await BeatAsync(1); + + // The next renewal still happens, so the lease outlives what it would have been without one. + for (var beat = 0; beat < 6; beat++) + { + _fixture.Clock.Advance(Beat); + var expected = _fixture.Clock.GetUtcNow() + Lease; + await Eventually.HoldsAsync(() => _fixture.Job(id).LeaseUntil == expected); + } + + attempt.IsCancellationRequested.ShouldBeFalse(); + (await _fixture.Store.ClaimAsync(Queue, "B", Token)).ShouldBeNull(); + + gate.SetResult(); + (await running).ShouldBeTrue(); + + _fixture.Job(id).Status.ShouldBe(JobStatus.Succeeded); + _fixture.Job(id).Attempts.ShouldBe(1); + } + + [Theory] + [InlineData(1)] + [InlineData(3)] + public async Task ABriefOutageDoesNotLoseASuccess(int maxAttempts) + { + _fixture.Probe.Behavior = (_, _) => + { + _faults.FailEverything(); + return Task.CompletedTask; + }; + var id = await _fixture.EnqueueAsync(options: new JobOptions { MaxAttempts = maxAttempts }); + + var running = Task.Run(() => FaultyExecutor().RunNextAsync(Queue, "A", Token), Token); + await Eventually.HoldsAsync(() => _fixture.Logs.Count(nameof(JobLog.SettleFailed)) >= 1, () => _fixture.Clock.Advance(TimeSpan.FromSeconds(1))); + + _faults.Recover(); + await Eventually.CompletesAsync(running, () => _fixture.Clock.Advance(TimeSpan.FromSeconds(1))); + + _fixture.Probe.Runs.Count.ShouldBe(1); + _fixture.Job(id).Status.ShouldBe(JobStatus.Succeeded); + _fixture.ShouldHaveRecorded(id, (1, JobAttemptOutcome.Succeeded)); + } + + [Fact] + public async Task ABriefOutageDoesNotLoseAFailure() + { + _fixture.Probe.Behavior = (_, _) => + { + _faults.FailEverything(); + throw new InvalidOperationException("It broke."); + }; + var id = await _fixture.EnqueueAsync(); + + var running = Task.Run(() => FaultyExecutor().RunNextAsync(Queue, "A", Token), Token); + await Eventually.HoldsAsync(() => _fixture.Logs.Count(nameof(JobLog.SettleFailed)) >= 1, () => _fixture.Clock.Advance(TimeSpan.FromSeconds(1))); + + _faults.Recover(); + await Eventually.CompletesAsync(running, () => _fixture.Clock.Advance(TimeSpan.FromSeconds(1))); + + var job = _fixture.Job(id); + job.Status.ShouldBe(JobStatus.Pending); + job.LastError.ShouldNotBeNull().ShouldContain("It broke."); + _fixture.ShouldHaveRecorded(id, (1, JobAttemptOutcome.Failed)); + } + + [Fact] + public async Task AnOutageLongerThanTheLeaseGivesUpAndTheJobRunsAgain() + { + _fixture.Probe.Behavior = (_, context) => + { + if (context.Attempt == 1) + { + _faults.FailEverything(); + } + + return Task.CompletedTask; + }; + var id = await _fixture.EnqueueAsync(); + + var running = Task.Run(() => FaultyExecutor().RunNextAsync(Queue, "A", Token), Token); + await Eventually.HoldsAsync(() => running.IsCompleted, () => _fixture.Clock.Advance(TimeSpan.FromSeconds(30))); + await Should.ThrowAsync(running); + + _faults.Recover(); + _fixture.Clock.Advance(Lease + TimeSpan.FromTicks(1)); + (await _fixture.Executor.RunNextAsync(Queue, "B", Token)).ShouldBeTrue(); + + _fixture.Job(id).Status.ShouldBe(JobStatus.Succeeded); + _fixture.ShouldHaveRecorded(id, (1, JobAttemptOutcome.LeaseExpired), (2, JobAttemptOutcome.Succeeded)); + } + + [Theory] + [InlineData("INSERT INTO \"cms_job_attempt\"")] + [InlineData("SELECT \"c\".\"id\"")] + public async Task AFailedClaimDoesNotUseUpAnAttempt(string failingCommand) + { + var id = await _fixture.EnqueueAsync(options: new JobOptions { MaxAttempts = 1 }); + var worker = FaultyExecutor(); + + _faults.Fail(command => command.CommandText.Contains(failingCommand, StringComparison.Ordinal)); + await Should.ThrowAsync(() => worker.RunNextAsync(Queue, "A", Token)); + + var job = _fixture.Job(id); + job.Status.ShouldBe(JobStatus.Pending); + job.Attempts.ShouldBe(0); + job.LeaseOwner.ShouldBeNull(); + job.LeaseUntil.ShouldBeNull(); + _fixture.Attempts(id).ShouldBeEmpty(); + _fixture.Probe.Runs.ShouldBeEmpty(); + + _faults.Recover(); + _fixture.Clock.Advance(Lease + TimeSpan.FromTicks(1)); + (await _fixture.Store.SweepAsync()).ShouldBe(0); + (await worker.RunNextAsync(Queue, "A", Token)).ShouldBeTrue(); + + _fixture.Job(id).Status.ShouldBe(JobStatus.Succeeded); + } + + [Fact] + public async Task ACancelledClaimLeavesTheJobAlone() + { + var id = await _fixture.EnqueueAsync(); + + await Should.ThrowAsync(() => _fixture.Store.ClaimAsync(Queue, "A", new CancellationToken(canceled: true))); + + _fixture.Job(id).Attempts.ShouldBe(0); + } + + [Fact] + public async Task ARowWithALowercaseIdDoesNotStopTheQueue() + { + const string Lowercase = "018f0000-0000-7000-8000-00000000abcd"; + AddRawRow(Lowercase); + var good = await _fixture.EnqueueAsync("good"); + + (await _fixture.Executor.RunNextAsync(Queue, "A", Token)).ShouldBeTrue(); + (await _fixture.Executor.RunNextAsync(Queue, "A", Token)).ShouldBeTrue(); + + var id = Guid.Parse(Lowercase); + _fixture.Job(id).Status.ShouldBe(JobStatus.Succeeded); + _fixture.Job(good).Status.ShouldBe(JobStatus.Succeeded); + _fixture.ShouldHaveRecorded(id, (1, JobAttemptOutcome.Succeeded)); + _fixture.Database.ReadStrings("select id from cms_job where id = upper(id)").Count.ShouldBe(2); + } + + [Fact] + public async Task ARowWhoseIdIsNotAGuidDoesNotStopTheQueue() + { + AddRawRow("not-a-guid"); + var good = await _fixture.EnqueueAsync("good"); + + var claim = _fixture.Store.ClaimAsync(Queue, "A", Token); + + (await claim).ShouldNotBeNull().Id.ShouldBe(good); + } + + [Fact] + public async Task ALeaseIsFencedByTheAttemptAsWellAsTheOwner() + { + var id = await _fixture.EnqueueAsync(); + + var stale = (await _fixture.Store.ClaimAsync(Queue, "A", Token)).ShouldNotBeNull(); + _fixture.Clock.Advance(Lease + TimeSpan.FromTicks(1)); + var current = (await _fixture.Store.ClaimAsync(Queue, "A", Token)).ShouldNotBeNull(); + current.Attempt.ShouldBe(2); + + (await _fixture.Store.RenewLeaseAsync(stale)).ShouldBeNull(); + (await _fixture.Store.SucceedAsync(stale)).ShouldBeFalse(); + (await _fixture.Store.FailAsync(stale, "too late")).ShouldBeFalse(); + (await _fixture.Store.ReleaseAsync(stale)).ShouldBeFalse(); + + _fixture.Job(id).Status.ShouldBe(JobStatus.Running); + (await _fixture.Store.SucceedAsync(current)).ShouldBeTrue(); + _fixture.ShouldHaveRecorded(id, (1, JobAttemptOutcome.LeaseExpired), (2, JobAttemptOutcome.Succeeded)); + } + + [Fact] + public async Task ARunnerOutOfReachOfTheDatabaseCarriesOnWhenItReturns() + { + var runner = _fixture.NewInstance( + TimeProvider.System, + options => + { + options.Queues[Queue] = 2; + options.PollInterval = TimeSpan.FromMilliseconds(5); + options.MaintenanceInterval = TimeSpan.FromMilliseconds(5); + }, + _faults).GetRequiredService(); + + _faults.FailEverything(); + await runner.StartAsync(Token); + await Eventually.HoldsAsync(() => + _fixture.Logs.Count(nameof(JobLog.WorkerFailed)) >= 4 && _fixture.Logs.Count(nameof(JobLog.MaintenanceFailed)) >= 4); + + _faults.Recover(); + var id = await _fixture.EnqueueAsync(); + await Eventually.HoldsAsync(() => _fixture.Job(id).Status == JobStatus.Succeeded); + + await runner.StopAsync(Token); + } + + [Fact] + public async Task ARecurringFailureDoesNotKeepTheSweepFromRunning() + { + var stuck = await _fixture.EnqueueAsync(options: new JobOptions { MaxAttempts = 1, Queue = "no-workers" }); + await _fixture.Store.ClaimAsync("no-workers", "dead", Token); + + var runner = _fixture.NewInstance(configure: options => options.Queues.Clear(), faults: _faults).GetRequiredService(); + _faults.Fail(command => command.CommandText.Contains("cms_recurring_job", StringComparison.Ordinal)); + await runner.StartAsync(Token); + + await Eventually.HoldsAsync( + () => _fixture.Job(stuck).Status == JobStatus.DeadLettered, + () => _fixture.Clock.Advance(Defaults.MaintenanceInterval)); + + _fixture.Logs.Count(nameof(JobLog.MaintenanceFailed)).ShouldBeGreaterThan(0); + await Eventually.CompletesAsync(runner.StopAsync(Token)); + } + + [Fact] + public async Task OneScheduleThatCannotFireDoesNotStopTheOthers() + { + var scheduler = _fixture.Scheduler; + await scheduler.AddRecurringAsync("broken", "* * * * *", new ProbePayload("broken"), cancellationToken: Token); + await scheduler.AddRecurringAsync("fine", "* * * * *", new ProbePayload("fine"), cancellationToken: Token); + _fixture.Clock.Advance(TimeSpan.FromMinutes(1)); + + var recurring = _fixture.NewInstance(faults: _faults).GetRequiredService(); + var brokenDue = _fixture.Recurring("broken").NextRunAt; + _faults.Fail(command => command.CommandText.StartsWith("UPDATE", StringComparison.Ordinal) + && command.Parameters.Cast().Any(parameter => Equals(parameter.Value, "broken"))); + + await recurring.FireDueAsync(Token); + + _fixture.Jobs().ShouldHaveSingleItem().Payload.ShouldContain("fine"); + _fixture.Recurring("broken").NextRunAt.ShouldBe(brokenDue); + _fixture.Logs.Count(nameof(JobLog.RecurringFireFailed)).ShouldBe(1); + + _faults.Recover(); + await recurring.FireDueAsync(Token); + + _fixture.Jobs().Count.ShouldBe(2); + } + + // A lease lapses in the middle of a job while two real runners are up: the instance that lost the database stops + // its attempt, and the other takes the job over with no moment where both run it. + [Fact] + public async Task WhenALeaseLapsesMidJobTheTwoRunnersNeverOverlap() + { + var active = 0; + var most = 0; + _fixture.Probe.Behavior = async (_, context) => + { + var now = Interlocked.Increment(ref active); + InterlockedMax(ref most, now); + try + { + if (context.Attempt == 1) + { + await Task.Delay(Timeout.Infinite, context.CancellationToken); + } + } + finally + { + Interlocked.Decrement(ref active); + } + }; + var id = await _fixture.EnqueueAsync(); + + var first = _fixture.NewInstance(configure: options => options.Queues[Queue] = 1, faults: _faults).GetRequiredService(); + await first.StartAsync(Token); + await Eventually.HoldsAsync(() => !_fixture.Probe.Runs.IsEmpty); + _faults.FailEverything(); + + var second = _fixture.NewInstance(configure: options => options.Queues[Queue] = 1).GetRequiredService(); + await second.StartAsync(Token); + + await Eventually.HoldsAsync(() => _fixture.Job(id).Status == JobStatus.Succeeded, () => _fixture.Clock.Advance(Beat)); + + _fixture.Job(id).Attempts.ShouldBe(2); + _fixture.Probe.Runs.Count.ShouldBe(2); + most.ShouldBe(1); + _fixture.ShouldHaveRecorded(id, (1, JobAttemptOutcome.LeaseExpired), (2, JobAttemptOutcome.Succeeded)); + + _faults.Recover(); + await Eventually.CompletesAsync(first.StopAsync(Token), () => _fixture.Clock.Advance(Beat)); + await Eventually.CompletesAsync(second.StopAsync(Token), () => _fixture.Clock.Advance(Beat)); + } + + [Fact] + public async Task AJobThatIgnoresTheLapseCannotOverwriteTheWorkerThatTookOver() + { + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _fixture.Probe.Behavior = (_, context) => context.Attempt == 1 ? gate.Task : Task.CompletedTask; + var id = await _fixture.EnqueueAsync(); + + var stalled = Task.Run(() => FaultyExecutor().RunNextAsync(Queue, "A", Token), Token); + await Eventually.HoldsAsync(() => !_fixture.Probe.Runs.IsEmpty); + _faults.FailEverything(); + await BeatAsync(4); + + _fixture.Clock.Advance(Beat + TimeSpan.FromTicks(1)); + (await _fixture.Executor.RunNextAsync(Queue, "B", Token)).ShouldBeTrue(); + _fixture.Job(id).Status.ShouldBe(JobStatus.Succeeded); + var completedAt = _fixture.Job(id).CompletedAt; + + _faults.Recover(); + gate.SetResult(); + (await stalled).ShouldBeTrue(); + + _fixture.Job(id).CompletedAt.ShouldBe(completedAt); + _fixture.ShouldHaveRecorded(id, (1, JobAttemptOutcome.LeaseExpired), (2, JobAttemptOutcome.Succeeded)); + } + + private void AddRawRow(string id) => _fixture.Database.Execute( + $"INSERT INTO cms_job(id, type, payload, queue, run_at, priority, status, attempts, max_attempts, created_at, origin) VALUES ('{id}', '{JobFixture.ProbeType}', '{{}}', '{Queue}', {JobFixture.Start.AddHours(-1).UtcTicks}, 0, 'pending', 0, 3, 0, 'Job')"); + + private static void InterlockedMax(ref int target, int value) + { + int seen; + while (value > (seen = Volatile.Read(ref target)) && Interlocked.CompareExchange(ref target, value, seen) != seen) + { + } + } +} diff --git a/tests/FluentCMS.Data.Tests/Jobs/JobFixture.cs b/tests/FluentCMS.Data.Tests/Jobs/JobFixture.cs new file mode 100644 index 000000000..f56c24c75 --- /dev/null +++ b/tests/FluentCMS.Data.Tests/Jobs/JobFixture.cs @@ -0,0 +1,271 @@ +using System.Collections.Concurrent; +using System.Data.Common; +using FluentCMS.Content; +using FluentCMS.Core.DependencyInjection; +using FluentCMS.Core.Jobs; +using FluentCMS.Core.Storage; +using FluentCMS.Core.Storage.Entities; +using FluentCMS.Data.Tests.Sqlite; +using FluentCMS.Jobs; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Diagnostics; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Shouldly; +using Xunit; + +namespace FluentCMS.Data.Tests.Jobs; + +public sealed record ProbePayload(string Key = "", string? Secret = null, JobActor? Actor = null) : IJobActorPayload; + +internal sealed record JobRun(ProbePayload Payload, JobContext Context, OperationContext? Ambient); + +// What the probe job did, shared by every instance of a test, and what it should do next. +internal sealed class JobProbe +{ + public ConcurrentQueue Runs { get; } = new(); + + public Func Behavior { get; set; } = (_, _) => Task.CompletedTask; + + public int RunsOf(string key) => Runs.Count(run => run.Payload.Key == key); +} + +internal sealed class ProbeJob(JobProbe probe, IOperationContextAccessor operation) : IJob +{ + public Task ExecuteAsync(ProbePayload payload, JobContext context) + { + probe.Runs.Enqueue(new JobRun(payload, context, operation.Current)); + return probe.Behavior(payload, context); + } +} + +internal sealed class RecordingJobEvents : IJobEventSink +{ + public ConcurrentQueue Failed { get; } = new(); + + public ConcurrentQueue DeadLettered { get; } = new(); + + public bool Throws { get; set; } + + public Task PublishAsync(JobFailedEvent failed) + { + Failed.Enqueue(failed); + return Throws ? throw new InvalidOperationException("The sink is down.") : Task.CompletedTask; + } + + public Task PublishAsync(JobDeadLetteredEvent deadLettered) + { + DeadLettered.Enqueue(deadLettered); + return Throws ? throw new InvalidOperationException("The sink is down.") : Task.CompletedTask; + } +} + +// Fails the commands of one instance, as if that instance could not reach the database while others can. +internal sealed class JobFaults : DbCommandInterceptor +{ + private volatile Func? _fails; + + public void Fail(Func when) => _fails = when; + + public void FailEverything() => _fails = _ => true; + + public void Recover() => _fails = null; + + public override InterceptionResult ReaderExecuting(DbCommand command, CommandEventData eventData, InterceptionResult result) + { + Check(command); + return result; + } + + public override ValueTask> ReaderExecutingAsync(DbCommand command, CommandEventData eventData, InterceptionResult result, CancellationToken cancellationToken = default) + { + Check(command); + return ValueTask.FromResult(result); + } + + public override InterceptionResult NonQueryExecuting(DbCommand command, CommandEventData eventData, InterceptionResult result) + { + Check(command); + return result; + } + + public override ValueTask> NonQueryExecutingAsync(DbCommand command, CommandEventData eventData, InterceptionResult result, CancellationToken cancellationToken = default) + { + Check(command); + return ValueTask.FromResult(result); + } + + private void Check(DbCommand command) + { + if (_fails?.Invoke(command) == true) + { + throw new InvalidOperationException("The database cannot be reached."); + } + } +} + +internal sealed class RecordingLogs : ILoggerProvider +{ + private readonly ConcurrentDictionary _events = new(StringComparer.Ordinal); + + public int Count(string eventName) => _events.GetValueOrDefault(eventName); + + public ILogger CreateLogger(string categoryName) => new Logger(_events); + + public void Dispose() + { + } + + private sealed class Logger(ConcurrentDictionary events) : ILogger + { + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + if (eventId.Name is { } name) + { + events.AddOrUpdate(name, 1, (_, count) => count + 1); + } + } + } +} + +// A migrated file database, with any number of instances on it. Each instance is its own container, as a second +// process would be, and shares only the database, the clock and the probe. +internal sealed class JobFixture : IDisposable +{ + public const string ProbeType = "probe"; + + public static readonly DateTimeOffset Start = new(2026, 11, 2, 12, 0, 0, TimeSpan.Zero); + + private readonly List _instances = []; + + public JobFixture() + { + Services = NewInstance(); + using var scope = Services.CreateScope(); + scope.ServiceProvider.GetRequiredService().Database.Migrate(); + } + + public SqliteFileDatabase Database { get; } = new(); + + public ManualTimeProvider Clock { get; } = new(Start); + + public JobProbe Probe { get; } = new(); + + public RecordingJobEvents Events { get; } = new(); + + // The first instance, for tests with only one. + public ServiceProvider Services { get; } + + public IJobScheduler Scheduler => Services.GetRequiredService(); + + public JobExecutor Executor => Services.GetRequiredService(); + + public JobStore Store => Services.GetRequiredService(); + + public RecordingLogs Logs { get; } = new(); + + public ServiceProvider NewInstance(TimeProvider? time = null, Action? configure = null, JobFaults? faults = null, Action? cms = null) + { + var instance = Database.CreateServices(configure: builder => + { + builder.Services.AddSingleton(time ?? Clock); + builder.Services.AddSingleton(Probe); + builder.Services.AddSingleton(Events); + builder.Services.AddSingleton(Logs); + builder.Jobs.AddJob(ProbeType); + configure?.Invoke(builder.Jobs.Options); + cms?.Invoke(builder); + + if (faults is not null) + { + builder.Services.ConfigureDbContext(options => options.AddInterceptors(faults)); + } + }); + + _instances.Add(instance); + return instance; + } + + public Task EnqueueAsync(string key = "a", JobOptions? options = null, string? secret = null, JobActor? actor = null) => + Scheduler.EnqueueAsync(new ProbePayload(key, secret, actor), options, TestContext.Current.CancellationToken); + + public Job Job(Guid id) + { + using var scope = Services.CreateScope(); + return scope.ServiceProvider.GetRequiredService().Set().AsNoTracking().Single(job => job.Id == id); + } + + public List Attempts(Guid id) + { + using var scope = Services.CreateScope(); + return [.. scope.ServiceProvider.GetRequiredService().Set().AsNoTracking().Where(attempt => attempt.JobId == id).OrderBy(attempt => attempt.Attempt)]; + } + + public List Jobs() + { + using var scope = Services.CreateScope(); + return [.. scope.ServiceProvider.GetRequiredService().Set().AsNoTracking()]; + } + + public RecurringJob Recurring(string name) + { + using var scope = Services.CreateScope(); + return scope.ServiceProvider.GetRequiredService().Set().AsNoTracking().Single(job => job.Name == name); + } + + // A row that no scheduler would write, through EF as cms_job needs. + public Guid AddRow(string type, string payload, int maxAttempts = 3) + { + using var scope = Services.CreateScope(); + var context = scope.ServiceProvider.GetRequiredService(); + var job = new Job { Type = type, Payload = payload, Queue = JobRunnerOptions.DefaultQueue, RunAt = Start, MaxAttempts = maxAttempts, CreatedAt = Start, Origin = Origin.Job }; + context.Add(job); + context.SaveChanges(); + + return job.Id; + } + + public void Dispose() + { + foreach (var instance in _instances) + { + instance.Dispose(); + } + + Database.Dispose(); + } +} + +internal static class Eventually +{ + // Waits for work on other threads. The bound is only there so that a hang fails the test: nothing here depends on + // how long the work takes. + public static async Task HoldsAsync(Func condition, Action? nudge = null) + { + using var bound = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + bound.CancelAfter(TimeSpan.FromSeconds(60)); + + while (!condition()) + { + nudge?.Invoke(); + await Task.Delay(5, bound.Token); + } + } + + public static async Task CompletesAsync(Task task, Action? nudge = null) + { + await HoldsAsync(() => task.IsCompleted, nudge); + await task; + } +} + +internal static class JobAssertions +{ + public static void ShouldHaveRecorded(this JobFixture fixture, Guid job, params (int Attempt, string? Outcome)[] attempts) => + fixture.Attempts(job).Select(attempt => (attempt.Attempt, attempt.Outcome)).ShouldBe(attempts); +} diff --git a/tests/FluentCMS.Data.Tests/Jobs/JobRunnerTests.cs b/tests/FluentCMS.Data.Tests/Jobs/JobRunnerTests.cs new file mode 100644 index 000000000..542588209 --- /dev/null +++ b/tests/FluentCMS.Data.Tests/Jobs/JobRunnerTests.cs @@ -0,0 +1,380 @@ +using FluentCMS.Core.DependencyInjection; +using FluentCMS.Core.Jobs; +using FluentCMS.Core.Schema; +using FluentCMS.Core.Storage; +using FluentCMS.Data.Sqlite; +using FluentCMS.Data.Tests.Sqlite; +using FluentCMS.Jobs; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Shouldly; +using Xunit; + +namespace FluentCMS.Data.Tests.Jobs; + +// The real loops on real threads. Tests that race use the system clock with short polls and check only what must hold +// however the threads fall; tests about time use the manual clock. +public sealed class JobRunnerTests : IDisposable +{ + private static readonly TimeSpan QuickPoll = TimeSpan.FromMilliseconds(5); + + private readonly JobFixture _fixture = new(); + + private static CancellationToken Token => TestContext.Current.CancellationToken; + + public void Dispose() => _fixture.Dispose(); + + private ServiceProvider QuickInstance(int workers = 4) => + _fixture.NewInstance(TimeProvider.System, options => + { + options.Queues[JobRunnerOptions.DefaultQueue] = workers; + options.PollInterval = QuickPoll; + options.MaintenanceInterval = QuickPoll; + }); + + private ServiceProvider ManualInstance() => + _fixture.NewInstance(configure: options => options.Queues[JobRunnerOptions.DefaultQueue] = 1); + + private void WakeTheLoops() => _fixture.Clock.Advance(new JobRunnerOptions().PollInterval); + + [Fact] + public async Task AHostStartsTheRunnerAfterMigratingAndStopsIt() + { + var ran = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + using var database = new SqliteFileDatabase(); + var builder = Host.CreateEmptyApplicationBuilder(new HostApplicationBuilderSettings { EnvironmentName = "Production" }); + var probe = new JobProbe { Behavior = (_, _) => { ran.SetResult(); return Task.CompletedTask; } }; + builder.Services.AddSingleton(probe); + builder.Services.AddSingleton(new SchemaBuilder().Build()); + builder.Services.AddFluentCms(cms => + { + cms.UseSqlite(database.ConnectionString); + cms.UseContentMigrations(typeof(CmsDbContext).Assembly); + cms.Jobs.AddJob(JobFixture.ProbeType); + cms.Jobs.Options.PollInterval = QuickPoll; + }); + + using var host = builder.Build(); + await host.StartAsync(Token); + + await host.Services.GetRequiredService().EnqueueAsync(new ProbePayload("a"), cancellationToken: Token); + await ran.Task.WaitAsync(TimeSpan.FromSeconds(60), Token); + + await host.StopAsync(Token); + probe.Runs.Count.ShouldBe(1); + } + + [Theory] + [InlineData("delete")] + [InlineData("wal")] + public async Task TwoRunnersOnOneDatabaseNeverRunAJobTwice(string journalMode) + { + _fixture.Database.Execute($"PRAGMA journal_mode = {journalMode}"); + + const int Jobs = 60; + var ids = new List(); + for (var job = 0; job < Jobs / 2; job++) + { + ids.Add(await _fixture.EnqueueAsync($"before-{job}")); + } + + var runners = new[] { QuickInstance(), QuickInstance() }.Select(instance => instance.GetRequiredService()).ToList(); + await Task.WhenAll(runners.Select(runner => runner.StartAsync(Token))); + + // Half arrive while the runners are claiming. + for (var job = 0; job < Jobs / 2; job++) + { + ids.Add(await _fixture.EnqueueAsync($"during-{job}")); + } + + await Eventually.HoldsAsync(() => _fixture.Jobs().Count(job => job.Status == JobStatus.Succeeded) == Jobs); + await Task.WhenAll(runners.Select(runner => runner.StopAsync(Token))); + + _fixture.Probe.Runs.Count.ShouldBe(Jobs); + _fixture.Probe.Runs.Select(run => run.Payload.Key).ShouldBeUnique(); + foreach (var id in ids) + { + _fixture.Job(id).Attempts.ShouldBe(1); + _fixture.ShouldHaveRecorded(id, (1, JobAttemptOutcome.Succeeded)); + } + } + + [Fact] + public async Task AFailingJobDoesNotTakeAWorkerDown() + { + _fixture.Probe.Behavior = (payload, _) => payload.Key == "bad" ? throw new InvalidOperationException("It broke.") : Task.CompletedTask; + var bad = await _fixture.EnqueueAsync("bad", new JobOptions { MaxAttempts = 1, Priority = 1 }); + var unknown = _fixture.AddRow("unknown", "{}", maxAttempts: 1); + var unreadable = _fixture.AddRow(JobFixture.ProbeType, """{"key":5}""", maxAttempts: 1); + var good = await _fixture.EnqueueAsync("good"); + + var runner = QuickInstance(workers: 1).GetRequiredService(); + await runner.StartAsync(Token); + await Eventually.HoldsAsync(() => _fixture.Job(good).Status == JobStatus.Succeeded); + await runner.StopAsync(Token); + + _fixture.Job(bad).Status.ShouldBe(JobStatus.DeadLettered); + _fixture.Job(unknown).Status.ShouldBe(JobStatus.DeadLettered); + _fixture.Job(unreadable).Status.ShouldBe(JobStatus.DeadLettered); + } + + [Fact] + public async Task EachQueueHasItsOwnWorkersUpToItsLimit() + { + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var running = 0; + var most = 0; + _fixture.Probe.Behavior = async (_, _) => + { + var now = Interlocked.Increment(ref running); + InterlockedMax(ref most, now); + await gate.Task; + Interlocked.Decrement(ref running); + }; + + for (var job = 0; job < 6; job++) + { + await _fixture.EnqueueAsync($"limited-{job}"); + } + + var runner = QuickInstance(workers: 2).GetRequiredService(); + await runner.StartAsync(Token); + await Eventually.HoldsAsync(() => Volatile.Read(ref running) == 2); + + _fixture.Probe.Runs.Count.ShouldBe(2); + + gate.SetResult(); + await Eventually.HoldsAsync(() => _fixture.Jobs().All(job => job.Status == JobStatus.Succeeded)); + await runner.StopAsync(Token); + + most.ShouldBe(2); + } + + [Fact] + public async Task AHeartbeatKeepsALongJobsLease() + { + var options = new JobRunnerOptions(); + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _fixture.Probe.Behavior = (_, _) => gate.Task; + var id = await _fixture.EnqueueAsync(); + + var runner = ManualInstance().GetRequiredService(); + await runner.StartAsync(Token); + await Eventually.HoldsAsync(() => !_fixture.Probe.Runs.IsEmpty); + + // The job outlives its first lease several times over. + var beats = (int)(options.LeaseDuration / options.HeartbeatInterval) * 3; + for (var beat = 1; beat <= beats; beat++) + { + _fixture.Clock.Advance(options.HeartbeatInterval); + + var expected = _fixture.Clock.GetUtcNow() + options.LeaseDuration; + await Eventually.HoldsAsync(() => _fixture.Job(id).LeaseUntil == expected); + + (await _fixture.Store.ClaimAsync(JobRunnerOptions.DefaultQueue, "thief", TestContext.Current.CancellationToken)).ShouldBeNull(); + (await _fixture.Store.SweepAsync()).ShouldBe(0); + } + + gate.SetResult(); + await Eventually.HoldsAsync(() => _fixture.Job(id).Status == JobStatus.Succeeded); + await StopAsync(runner); + + _fixture.Probe.Runs.Count.ShouldBe(1); + _fixture.Job(id).Attempts.ShouldBe(1); + } + + [Fact] + public async Task AJobWithoutAHeartbeatLosesItsLease() + { + var options = new JobRunnerOptions(); + var id = await _fixture.EnqueueAsync(); + + await _fixture.Store.ClaimAsync(JobRunnerOptions.DefaultQueue, "silent", TestContext.Current.CancellationToken); + _fixture.Clock.Advance(options.LeaseDuration + TimeSpan.FromTicks(1)); + + (await _fixture.Store.ClaimAsync(JobRunnerOptions.DefaultQueue, "next", TestContext.Current.CancellationToken)).ShouldNotBeNull().Id.ShouldBe(id); + } + + [Fact] + public async Task ShutdownLetsARunningJobFinishAndTakesNoMore() + { + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _fixture.Probe.Behavior = (_, _) => gate.Task; + var running = await _fixture.EnqueueAsync("running"); + + var runner = ManualInstance().GetRequiredService(); + await runner.StartAsync(Token); + await Eventually.HoldsAsync(() => !_fixture.Probe.Runs.IsEmpty); + + var waiting = await _fixture.EnqueueAsync("waiting"); + var stopping = runner.StopAsync(Token); + + stopping.IsCompleted.ShouldBeFalse(); + + gate.SetResult(); + await Eventually.CompletesAsync(stopping); + + _fixture.Job(running).Status.ShouldBe(JobStatus.Succeeded); + _fixture.Job(waiting).Status.ShouldBe(JobStatus.Pending); + _fixture.Probe.RunsOf("waiting").ShouldBe(0); + } + + [Fact] + public async Task AJobThatOutlastsTheGracePeriodIsReleasedAndRunsAgainWithoutLosingAnAttempt() + { + _fixture.Probe.Behavior = (_, context) => Task.Delay(Timeout.Infinite, context.CancellationToken); + var id = await _fixture.EnqueueAsync(options: new JobOptions { MaxAttempts = 1 }); + + // The clock moves on while the release is written, however long that takes, so the runner must not give up on it. + var runner = _fixture.NewInstance(configure: options => + { + options.Queues[JobRunnerOptions.DefaultQueue] = 1; + options.ReleaseTimeout = TimeSpan.FromDays(30); + }).GetRequiredService(); + await runner.StartAsync(Token); + await Eventually.HoldsAsync(() => !_fixture.Probe.Runs.IsEmpty); + + await Eventually.CompletesAsync(runner.StopAsync(Token), WakeTheLoops); + + var released = _fixture.Job(id); + released.Status.ShouldBe(JobStatus.Pending); + released.Attempts.ShouldBe(1); + released.MaxAttempts.ShouldBe(2); + released.LeaseOwner.ShouldBeNull(); + released.LeaseUntil.ShouldBeNull(); + released.LastError.ShouldBeNull(); + _fixture.ShouldHaveRecorded(id, (1, JobAttemptOutcome.Released)); + _fixture.Events.Failed.ShouldBeEmpty(); + + _fixture.Probe.Behavior = (_, _) => Task.CompletedTask; + (await _fixture.Executor.RunNextAsync(JobRunnerOptions.DefaultQueue, "after-restart", Token)).ShouldBeTrue(); + + _fixture.Job(id).Status.ShouldBe(JobStatus.Succeeded); + _fixture.ShouldHaveRecorded(id, (1, JobAttemptOutcome.Released), (2, JobAttemptOutcome.Succeeded)); + } + + [Fact] + public async Task AHostOutOfPatienceCutsTheGracePeriodShort() + { + _fixture.Probe.Behavior = (_, context) => Task.Delay(Timeout.Infinite, context.CancellationToken); + var id = await _fixture.EnqueueAsync(); + + var runner = ManualInstance().GetRequiredService(); + await runner.StartAsync(Token); + await Eventually.HoldsAsync(() => !_fixture.Probe.Runs.IsEmpty); + + await Eventually.CompletesAsync(runner.StopAsync(new CancellationToken(canceled: true))); + + _fixture.Job(id).Status.ShouldBe(JobStatus.Pending); + } + + [Fact] + public async Task AJobThatIgnoresCancellationDoesNotHoldUpTheStop() + { + var gate = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _fixture.Probe.Behavior = (_, _) => gate.Task; + var id = await _fixture.EnqueueAsync(); + + var runner = ManualInstance().GetRequiredService(); + await runner.StartAsync(Token); + await Eventually.HoldsAsync(() => !_fixture.Probe.Runs.IsEmpty); + + await Eventually.CompletesAsync(runner.StopAsync(Token), WakeTheLoops); + + // Its heartbeat has stopped, so another instance can take the job once the lease has run out. A live heartbeat + // would renew the lease as the clock moves, and this would never get the job. A last renewal can still be on + // its way, which is why the clock moves more than once. + ClaimedJob? taken = null; + for (var tries = 0; taken is null && tries < 20; tries++) + { + _fixture.Clock.Advance(new JobRunnerOptions().LeaseDuration + TimeSpan.FromTicks(1)); + taken = await _fixture.Store.ClaimAsync(JobRunnerOptions.DefaultQueue, "next", TestContext.Current.CancellationToken); + } + + taken.ShouldNotBeNull().Id.ShouldBe(id); + + gate.SetResult(); + } + + [Fact] + public async Task StoppingWithoutStartingDoesNothing() => + await ManualInstance().GetRequiredService().StopAsync(Token); + + [Fact] + public async Task BadOptionsStopTheStart() + { + var runner = _fixture.NewInstance(configure: options => options.HeartbeatInterval = options.LeaseDuration).GetRequiredService(); + + (await Should.ThrowAsync(() => runner.StartAsync(Token))).Code.ShouldBe("FCMS2506"); + } + + [Fact] + public async Task ARunnerSweepsAndFiresOnItsOwn() + { + await _fixture.Scheduler.AddRecurringAsync("r", "* * * * *", new ProbePayload("recurring"), cancellationToken: Token); + var stuck = await _fixture.EnqueueAsync("stuck", new JobOptions { MaxAttempts = 1, Queue = "no-workers" }); + await _fixture.Store.ClaimAsync("no-workers", "dead", TestContext.Current.CancellationToken); + + var options = new JobRunnerOptions(); + var runner = ManualInstance().GetRequiredService(); + await runner.StartAsync(Token); + + await Eventually.HoldsAsync( + () => _fixture.Probe.RunsOf("recurring") >= 1 && _fixture.Job(stuck).Status == JobStatus.DeadLettered, + () => _fixture.Clock.Advance(options.MaintenanceInterval)); + + await Eventually.CompletesAsync(runner.StopAsync(Token), WakeTheLoops); + } + + [Fact] + public async Task AReplacementRunnerIsWhatTheHostStarts() + { + using var database = new SqliteFileDatabase(); + var replacement = new ReplacementRunner(); + var builder = Host.CreateEmptyApplicationBuilder(new HostApplicationBuilderSettings { EnvironmentName = "Production" }); + builder.Services.AddSingleton(replacement); + builder.Services.AddSingleton(new SchemaBuilder().Build()); + builder.Services.AddFluentCms(cms => + { + cms.UseSqlite(database.ConnectionString); + cms.UseContentMigrations(typeof(CmsDbContext).Assembly); + }); + + using var host = builder.Build(); + await host.StartAsync(Token); + await host.StopAsync(Token); + + replacement.Started.ShouldBeTrue(); + replacement.Stopped.ShouldBeTrue(); + } + + private async Task StopAsync(IJobRunner runner) => await Eventually.CompletesAsync(runner.StopAsync(Token), WakeTheLoops); + + private static void InterlockedMax(ref int target, int value) + { + int seen; + while (value > (seen = Volatile.Read(ref target)) && Interlocked.CompareExchange(ref target, value, seen) != seen) + { + } + } + + private sealed class ReplacementRunner : IJobRunner + { + public bool Started { get; private set; } + + public bool Stopped { get; private set; } + + public Task StartAsync(CancellationToken cancellationToken = default) + { + Started = true; + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken = default) + { + Stopped = true; + return Task.CompletedTask; + } + } +} diff --git a/tests/FluentCMS.Data.Tests/Jobs/ManualTimeProvider.cs b/tests/FluentCMS.Data.Tests/Jobs/ManualTimeProvider.cs new file mode 100644 index 000000000..d9fc5e046 --- /dev/null +++ b/tests/FluentCMS.Data.Tests/Jobs/ManualTimeProvider.cs @@ -0,0 +1,106 @@ +namespace FluentCMS.Data.Tests.Jobs; + +// A clock that moves only when a test says so, and fires the timers that fall due as it moves. +internal sealed class ManualTimeProvider(DateTimeOffset start) : TimeProvider +{ + private readonly Lock _lock = new(); + private readonly List _timers = []; + private DateTimeOffset _now = start; + + public override DateTimeOffset GetUtcNow() + { + lock (_lock) + { + return _now; + } + } + + public void Advance(TimeSpan by) + { + DateTimeOffset target; + lock (_lock) + { + target = _now + by; + } + + while (true) + { + ManualTimer? next; + lock (_lock) + { + next = _timers.Where(timer => timer.DueAt <= target).MinBy(timer => timer.DueAt); + if (next is null) + { + _now = target; + return; + } + + if (next.DueAt > _now) + { + _now = next.DueAt!.Value; + } + + next.Reschedule(_now); + if (next.DueAt is null) + { + _timers.Remove(next); + } + } + + next.Fire(); + } + } + + public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period) + { + var timer = new ManualTimer(this, callback, state); + timer.Change(dueTime, period); + return timer; + } + + private sealed class ManualTimer(ManualTimeProvider clock, TimerCallback callback, object? state) : ITimer + { + private TimeSpan _period = Timeout.InfiniteTimeSpan; + + public DateTimeOffset? DueAt { get; private set; } + + public void Reschedule(DateTimeOffset now) => + DueAt = _period == Timeout.InfiniteTimeSpan || _period == TimeSpan.Zero ? null : now + _period; + + public void Fire() => callback(state); + + public bool Change(TimeSpan dueTime, TimeSpan period) + { + lock (clock._lock) + { + _period = period; + clock._timers.Remove(this); + + if (dueTime == Timeout.InfiniteTimeSpan) + { + DueAt = null; + return true; + } + + DueAt = clock._now + dueTime; + clock._timers.Add(this); + return true; + } + } + + public void Dispose() + { + lock (clock._lock) + { + DueAt = null; + clock._timers.Remove(this); + } + } + + public ValueTask DisposeAsync() + { + Dispose(); + return ValueTask.CompletedTask; + } + } +} diff --git a/tests/FluentCMS.Data.Tests/Jobs/RecurringJobTests.cs b/tests/FluentCMS.Data.Tests/Jobs/RecurringJobTests.cs new file mode 100644 index 000000000..1ea131bb5 --- /dev/null +++ b/tests/FluentCMS.Data.Tests/Jobs/RecurringJobTests.cs @@ -0,0 +1,192 @@ +using FluentCMS.Content; +using FluentCMS.Core.Jobs; +using FluentCMS.Core.Storage; +using FluentCMS.Core.Storage.Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Shouldly; +using Xunit; + +namespace FluentCMS.Data.Tests.Jobs; + +public sealed class RecurringJobTests : IDisposable +{ + private const string EveryFiveMinutes = "*/5 * * * *"; + private const string Name = "sweep-things"; + + private readonly JobFixture _fixture = new(); + + private static CancellationToken Token => TestContext.Current.CancellationToken; + + public void Dispose() => _fixture.Dispose(); + + private Task AddAsync(string cron = EveryFiveMinutes, string name = Name) => + _fixture.Scheduler.AddRecurringAsync(name, cron, new ProbePayload(name), cancellationToken: Token); + + private Task FireAsync() => _fixture.Services.GetRequiredService().FireDueAsync(Token); + + [Fact] + public async Task AJobFiresOnScheduleAndRuns() + { + await AddAsync(); + + var stored = _fixture.Recurring(Name); + stored.Enabled.ShouldBeTrue(); + stored.Type.ShouldBe(JobFixture.ProbeType); + stored.Queue.ShouldBe(JobRunnerOptions.DefaultQueue); + stored.NextRunAt.ShouldBe(JobFixture.Start.AddMinutes(5)); + + _fixture.Clock.Advance(TimeSpan.FromMinutes(5) - TimeSpan.FromTicks(1)); + await FireAsync(); + _fixture.Jobs().ShouldBeEmpty(); + + _fixture.Clock.Advance(TimeSpan.FromTicks(1)); + await FireAsync(); + + var job = _fixture.Jobs().ShouldHaveSingleItem(); + job.Type.ShouldBe(JobFixture.ProbeType); + job.RunAt.ShouldBe(JobFixture.Start.AddMinutes(5)); + job.Origin.ShouldBe(Origin.Job); + _fixture.Recurring(Name).NextRunAt.ShouldBe(JobFixture.Start.AddMinutes(10)); + + await FireAsync(); + _fixture.Jobs().Count.ShouldBe(1); + + (await _fixture.Executor.RunNextAsync(JobRunnerOptions.DefaultQueue, "worker", Token)).ShouldBeTrue(); + _fixture.Probe.RunsOf(Name).ShouldBe(1); + + _fixture.Clock.Advance(TimeSpan.FromMinutes(5)); + await FireAsync(); + _fixture.Jobs().Count.ShouldBe(2); + } + + [Fact] + public async Task RunsMissedWhileNothingWasUpCollapseIntoOne() + { + await AddAsync(); + + _fixture.Clock.Advance(TimeSpan.FromHours(1) + TimeSpan.FromMinutes(2)); + await FireAsync(); + + _fixture.Jobs().ShouldHaveSingleItem(); + _fixture.Recurring(Name).NextRunAt.ShouldBe(JobFixture.Start.AddHours(1).AddMinutes(5)); + } + + [Theory] + [InlineData("delete")] + [InlineData("wal")] + public async Task AnOccurrenceFiresOnceAcrossTwoRunners(string journalMode) + { + _fixture.Database.Execute($"PRAGMA journal_mode = {journalMode}"); + await AddAsync(); + + var instances = new[] { _fixture.NewInstance(), _fixture.NewInstance() } + .Select(instance => instance.GetRequiredService()).ToList(); + + const int Occurrences = 20; + for (var occurrence = 1; occurrence <= Occurrences; occurrence++) + { + _fixture.Clock.Advance(TimeSpan.FromMinutes(5)); + using var start = new ManualResetEventSlim(); + + var firing = instances.Select(recurring => Task.Run(async () => + { + start.Wait(Token); + await recurring.FireDueAsync(Token); + }, Token)).ToArray(); + start.Set(); + await Task.WhenAll(firing); + + _fixture.Jobs().Count.ShouldBe(occurrence); + } + + _fixture.Jobs().Select(job => job.RunAt).ShouldBeUnique(); + } + + [Theory] + [InlineData("not a cron")] + [InlineData("* * * *")] + [InlineData("* * * * * *")] + [InlineData("99 * * * *")] + [InlineData("0 0 30 2 *")] + public async Task ABadCronIsRefusedWhenAdded(string cron) + { + var failure = await Should.ThrowAsync(() => AddAsync(cron)); + + failure.ParamName.ShouldBe("cron"); + failure.Message.ShouldContain("FCMS2503"); + + using var scope = _fixture.Services.CreateScope(); + (await scope.ServiceProvider.GetRequiredService().Set().AnyAsync(Token)).ShouldBeFalse(); + } + + [Fact] + public async Task AddingAgainWithTheSameCronKeepsTheNextRun() + { + await AddAsync(); + + _fixture.Clock.Advance(TimeSpan.FromMinutes(7)); + await AddAsync(); + + _fixture.Recurring(Name).NextRunAt.ShouldBe(JobFixture.Start.AddMinutes(5), "a restart must not skip the occurrence it was down for"); + } + + [Fact] + public async Task AddingAgainWithAnotherCronMovesTheNextRun() + { + await AddAsync(); + + await AddAsync("0 * * * *"); + + var stored = _fixture.Recurring(Name); + stored.Cron.ShouldBe("0 * * * *"); + stored.NextRunAt.ShouldBe(JobFixture.Start.AddHours(1)); + } + + [Fact] + public async Task InstancesAddingAtTheSameStartupAgree() + { + var schedulers = Enumerable.Range(0, 4) + .Select(_ => _fixture.NewInstance().GetRequiredService()); + + await Task.WhenAll(schedulers.Select(scheduler => Task.Run( + () => scheduler.AddRecurringAsync(Name, EveryFiveMinutes, new ProbePayload(Name), cancellationToken: Token), + Token))); + + _fixture.Recurring(Name).Cron.ShouldBe(EveryFiveMinutes); + } + + [Fact] + public async Task ADisabledJobDoesNotFire() + { + await AddAsync(); + using (var scope = _fixture.Services.CreateScope()) + { + await scope.ServiceProvider.GetRequiredService().Set() + .ExecuteUpdateAsync(job => job.SetProperty(row => row.Enabled, false), Token); + } + + _fixture.Clock.Advance(TimeSpan.FromHours(1)); + await FireAsync(); + + _fixture.Jobs().ShouldBeEmpty(); + } + + [Fact] + public async Task ARowEditedToABadCronDoesNotStopTheOthers() + { + await AddAsync(name: "broken"); + await AddAsync(name: "fine"); + using (var scope = _fixture.Services.CreateScope()) + { + await scope.ServiceProvider.GetRequiredService().Set() + .Where(job => job.Name == "broken") + .ExecuteUpdateAsync(job => job.SetProperty(row => row.Cron, "nonsense"), Token); + } + + _fixture.Clock.Advance(TimeSpan.FromMinutes(5)); + await FireAsync(); + + _fixture.Jobs().ShouldHaveSingleItem().Payload.ShouldContain("fine"); + } +} diff --git a/tests/FluentCMS.Data.Tests/Migrations/MigrationRunnerTests.cs b/tests/FluentCMS.Data.Tests/Migrations/MigrationRunnerTests.cs index 427e3e4ea..c9778770f 100644 --- a/tests/FluentCMS.Data.Tests/Migrations/MigrationRunnerTests.cs +++ b/tests/FluentCMS.Data.Tests/Migrations/MigrationRunnerTests.cs @@ -34,7 +34,7 @@ public async Task AFreshDatabaseIsFullyMigratedAtStartup() await host.StartAsync(Token); _database.Tables().ShouldBe( - ["__EFMigrationsLock", "cms__migrations", "cms_article", "cms_content__migrations", "cms_document", "cms_reference", "cms_role", "cms_role_permission", "cms_site", "cms_site_locale", "cms_version"], + ["__EFMigrationsLock", "cms__migrations", "cms_article", "cms_content__migrations", "cms_document", "cms_job", "cms_job_attempt", "cms_recurring_job", "cms_reference", "cms_role", "cms_role_permission", "cms_site", "cms_site_locale", "cms_version"], ignoreOrder: true); _database.ReadStrings("select MigrationId from cms__migrations").ShouldContain(id => id.EndsWith("_InitialSystem", StringComparison.Ordinal)); _database.ReadStrings("select MigrationId from cms_content__migrations").ShouldHaveSingleItem().ShouldEndWith("_Initial"); @@ -250,7 +250,7 @@ public void TheContentContextHasItsHistoryTableAndExcludesEverythingElse() var tables = context.GetService().Model.GetRelationalModel().Tables; tables.Where(table => !table.IsExcludedFromMigrations).Select(table => table.Name).ShouldBe(["cms_article"]); - tables.Where(table => table.IsExcludedFromMigrations).Select(table => table.Name).ShouldBe(["cms_document", "cms_reference", "cms_role", "cms_role_permission", "cms_site", "cms_site_locale", "cms_version"], ignoreOrder: true); + tables.Where(table => table.IsExcludedFromMigrations).Select(table => table.Name).ShouldBe(["cms_document", "cms_job", "cms_job_attempt", "cms_recurring_job", "cms_reference", "cms_role", "cms_role_permission", "cms_site", "cms_site_locale", "cms_version"], ignoreOrder: true); } [Fact] diff --git a/tests/FluentCMS.Data.Tests/Providers/OfflineProviderTests.cs b/tests/FluentCMS.Data.Tests/Providers/OfflineProviderTests.cs index bf04e5199..efb1eb214 100644 --- a/tests/FluentCMS.Data.Tests/Providers/OfflineProviderTests.cs +++ b/tests/FluentCMS.Data.Tests/Providers/OfflineProviderTests.cs @@ -63,6 +63,9 @@ public void PostgreSqlStoresJsonAsJsonbAndDatesAsTimestamptz() ColumnType(model, "cms_post", "published_on").ShouldBe("timestamp with time zone"); ColumnType(model, typeof(ContentVersion).FullName!, nameof(ContentVersion.Snapshot)).ShouldBe("jsonb"); ColumnType(model, typeof(Document).FullName!, nameof(Document.SchedulePublishAt)).ShouldBe("timestamp with time zone"); + ColumnType(model, typeof(Job).FullName!, nameof(Job.Payload)).ShouldBe("jsonb"); + ColumnType(model, typeof(Job).FullName!, nameof(Job.RunAt)).ShouldBe("timestamp with time zone"); + ColumnType(model, typeof(RecurringJob).FullName!, nameof(RecurringJob.Payload)).ShouldBe("jsonb"); } [Fact] @@ -88,6 +91,8 @@ public void SqlServerStoresCheckedJsonAsNvarcharMax() ColumnType(model, "cms_post", "published_on").ShouldBe("datetimeoffset"); ColumnType(model, typeof(ContentVersion).FullName!, nameof(ContentVersion.Snapshot)).ShouldBe("nvarchar(max)"); ColumnType(model, typeof(Document).FullName!, nameof(Document.SchedulePublishAt)).ShouldBe("datetimeoffset"); + ColumnType(model, typeof(Job).FullName!, nameof(Job.Payload)).ShouldBe("nvarchar(max)"); + ColumnType(model, typeof(Job).FullName!, nameof(Job.RunAt)).ShouldBe("datetimeoffset"); model.FindEntityType("cms_post")!.GetCheckConstraints().Select(check => check.Sql).ShouldContain("ISJSON([metadata]) = 1"); } @@ -119,7 +124,7 @@ public void ThePostgreSqlSystemMigrationsMatchTheModel() using var scope = services.CreateScope(); var database = scope.ServiceProvider.GetRequiredService().Database; - database.GetMigrations().Count().ShouldBe(4); + database.GetMigrations().Count().ShouldBe(5); database.HasPendingModelChanges().ShouldBeFalse(); } @@ -130,7 +135,7 @@ public void TheSqlServerSystemMigrationsMatchTheModel() using var scope = services.CreateScope(); var database = scope.ServiceProvider.GetRequiredService().Database; - database.GetMigrations().Count().ShouldBe(4); + database.GetMigrations().Count().ShouldBe(5); database.HasPendingModelChanges().ShouldBeFalse(); } diff --git a/tests/FluentCMS.Data.Tests/Sqlite/JobClaimTests.cs b/tests/FluentCMS.Data.Tests/Sqlite/JobClaimTests.cs index 2e2859a11..2b842b3f1 100644 --- a/tests/FluentCMS.Data.Tests/Sqlite/JobClaimTests.cs +++ b/tests/FluentCMS.Data.Tests/Sqlite/JobClaimTests.cs @@ -1,12 +1,14 @@ +using FluentCMS.Content; using FluentCMS.Core.Storage; +using FluentCMS.Core.Storage.Entities; using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.DependencyInjection; using Shouldly; using Xunit; namespace FluentCMS.Data.Tests.Sqlite; -// cms_job arrives with #2698, so this is a scratch table with the columns of spec section 4.12. public sealed class JobClaimTests : IDisposable { private const string Queue = "default"; @@ -21,26 +23,8 @@ public sealed class JobClaimTests : IDisposable public JobClaimTests() { _services = _database.CreateServices(); - Execute(""" - CREATE TABLE cms_job ( - id TEXT NOT NULL PRIMARY KEY, - type TEXT NOT NULL, - payload TEXT NOT NULL CHECK (json_valid(payload)), - queue TEXT NOT NULL, - site_id TEXT NULL, - run_at INTEGER NOT NULL, - priority INTEGER NOT NULL, - status TEXT NOT NULL, - attempts INTEGER NOT NULL, - max_attempts INTEGER NOT NULL, - lease_until INTEGER NULL, - lease_owner TEXT NULL, - last_error TEXT NULL, - created_at INTEGER NOT NULL, - created_by TEXT NULL, - origin TEXT NOT NULL, - completed_at INTEGER NULL) - """); + using var scope = _services.CreateScope(); + scope.ServiceProvider.GetRequiredService().Database.Migrate(); } public void Dispose() @@ -49,29 +33,13 @@ public void Dispose() _database.Dispose(); } - private static string StoredId(Guid id) => id.ToString().ToUpperInvariant(); - - private void Execute(string sql, params (string Name, object? Value)[] parameters) - { - using var connection = new SqliteConnection(_database.ConnectionString); - connection.Open(); - using var command = connection.CreateCommand(); - command.CommandText = sql; - foreach (var (name, value) in parameters) - { - command.Parameters.AddWithValue(name, value ?? DBNull.Value); - } - - command.ExecuteNonQuery(); - } - private object? Read(Guid job, string column) { using var connection = new SqliteConnection(_database.ConnectionString); connection.Open(); using var command = connection.CreateCommand(); command.CommandText = $"SELECT {column} FROM cms_job WHERE id = $id"; - command.Parameters.AddWithValue("$id", StoredId(job)); + command.Parameters.AddWithValue("$id", job.ToString().ToUpperInvariant()); return command.ExecuteScalar(); } @@ -85,23 +53,26 @@ private Guid AddJob( DateTimeOffset? leaseUntil = null, string? leaseOwner = null) { - var id = Guid.CreateVersion7(); - Execute( - """ - INSERT INTO cms_job (id, type, payload, queue, run_at, priority, status, attempts, max_attempts, lease_until, lease_owner, created_at, origin) - VALUES ($id, 'test', '{}', $queue, $runAt, $priority, $status, $attempts, $maxAttempts, $leaseUntil, $leaseOwner, 0, 'Job') - """, - ("$id", StoredId(id)), - ("$queue", queue), - ("$runAt", (runAt ?? Now.AddMinutes(-1)).UtcTicks), - ("$priority", priority), - ("$status", status), - ("$attempts", attempts), - ("$maxAttempts", maxAttempts), - ("$leaseUntil", leaseUntil?.UtcTicks), - ("$leaseOwner", leaseOwner)); - - return id; + using var scope = _services.CreateScope(); + var context = scope.ServiceProvider.GetRequiredService(); + var job = new Job + { + Type = "test", + Payload = "{}", + Queue = queue, + RunAt = runAt ?? Now.AddMinutes(-1), + Priority = priority, + Status = status, + Attempts = attempts, + MaxAttempts = maxAttempts, + LeaseUntil = leaseUntil, + LeaseOwner = leaseOwner, + Origin = Origin.Job, + }; + context.Add(job); + context.SaveChanges(); + + return job.Id; } private async Task ClaimAsync(string owner = "worker", string queue = Queue) @@ -180,7 +151,7 @@ public async Task TheHighestPriorityGoesFirstThenTheEarliestRunAt() [InlineData("wal")] public async Task OfTwoConcurrentClaimersOnlyOneWins(string journalMode) { - Execute($"PRAGMA journal_mode = {journalMode}"); + _database.Execute($"PRAGMA journal_mode = {journalMode}"); for (var round = 0; round < 25; round++) { @@ -209,7 +180,7 @@ public async Task OfTwoConcurrentClaimersOnlyOneWins(string journalMode) [InlineData("wal")] public async Task ConcurrentClaimersGetEachJobOnce(string journalMode) { - Execute($"PRAGMA journal_mode = {journalMode}"); + _database.Execute($"PRAGMA journal_mode = {journalMode}"); var jobs = Enumerable.Range(0, 20).Select(_ => AddJob()).ToList(); diff --git a/tests/FluentCMS.Data.Tests/Sqlite/SystemMigrationTests.cs b/tests/FluentCMS.Data.Tests/Sqlite/SystemMigrationTests.cs index deb3e5319..f1f57e209 100644 --- a/tests/FluentCMS.Data.Tests/Sqlite/SystemMigrationTests.cs +++ b/tests/FluentCMS.Data.Tests/Sqlite/SystemMigrationTests.cs @@ -46,11 +46,11 @@ public void AppliesToAFreshFileDatabaseWithItsOwnHistoryTable() using var scope = services.CreateScope(); var context = scope.ServiceProvider.GetRequiredService(); - context.Database.GetPendingMigrations().Count().ShouldBe(4); + context.Database.GetPendingMigrations().Count().ShouldBe(5); context.Database.Migrate(); - Tables(context).ShouldBe(["__EFMigrationsLock", "cms__migrations", "cms_document", "cms_reference", "cms_role", "cms_role_permission", "cms_site", "cms_site_locale", "cms_version"], ignoreOrder: true); - Count(context, SystemDbContext.HistoryTable).ShouldBe(4); + Tables(context).ShouldBe(["__EFMigrationsLock", "cms__migrations", "cms_document", "cms_job", "cms_job_attempt", "cms_recurring_job", "cms_reference", "cms_role", "cms_role_permission", "cms_site", "cms_site_locale", "cms_version"], ignoreOrder: true); + Count(context, SystemDbContext.HistoryTable).ShouldBe(5); context.Database.GetPendingMigrations().ShouldBeEmpty(); } @@ -70,7 +70,7 @@ public void TheModelHoldsOnlySystemTablesWhateverTheSchema() using var scope = services.CreateScope(); var context = scope.ServiceProvider.GetRequiredService(); - context.Model.GetEntityTypes().Select(type => type.GetTableName()).ShouldBe(["cms_document", "cms_reference", "cms_role", "cms_role_permission", "cms_site", "cms_site_locale", "cms_version"], ignoreOrder: true); + context.Model.GetEntityTypes().Select(type => type.GetTableName()).ShouldBe(["cms_document", "cms_job", "cms_job_attempt", "cms_recurring_job", "cms_reference", "cms_role", "cms_role_permission", "cms_site", "cms_site_locale", "cms_version"], ignoreOrder: true); } [Fact] diff --git a/tests/FluentCMS.Data.Tests/Storage/ModelBuildTimeTests.cs b/tests/FluentCMS.Data.Tests/Storage/ModelBuildTimeTests.cs index fee570491..1aff359fb 100644 --- a/tests/FluentCMS.Data.Tests/Storage/ModelBuildTimeTests.cs +++ b/tests/FluentCMS.Data.Tests/Storage/ModelBuildTimeTests.cs @@ -50,7 +50,7 @@ public void FiftyTypesBuildWithinTheBound() watch.Stop(); TestContext.Current.TestOutputHelper?.WriteLine($"Model of {TypeCount} types built in {watch.ElapsedMilliseconds} ms"); - model.GetEntityTypes().Count().ShouldBe(TypeCount + (TypeCount / 2) + 7); + model.GetEntityTypes().Count().ShouldBe(TypeCount + (TypeCount / 2) + 10); watch.Elapsed.ShouldBeLessThan(Bound); } } diff --git a/tests/FluentCMS.Data.Tests/Storage/SystemModelTests.cs b/tests/FluentCMS.Data.Tests/Storage/SystemModelTests.cs index 06fe8aaa3..dfa5496ac 100644 --- a/tests/FluentCMS.Data.Tests/Storage/SystemModelTests.cs +++ b/tests/FluentCMS.Data.Tests/Storage/SystemModelTests.cs @@ -4,6 +4,7 @@ using FluentCMS.Core.Tests.Schema; using FluentCMS.Publishing; using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Metadata; using Shouldly; using Xunit; @@ -77,6 +78,35 @@ public void VersionHasTheColumnsAndKeysOfTheSpecification() IndexColumns(entity, "uq_cms_version_document_locale_number").ShouldBe(["document_id", "locale", "number"]); } + [Fact] + public void JobTablesHaveTheColumnsOfTheSpecification() + { + using var database = NewDatabase(); + using var context = database.CreateContext(); + + var job = context.Model.FindEntityType(typeof(Job))!; + job.GetTableName().ShouldBe("cms_job"); + job.GetProperties().Select(property => property.GetColumnName()).Order(StringComparer.Ordinal).ShouldBe( + ["attempts", "completed_at", "created_at", "created_by", "id", "last_error", "lease_owner", "lease_until", "max_attempts", "origin", "payload", "priority", "queue", "run_at", "site_id", "status", "type"]); + var claim = context.GetService().Model.FindEntityType(typeof(Job))!.GetIndexes().Single(index => index.GetDatabaseName() == "ix_cms_job_claim"); + claim.Properties.Select(property => property.GetColumnName()).ShouldBe(["queue", "priority", "run_at", "id"]); + claim.IsDescending.ShouldNotBeNull().ShouldBe([false, true, false, false]); + claim.GetFilter().ShouldBe("status IN ('pending', 'running')"); + IndexColumns(job, "ix_cms_job_status_lease").ShouldBe(["status", "lease_until"]); + + var attempt = context.Model.FindEntityType(typeof(JobAttempt))!; + attempt.GetTableName().ShouldBe("cms_job_attempt"); + attempt.GetProperties().Select(property => property.GetColumnName()).Order(StringComparer.Ordinal).ShouldBe( + ["attempt", "error", "finished_at", "job_id", "outcome", "started_at"]); + attempt.FindPrimaryKey()!.Properties.Select(property => property.GetColumnName()).ShouldBe(["job_id", "attempt"]); + + // The specification lists no type, but a schedule has to say which job it fires. + var recurring = context.Model.FindEntityType(typeof(RecurringJob))!; + recurring.GetTableName().ShouldBe("cms_recurring_job"); + recurring.GetProperties().Select(property => property.GetColumnName()).Order(StringComparer.Ordinal).ShouldBe( + ["cron", "enabled", "name", "next_run_at", "payload", "queue", "type"]); + } + [Fact] public void DocumentRoundTripsWithAGeneratedVersion7Id() {