Add the data provider packages, dialects and system migrations

Directory.Packages.props

@@ -10,9 +10,12 @@
     <PackageVersion Include="Microsoft.EntityFrameworkCore" Version="10.0.14" />
     <PackageVersion Include="Microsoft.EntityFrameworkCore.Design" Version="10.0.14" />
     <PackageVersion Include="Microsoft.EntityFrameworkCore.Relational" Version="10.0.14" />
+    <PackageVersion Include="Microsoft.EntityFrameworkCore.SqlServer" Version="10.0.14" />
     <PackageVersion Include="Microsoft.EntityFrameworkCore.Sqlite" Version="10.0.14" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="10.0.14" />
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.11.0" />
+    <PackageVersion Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="10.0.3" />
+    <PackageVersion Include="System.Security.Cryptography.Pkcs" Version="10.0.14" />
     <PackageVersion Include="xunit.v3" Version="4.0.2" />
     <PackageVersion Include="xunit.runner.visualstudio" Version="4.0.1" />
     <PackageVersion Include="Shouldly" Version="4.3.0" />

docs/system-migrations.md

@@ -0,0 +1,29 @@
+# System migrations
+
+`SystemDbContext` is the model of the system tables: what `SystemModelBuilder` maps, with no content tables and with the history table `cms__migrations`. Each data provider package holds the migrations of that model for its database, in its `Migrations` folder.
+
+## Adding a migration
+
+Do this in the pull request that changes `SystemModelBuilder`, such as one that adds `cms_site` or `cms_job`. You need the `dotnet-ef` tool (`dotnet tool install --global dotnet-ef`). No database is needed, because the design-time factory in each package never connects.
+
+1. Map the table in `SystemModelBuilder`.
+
+2. Scaffold the migration once per provider, with the same name. `dotnet ef` starts a fresh context each time, which matters: scaffolding on a context that has already called `HasPendingModelChanges()` silently emits `CreateTable` for every table.
+
+   ```bash
+   dotnet ef migrations add AddSites --project src/FluentCMS.Data.Sqlite --context SystemDbContext --output-dir Migrations
+   ```
+
+   ```bash
+   dotnet ef migrations add AddSites --project src/FluentCMS.Data.PostgreSql --context SystemDbContext --output-dir Migrations
+   ```
+
+   ```bash
+   dotnet ef migrations add AddSites --project src/FluentCMS.Data.SqlServer --context SystemDbContext --output-dir Migrations
+   ```
+
+3. Add `// <auto-generated />` as the first line of each new migration file. EF marks only the designer and snapshot files, and the composite index columns in the migration file otherwise fail the build with CA1861.
+
+4. Run `dotnet test --project tests/FluentCMS.Data.Tests -c Release`. The tests fail if a provider's snapshot no longer matches the model, so a skipped provider is caught.
+
+Do not edit a migration that has shipped. The PostgreSQL and SQL Server migrations have never been applied to a database; [#2711](https://github.com/fluentcms/FluentCMS/issues/2711) validates them.

src/FluentCMS.Core/DependencyInjection/FluentCmsBuilder.cs

@@ -0,0 +1,32 @@
+using FluentCMS.Core.Storage;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace FluentCMS.Core.DependencyInjection;
+
+public sealed class FluentCmsBuilder
+{
+    internal FluentCmsBuilder(IServiceCollection services)
+    {
+        Services = services;
+    }
+
+    public IServiceCollection Services { get; }
+
+    internal IDataProvider? DataProvider { get; private set; }
+
+    // The seam for a data provider package, whose own UseXxx method calls this.
+    public FluentCmsBuilder UseDataProvider(IDataProvider provider)
+    {
+        ArgumentNullException.ThrowIfNull(provider);
+
+        if (DataProvider is not null)
+        {
+            throw new FluentCmsConfigurationException(
+                StorageErrorCodes.SeveralDataProviders,
+                $"AddFluentCms got a second data provider, {provider.Name}, after {DataProvider.Name}; a host uses exactly one.");
+        }
+
+        DataProvider = provider;
+        return this;
+    }
+}

src/FluentCMS.Core/DependencyInjection/FluentCmsConfigurationException.cs

@@ -0,0 +1,13 @@
+namespace FluentCMS.Core.DependencyInjection;
+
+// A host is set up in a way that FluentCMS cannot start with.
+public sealed class FluentCmsConfigurationException : InvalidOperationException
+{
+    internal FluentCmsConfigurationException(string code, string message)
+        : base($"{code}: {message}")
+    {
+        Code = code;
+    }
+
+    public string Code { get; }
+}

src/FluentCMS.Core/DependencyInjection/FluentCmsServiceCollectionExtensions.cs

@@ -0,0 +1,46 @@
+using FluentCMS.Core.Schema;
+using FluentCMS.Core.Storage;
+using FluentCMS.Schema;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.DependencyInjection.Extensions;
+
+namespace FluentCMS.Core.DependencyInjection;
+
+public static class FluentCmsServiceCollectionExtensions
+{
+    public static IServiceCollection AddFluentCms(this IServiceCollection services, Action<FluentCmsBuilder> configure)
+    {
+        ArgumentNullException.ThrowIfNull(services);
+        ArgumentNullException.ThrowIfNull(configure);
+
+        var builder = new FluentCmsBuilder(services);
+        configure(builder);
+
+        var provider = builder.DataProvider ?? throw new FluentCmsConfigurationException(
+            StorageErrorCodes.NoDataProvider,
+            "AddFluentCms has no data provider. Add the package for your database and call cms.UseSqlite(...), cms.UsePostgreSql(...) or cms.UseSqlServer(...) inside AddFluentCms.");
+
+        services.AddSingleton(provider);
+        services.AddSingleton(provider.Conventions);
+        services.AddSingleton(provider.JsonDialect);
+        services.AddSingleton(provider.JobClaimDialect);
+
+        // Until content types can be added to the builder, a schema comes from the container, and none means system tables only.
+        services.TryAddSingleton(_ => new SchemaBuilder().Build());
+
+        services.AddDbContext<CmsDbContext>((serviceProvider, options) =>
+        {
+            provider.ConfigureDbContext(options);
+            options.UseCmsModel(serviceProvider.GetRequiredService<SchemaModel>(), provider.Conventions, provider.JsonDialect);
+        });
+
+        services.AddDbContext<SystemDbContext>(options =>
+        {
+            provider.ConfigureSystemMigrations(options);
+            options.UseSystemModel(provider.Conventions);
+        });
+
+        return services;
+    }
+}

src/FluentCMS.Core/Storage/CmsDbContext.cs

@@ -25,6 +25,8 @@ public CmsDbContext(DbContextOptions<CmsDbContext> options)
 
     internal string ConventionsKey => _model.Conventions.CacheKey;
 
+    internal string? JsonDialectKey => _model.JsonDialect?.GetType().FullName;
+
     // The rows of a content table, such as cms_article or cms_article_i18n.
     public DbSet<Dictionary<string, object>> Rows(string table) => Set<Dictionary<string, object>>(table);
 
@@ -35,5 +37,6 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
     {
         new SystemModelBuilder(_model.Conventions).Apply(modelBuilder);
         new ContentModelBuilder(_model.Schema, _model.Conventions).Apply(modelBuilder);
+        _model.JsonDialect?.ConfigureFunctions(modelBuilder);
     }
 }

src/FluentCMS.Core/Storage/CmsDbContextOptionsExtensions.cs

@@ -1,3 +1,4 @@
+using FluentCMS.Core.Schema;
 using FluentCMS.Schema;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore.Infrastructure;
@@ -6,16 +7,27 @@ namespace FluentCMS.Core.Storage;
 
 public static class CmsDbContextOptionsExtensions
 {
-    // Gives CmsDbContext the schema to build its model from. Without conventions the model stays neutral;
-    // a data provider package passes its own.
-    public static TBuilder UseCmsModel<TBuilder>(this TBuilder builder, SchemaModel schema, IStorageConventions? conventions = null)
+    private static readonly SchemaModel NoContent = new SchemaBuilder().Build();
+
+    // Gives CmsDbContext the schema to build its model from. Without conventions the model stays neutral, and
+    // without a dialect it has no JSON functions; a data provider package passes its own.
+    public static TBuilder UseCmsModel<TBuilder>(this TBuilder builder, SchemaModel schema, IStorageConventions? conventions = null, IJsonSqlDialect? jsonDialect = null)
         where TBuilder : DbContextOptionsBuilder
     {
         ArgumentNullException.ThrowIfNull(builder);
         ArgumentNullException.ThrowIfNull(schema);
 
-        ((IDbContextOptionsBuilderInfrastructure)builder).AddOrUpdateExtension(new CmsModelOptionsExtension(schema, conventions ?? new StorageConventions()));
+        ((IDbContextOptionsBuilderInfrastructure)builder).AddOrUpdateExtension(new CmsModelOptionsExtension(schema, conventions ?? new StorageConventions(), jsonDialect));
         builder.ReplaceService<IModelCacheKeyFactory, CmsModelCacheKeyFactory>();
         return builder;
     }
+
+    // Gives SystemDbContext the conventions to build its model with.
+    public static TBuilder UseSystemModel<TBuilder>(this TBuilder builder, IStorageConventions conventions)
+        where TBuilder : DbContextOptionsBuilder
+    {
+        ArgumentNullException.ThrowIfNull(conventions);
+
+        return builder.UseCmsModel(NoContent, conventions);
+    }
 }

src/FluentCMS.Core/Storage/CmsModelCacheKeyFactory.cs

@@ -5,7 +5,10 @@ namespace FluentCMS.Core.Storage;
 
 internal sealed class CmsModelCacheKeyFactory : IModelCacheKeyFactory
 {
-    public object Create(DbContext context, bool designTime) => context is CmsDbContext cms
-        ? (context.GetType(), cms.ModelKey, cms.ConventionsKey, designTime)
-        : (context.GetType(), designTime);
+    public object Create(DbContext context, bool designTime) => context switch
+    {
+        CmsDbContext cms => (context.GetType(), cms.ModelKey, cms.ConventionsKey, cms.JsonDialectKey, designTime),
+        SystemDbContext system => (context.GetType(), system.ConventionsKey, designTime),
+        _ => (context.GetType(), designTime),
+    };
 }

src/FluentCMS.Core/Storage/CmsModelOptionsExtension.cs

@@ -6,14 +6,16 @@ namespace FluentCMS.Core.Storage;
 
 // Carries the schema to CmsDbContext through its options, so the context has the one constructor that
 // AddDbContext, pooling and design-time factories expect.
-internal sealed class CmsModelOptionsExtension(SchemaModel schema, IStorageConventions conventions) : IDbContextOptionsExtension
+internal sealed class CmsModelOptionsExtension(SchemaModel schema, IStorageConventions conventions, IJsonSqlDialect? jsonDialect) : IDbContextOptionsExtension
 {
     private ExtensionInfo? _info;
 
     public SchemaModel Schema { get; } = schema;
 
     public IStorageConventions Conventions { get; } = conventions;
 
+    public IJsonSqlDialect? JsonDialect { get; } = jsonDialect;
+
     public DbContextOptionsExtensionInfo Info => _info ??= new ExtensionInfo(this);
 
     public void ApplyServices(IServiceCollection services)

src/FluentCMS.Core/Storage/IDataProvider.cs

@@ -0,0 +1,24 @@
+using Microsoft.EntityFrameworkCore;
+
+namespace FluentCMS.Core.Storage;
+
+// Everything a data provider package gives core. Its UseXxx method passes one to FluentCmsBuilder.UseDataProvider.
+public interface IDataProvider
+{
+    string Name { get; }
+
+    // The default of Database:MigrateOnStartup. The migration runner acts on it.
+    bool MigrateOnStartup { get; }
+
+    IStorageConventions Conventions { get; }
+
+    IJsonSqlDialect JsonDialect { get; }
+
+    IJobClaimDialect JobClaimDialect { get; }
+
+    // Points the options at the database.
+    void ConfigureDbContext(DbContextOptionsBuilder options);
+
+    // The same, for SystemDbContext: adds the assembly that holds the system migrations and their history table.
+    void ConfigureSystemMigrations(DbContextOptionsBuilder options);
+}

src/FluentCMS.Core/Storage/IJobClaimDialect.cs

@@ -0,0 +1,24 @@
+using Microsoft.EntityFrameworkCore.Infrastructure;
+
+namespace FluentCMS.Core.Storage;
+
+// The next job due on a queue, leased to one worker until LeaseUntil.
+public sealed record JobClaim(string Queue, string Owner, DateTimeOffset Now, DateTimeOffset LeaseUntil);
+
+// The values of cms_job.status that claiming reads and writes.
+public static class JobClaimStatus
+{
+    public const string Pending = "pending";
+
+    public const string Running = "running";
+}
+
+// Claims one job from cms_job, so that of any number of concurrent workers exactly one gets each job. A candidate
+// is pending or running, due, under max_attempts, and has no lease or an expired one, which is how a job comes back
+// after its worker died. The next is the highest priority, then the earliest run_at. The claim sets status to
+// running, takes the lease and adds one to attempts. Dates are bound as the provider's conventions store them.
+public interface IJobClaimDialect
+{
+    // The id of the claimed job, or null if there was nothing to claim.
+    Task<Guid?> ClaimAsync(DatabaseFacade database, JobClaim claim, CancellationToken cancellationToken = default);
+}

src/FluentCMS.Core/Storage/IJsonSqlDialect.cs

@@ -0,0 +1,33 @@
+using System.Linq.Expressions;
+using Microsoft.EntityFrameworkCore;
+
+namespace FluentCMS.Core.Storage;
+
+// Builds the JSON filters of the query compiler as LINQ expressions over a provider's EF function mappings,
+// limited to what Q7 supports. Each method returns a bool expression, so filters compose with And, Or and Not.
+//
+// A json argument is a string expression holding JSON: EF.Property<string>(row, "sections"), or the block that
+// AnyBlock hands to its predicate. A value argument is the compiler's parameter expression; bind it by reading
+// a member of a holder object, because a bare constant becomes a SQL literal.
+public interface IJsonSqlDialect
+{
+    // Declares the functions the expressions below call. CmsDbContext applies this to its model.
+    void ConfigureFunctions(ModelBuilder modelBuilder);
+
+    // The block list holds a block of this type. blockType is a string expression.
+    Expression HasBlock(Expression json, Expression blockType);
+
+    // The block list holds a block of this type that also satisfies the predicate, so that every condition
+    // applies to the same block. The predicate receives the block and calls the Scalar and Array methods on it.
+    Expression AnyBlock(Expression json, Expression blockType, Func<Expression, Expression> predicate);
+
+    // The value at the top-level key has this type and equals the value, whose CLR type is JsonScalar.ClrType(type).
+    // A JSON number never equals the text '12'.
+    Expression ScalarEquals(Expression json, string field, JsonScalarType type, Expression value);
+
+    // The value at the top-level key is JSON text that contains the string value, case-sensitively.
+    Expression ScalarContains(Expression json, string field, Expression value);
+
+    // The value at the top-level key is an array holding the string value as an element.
+    Expression ArrayContains(Expression json, string field, Expression value);
+}

src/FluentCMS.Core/Storage/JsonScalar.cs

@@ -0,0 +1,57 @@
+using System.Linq.Expressions;
+using FluentCMS.Core.Schema.FieldTypes;
+
+namespace FluentCMS.Core.Storage;
+
+// How a scalar inside JSON is compared. A JSON extraction keeps the value's own type, so each kind has its own comparison.
+public enum JsonScalarType
+{
+    Text,
+    WholeNumber,
+    DecimalNumber,
+    Boolean,
+}
+
+public static class JsonScalar
+{
+    // The comparison for a field type, or null if the type has no scalar value inside JSON. Dates are compared as
+    // the ISO 8601 text that the JSON holds.
+    public static JsonScalarType? TypeOf(string fieldType) => fieldType switch
+    {
+        FieldTypeNames.Text or FieldTypeNames.LongText or FieldTypeNames.Email or FieldTypeNames.Url or FieldTypeNames.Slug
+            or FieldTypeNames.Select or FieldTypeNames.Date or FieldTypeNames.DateTime => JsonScalarType.Text,
+        FieldTypeNames.Integer => JsonScalarType.WholeNumber,
+        FieldTypeNames.Decimal => JsonScalarType.DecimalNumber,
+        FieldTypeNames.Boolean => JsonScalarType.Boolean,
+        _ => null,
+    };
+
+    public static Type ClrType(JsonScalarType type) => type switch
+    {
+        JsonScalarType.Text => typeof(string),
+        JsonScalarType.WholeNumber => typeof(long),
+        JsonScalarType.DecimalNumber => typeof(decimal),
+        JsonScalarType.Boolean => typeof(bool),
+        _ => throw new ArgumentOutOfRangeException(nameof(type), type, null),
+    };
+
+    // Q7 caps paths at one level. Dialects put the result into a path or key literal, so only a plain name may pass.
+    public static string CheckField(string field)
+    {
+        ArgumentException.ThrowIfNullOrEmpty(field);
+
+        return field.All(character => char.IsAsciiLetterOrDigit(character) || character == '_')
+            ? field
+            : throw new ArgumentException($"'{field}' is not a one-level JSON path.", nameof(field));
+    }
+
+    public static void CheckValue(JsonScalarType type, Expression value)
+    {
+        ArgumentNullException.ThrowIfNull(value);
+
+        if (value.Type != ClrType(type))
+        {
+            throw new ArgumentException($"A {type} comparison needs a {ClrType(type).Name} value, not {value.Type.Name}.", nameof(value));
+        }
+    }
+}

src/FluentCMS.Core/Storage/StorageErrorCodes.cs

@@ -0,0 +1,8 @@
+namespace FluentCMS.Core.Storage;
+
+// FCMS1301 is the runner's missing column error (#2690).
+internal static class StorageErrorCodes
+{
+    public const string NoDataProvider = "FCMS1302";
+    public const string SeveralDataProviders = "FCMS1303";
+}