Tau.Acuvim/portal/src/Tau.Acuvim.Portal/Program.cs
Diseri Pearson a92b4277ae Phase 14: Push + ingest pipeline (end-to-end fleet aggregation)
Customer-stack measurements now flow to the Admin-stack central DB via
HTTPS POST, with firmware buffer-and-replay back-fills handled correctly.

Client side (push)
- monitoring.PowerMeasurements gains ReceivedAt (default NOW()) +
  index. Push selects WHERE ReceivedAt > LastCursor, so back-dated
  rows from offline-buffer replays are picked up automatically.
- app.FleetPushState table holds per-resource cursors + backoff state.
- FleetPushClient: HttpClient wrapper, X-Customer-Token header,
  X-Batch-Type, X-Push-Cursor. 413 returns retry-after halving signal.
- FleetPushService: BackgroundService loop. Per tick: sites (full set),
  devices (full set), measurements (cursor-driven up to 3 batches).
  Exponential backoff per resource on failure (1m → 30m cap).
  Honors 429 Retry-After. Only registered when RunMode=Client AND
  FleetIngest__Enabled=true.

Admin side (ingest)
- /api/fleet/ingest: anonymous, X-Customer-Token authed against
  fleet.Customers via SHA-256 indexed lookup. 401 on bad token; 400
  on bad batch type.
- FleetIngestService dispatches by X-Batch-Type:
  sites/devices → upsert by (CustomerId, Id) with ON CONFLICT UPDATE
  measurements → bulk INSERT ON CONFLICT (Time, CustomerId, DeviceId)
                 DO NOTHING (idempotent under re-delivery).
- Updates fleet.Customers.FirstSeenAt/LastSeenAt on each successful batch.
- Writes fleet.IngestEvents audit row per batch (accepted, rejected,
  bytes, client cursor, time-spread, error).
- FleetTimescaleBootstrapper runs after MigrateAsync in Admin mode:
  CREATE EXTENSION timescaledb, create_hypertable on fleet.PowerMeasurements,
  chunk interval 7 days, compression with segmentby=(CustomerId,DeviceId)
  + compress_orderby "Time" DESC, compression policy 7 days, hourly_per_device
  continuous aggregate (realtime, materialized_only=false, 30-day start_offset
  so back-fills get materialized on next refresh tick).

Wiring
- docker-compose.yml threads Application__RunMode + FleetIngest__* from
  .env (defaults safely off) so a single dev host can run two stacks.
- .env.example documents the new vars under their own section.

Tests
- FleetIngestValidationTests (2 new). 53/53 passing.

Verified end-to-end on the dev host
- Client (portal-dev_portal, RunMode=Client, FleetIngest__Enabled=true)
  pushes to Admin (portal-admin-test, RunMode=Admin, separate admin_fleet DB)
  via container DNS.
- Customer registered on Admin (DEV0001), token captured, dropped into
  Client .env, Client restarted, push service started on schedule.
- Ingested measurements (including a 2026-04-01 back-dated sample
  simulating firmware replay) all land in fleet.PowerMeasurements with
  the correct CustomerId.
- Customer.FirstSeenAt/LastSeenAt update, IngestEvents records every
  batch (sites + devices per tick, measurements when cursor advances).
- Hypertable confirmed via timescaledb_information.hypertables;
  hourly_per_device CA confirmed via timescaledb_information.continuous_aggregates.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-18 10:17:58 +02:00

303 lines
13 KiB
C#

using Microsoft.AspNetCore.DataProtection;
using Microsoft.AspNetCore.Identity;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.FileProviders;
using Microsoft.Extensions.Options;
using Serilog;
using Tau.Acuvim.Portal.Configuration;
using Tau.Acuvim.Portal.Constants;
using Tau.Acuvim.Portal.Data;
using Tau.Acuvim.Portal.Domain.Identity;
using Tau.Acuvim.Portal.Endpoints;
using Tau.Acuvim.Portal.Services;
Log.Logger = new LoggerConfiguration()
.WriteTo.Console()
.CreateBootstrapLogger();
try
{
var builder = WebApplication.CreateBuilder(args);
builder.Configuration.Sources.Insert(0, new Microsoft.Extensions.Configuration.Json.JsonConfigurationSource
{
Path = "appsettings.template.json",
Optional = false,
ReloadOnChange = false
});
builder.Configuration.AddJsonFile("appsettings.Local.json", optional: true, reloadOnChange: true);
builder.Host.UseSerilog((ctx, lc) => lc
.ReadFrom.Configuration(ctx.Configuration)
.WriteTo.Console());
builder.Services.Configure<ApplicationOptions>(builder.Configuration.GetSection(ApplicationOptions.SectionName));
builder.Services.Configure<DatabaseOptions>(builder.Configuration.GetSection(DatabaseOptions.SectionName));
builder.Services.Configure<TimescaleDbOptions>(builder.Configuration.GetSection(TimescaleDbOptions.SectionName));
builder.Services.Configure<GrafanaOptions>(builder.Configuration.GetSection(GrafanaOptions.SectionName));
builder.Services.Configure<WhiteLabelOptions>(builder.Configuration.GetSection(WhiteLabelOptions.SectionName));
builder.Services.Configure<AuthenticationOptions>(builder.Configuration.GetSection(AuthenticationOptions.SectionName));
builder.Services.Configure<MonitoringOptions>(builder.Configuration.GetSection(MonitoringOptions.SectionName));
builder.Services.Configure<FleetIngestOptions>(builder.Configuration.GetSection(FleetIngestOptions.SectionName));
var applicationOptions = builder.Configuration.GetSection(ApplicationOptions.SectionName).Get<ApplicationOptions>()
?? new ApplicationOptions();
var authOptions = builder.Configuration.GetSection(AuthenticationOptions.SectionName).Get<AuthenticationOptions>()
?? new AuthenticationOptions();
var databaseOptions = builder.Configuration.GetSection(DatabaseOptions.SectionName).Get<DatabaseOptions>()
?? new DatabaseOptions();
var timescaleOptions = builder.Configuration.GetSection(TimescaleDbOptions.SectionName).Get<TimescaleDbOptions>()
?? new TimescaleDbOptions();
var whiteLabelOptions = builder.Configuration.GetSection(WhiteLabelOptions.SectionName).Get<WhiteLabelOptions>()
?? new WhiteLabelOptions();
var fleetIngestOptions = builder.Configuration.GetSection(FleetIngestOptions.SectionName).Get<FleetIngestOptions>()
?? new FleetIngestOptions();
RunModeGuards.ValidateConfig(applicationOptions, databaseOptions, fleetIngestOptions);
var resolution = ConnectionStringResolver.Resolve(databaseOptions, timescaleOptions, builder.Environment);
Log.Information("RunMode={RunMode}, database connection resolved via {Source}",
applicationOptions.RunMode, resolution.Source);
builder.Services.AddSingleton(new DatabaseResolutionInfo(resolution.Source));
// ──────────────────────────────────────────────────────────────────────
// DbContext + Identity registration — branches on RunMode.
// Only one context is registered; the other never resolves.
// ──────────────────────────────────────────────────────────────────────
if (applicationOptions.RunMode == RunMode.Client)
{
builder.Services.AddDbContext<AppDbContext>(options => options.UseNpgsql(resolution.ConnectionString));
builder.Services.AddScoped<IWhiteLabelStore>(sp => sp.GetRequiredService<AppDbContext>());
builder.Services
.AddIdentity<ApplicationUser, IdentityRole>(ConfigureIdentity(authOptions))
.AddEntityFrameworkStores<AppDbContext>()
.AddDefaultTokenProviders();
}
else
{
builder.Services.AddDbContext<AdminDbContext>(options => options.UseNpgsql(resolution.ConnectionString));
builder.Services.AddScoped<IWhiteLabelStore>(sp => sp.GetRequiredService<AdminDbContext>());
builder.Services
.AddIdentity<ApplicationUser, IdentityRole>(ConfigureIdentity(authOptions))
.AddEntityFrameworkStores<AdminDbContext>()
.AddDefaultTokenProviders();
}
builder.Services.ConfigureApplicationCookie(options =>
{
options.Cookie.Name = authOptions.CookieName;
options.Cookie.HttpOnly = true;
options.Cookie.SameSite = SameSiteMode.Lax;
options.Cookie.SecurePolicy = builder.Environment.IsDevelopment()
? CookieSecurePolicy.SameAsRequest
: CookieSecurePolicy.Always;
options.ExpireTimeSpan = TimeSpan.FromHours(8);
options.SlidingExpiration = true;
options.Events.OnRedirectToLogin = ctx =>
{
ctx.Response.StatusCode = StatusCodes.Status401Unauthorized;
return Task.CompletedTask;
};
options.Events.OnRedirectToAccessDenied = ctx =>
{
ctx.Response.StatusCode = StatusCodes.Status403Forbidden;
return Task.CompletedTask;
};
});
builder.Services.AddAuthorization(options =>
{
options.AddPolicy(Policies.AdminOnly, policy => policy.RequireRole(Roles.Admin));
});
var keyRingPath = builder.Configuration["DataProtection:KeyRing"] ?? "/data/keys";
if (Directory.Exists(keyRingPath))
{
builder.Services.AddDataProtection()
.PersistKeysToFileSystem(new DirectoryInfo(keyRingPath))
.SetApplicationName("Tau.Acuvim.Portal");
}
// ──────────────────────────────────────────────────────────────────────
// Service registration — split by mode.
// ──────────────────────────────────────────────────────────────────────
builder.Services.AddScoped<BrandingService>();
builder.Services.AddScoped<IdentityBootstrapper>();
builder.Services.AddSingleton<GrafanaService>();
builder.Services.AddScoped<ConfigOverviewService>();
if (applicationOptions.RunMode == RunMode.Client)
{
builder.Services.AddScoped<RateService>();
builder.Services.AddSingleton<CostCalculator>();
builder.Services.AddScoped<TimescaleBootstrapper>();
builder.Services.AddScoped<MeasurementIngestService>();
builder.Services.AddScoped<MeasurementQueryService>();
if (fleetIngestOptions.Enabled)
{
builder.Services.AddHttpClient<FleetPushClient>(c =>
{
c.Timeout = TimeSpan.FromSeconds(30);
});
builder.Services.AddHostedService<FleetPushService>();
}
}
else
{
builder.Services.AddScoped<CustomerService>();
builder.Services.AddScoped<FleetIngestService>();
builder.Services.AddScoped<FleetTimescaleBootstrapper>();
}
builder.Services.AddHealthChecks()
.AddNpgSql(resolution.ConnectionString, name: "database", tags: new[] { "ready" });
builder.Services.AddEndpointsApiExplorer();
builder.Services.AddSwaggerGen(c =>
{
c.SwaggerDoc("v1", new() { Title = $"Tau Acuvim Portal API ({applicationOptions.RunMode})", Version = "v1" });
});
builder.Services.AddCors(options =>
{
options.AddPolicy("DevSpa", policy => policy
.WithOrigins("http://localhost:5174")
.AllowAnyMethod()
.AllowAnyHeader()
.AllowCredentials());
});
var app = builder.Build();
if (app.Environment.IsProduction())
{
var resolved = app.Services.GetRequiredService<IOptions<AuthenticationOptions>>().Value;
if (string.Equals(resolved.DefaultAdminPassword, "ChangeMe123!", StringComparison.Ordinal))
{
throw new InvalidOperationException(
"Authentication:DefaultAdminPassword is still the template default. " +
"Set it via env var Authentication__DefaultAdminPassword before running in Production.");
}
}
await RunModeGuards.ValidateDatabaseShapeAsync(app.Services, applicationOptions.RunMode);
if (databaseOptions.MigrateOnStartup)
{
using var scope = app.Services.CreateScope();
if (applicationOptions.RunMode == RunMode.Client)
{
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
await db.Database.MigrateAsync();
var timescale = scope.ServiceProvider.GetRequiredService<TimescaleBootstrapper>();
await timescale.EnsureHypertablesAsync();
}
else
{
var db = scope.ServiceProvider.GetRequiredService<AdminDbContext>();
await db.Database.MigrateAsync();
var fleetTimescale = scope.ServiceProvider.GetRequiredService<FleetTimescaleBootstrapper>();
await fleetTimescale.EnsureAsync();
}
var bootstrapper = scope.ServiceProvider.GetRequiredService<IdentityBootstrapper>();
await bootstrapper.SeedAsync();
}
Directory.CreateDirectory(whiteLabelOptions.LogoStoragePath);
app.UseSerilogRequestLogging();
app.Use(async (context, next) =>
{
context.Response.Headers["X-Content-Type-Options"] = "nosniff";
context.Response.Headers["X-Frame-Options"] = "SAMEORIGIN";
context.Response.Headers["Referrer-Policy"] = "strict-origin-when-cross-origin";
await next();
});
if (app.Environment.IsDevelopment())
{
app.UseCors("DevSpa");
app.UseSwagger();
app.UseSwaggerUI(c => c.SwaggerEndpoint("/swagger/v1/swagger.json", $"Tau Acuvim Portal API ({applicationOptions.RunMode})"));
}
else
{
app.UseHsts();
}
app.UseDefaultFiles();
app.UseStaticFiles();
app.UseStaticFiles(new StaticFileOptions
{
FileProvider = new PhysicalFileProvider(Path.GetFullPath(whiteLabelOptions.LogoStoragePath)),
RequestPath = "/branding-assets"
});
app.UseAuthentication();
app.UseAuthorization();
// ──────────────────────────────────────────────────────────────────────
// Endpoint mapping — shared first, then mode-specific.
// ──────────────────────────────────────────────────────────────────────
app.MapAuthEndpoints();
app.MapAdminUserEndpoints();
app.MapBrandingEndpoints();
app.MapGrafanaEndpoints();
app.MapAdminConfigEndpoints();
app.MapAppInfoEndpoints();
if (applicationOptions.RunMode == RunMode.Client)
{
app.MapRatesEndpoints();
app.MapAdminRatesEndpoints();
app.MapSitesEndpoints();
app.MapMeasurementsEndpoints();
}
else
{
app.MapAdminCustomersEndpoints();
app.MapFleetIngestEndpoints();
}
app.MapHealthChecks("/health", new Microsoft.AspNetCore.Diagnostics.HealthChecks.HealthCheckOptions
{
Predicate = _ => false
}).AllowAnonymous();
app.MapHealthChecks("/health/ready", new Microsoft.AspNetCore.Diagnostics.HealthChecks.HealthCheckOptions
{
Predicate = check => check.Tags.Contains("ready")
}).AllowAnonymous();
app.MapFallbackToFile("index.html");
app.Run();
}
catch (Exception ex)
{
Log.Fatal(ex, "Portal terminated unexpectedly");
}
finally
{
Log.CloseAndFlush();
}
static Action<IdentityOptions> ConfigureIdentity(AuthenticationOptions authOptions) => options =>
{
options.Password.RequireDigit = true;
options.Password.RequireLowercase = true;
options.Password.RequireUppercase = true;
options.Password.RequireNonAlphanumeric = false;
options.Password.RequiredLength = 8;
options.User.RequireUniqueEmail = true;
options.SignIn.RequireConfirmedEmail = authOptions.RequireConfirmedEmail;
options.Lockout.DefaultLockoutTimeSpan = TimeSpan.FromMinutes(15);
options.Lockout.MaxFailedAccessAttempts = 5;
};