using Admin.NET.Plugin.AiDOP.DataPlatform.Executors;
using Admin.NET.Plugin.AiDOP.Entity.DataPlatform;
using Microsoft.Extensions.Logging;
using SqlSugar;
namespace Admin.NET.Plugin.AiDOP.DataPlatform.S0Dim;
///
/// 单个 S0 维度的物化:source → staging → dim,FULL REPLACE 语义。
///
/// 时序与事务边界:
///
/// [0] 前置校验(只读):mdp_entity 配置与 definition 一致 + 源侧业务键无重复 + 数据量未超单页
/// [1] source_count(只读,按租户)
/// ── 阶段 I:staging(非事务,可重跑)
/// [2] purge 本租户+本源系统+本源表 分区
/// [3] FULL pull(源侧/写入侧双重租户过滤)
/// [4] staging_count(本批四段谓词)
/// ── 阶段 II:闸门
/// [5] stg_count != source_count → 本维度 FAILED,**dim 不动**(保持上一轮完整快照)
/// ── 阶段 III:dim(单事务)
/// [6] DELETE tenant 行 + INSERT(无 ON DUPLICATE KEY)+ 事务内阻断级对账 → 任一失败整体回滚
///
///
/// 失败时 dim 永远停在**上一轮的完整快照**,不会半新半旧。
///
public sealed class S0DimMaterializer : ITransient
{
private readonly ISqlSugarClient _db;
private readonly MdpSourcePullDispatcher _pullDispatcher;
private readonly S0DimStagingPurge _purge;
private readonly S0DimSameDbStagingLoader _sameDbLoader;
private readonly S0DimReconciler _reconciler;
private readonly ILogger _logger;
/// 构造。
public S0DimMaterializer(
ISqlSugarClient db,
MdpSourcePullDispatcher pullDispatcher,
S0DimStagingPurge purge,
S0DimSameDbStagingLoader sameDbLoader,
S0DimReconciler reconciler,
ILogger logger)
{
_db = db;
_pullDispatcher = pullDispatcher;
_purge = purge;
_sameDbLoader = sameDbLoader;
_reconciler = reconciler;
_logger = logger;
}
/// 物化一个维度。
public async Task RunAsync(
S0DimDefinition def, long tenantId, string batchId, CancellationToken ct = default)
{
def.Validate();
if (tenantId <= 0) throw new InvalidOperationException($"[{def.Key}] 拒绝无效租户:{tenantId}");
if (string.IsNullOrWhiteSpace(batchId)) throw new InvalidOperationException($"[{def.Key}] batchId 不得为空");
var result = new S0DimMaterializeResult { Key = def.Key, TenantId = tenantId, BatchId = batchId };
var ps = new List
{
new("@TenantId", tenantId),
new("@SourceSystem", def.SourceSystem),
new("@SourceTable", def.SourceTable),
new("@BatchId", batchId)
};
// 🔴 全局 CommandTimeOut = 30s(Admin.NET.Core/SqlSugar/SqlSugarSetup.cs:155),对本管线不够用。
// 实测(mdp_transform_run_log,2026-09-08):ITEM 单维度平均 132.7s、最大 1169.8s;
// ITEM_BOM 的同库 INSERT ... SELECT 单条就要 ~27s,**紧贴 30s 线**,已实际炸过三次:
// 45747 ITEM "The Command Timeout expired before the operation completed."
// 45941 ITEM_BOM "Connection must be Open; current state is Closed"
// 45945 ITEM_BOM "The Command Timeout expired before the operation completed."
// 手工逐个刷时人会重试,所以问题被掩盖;自动刷新上线后会变成每拍稳定失败。
// 这里只在物化区间内临时放宽,Dispose 时还原(异常路径同样还原),
// 不改全局常量、不影响其它模块 —— 范式取自
// MaterialWarehouse/InventoryMdpSyncService.cs:627-644。
using var commandTimeout = new LongCommandTimeoutScope(_db, MaterializeCommandTimeoutSeconds);
try
{
// [0] 前置校验:配置漂移与源侧数据质量,全部在动任何数据之前
var batchSize = await AssertEntityContractAsync(def, ct);
var sourceCount = await _db.Ado.GetIntAsync(S0DimSqlBuilder.BuildSourceCountSql(def), ps);
await AssertNoSourceDuplicateAsync(def, tenantId, sourceCount, batchSize, ps);
// [2] purge:三段谓词精确到本租户 + 本源系统 + 本源表
result.StagingPurged = await _purge.PurgeAsync(def, tenantId, ct);
// [3] FULL pull
// 租户三重防护:① MdpDbPullExecutor 对含租户列的源表加 WHERE tenant_id=@scopeTenantId;
// ② MdpStagingWriter 对 tenantValue != ctx.TenantId 的行直接跳过;
// ③ RequireMatchingSourceTenant=true → 源行无租户时不拿 ctx 兜底,直接跳过(绝不落 0)
var pullCtx = new MdpPullContext
{
TenantId = tenantId,
BatchId = batchId,
FullRefresh = true,
RequireMatchingSourceTenant = true
};
// 同库时优先走集合式装载(一条 INSERT ... SELECT,DB 往返 O(1));
// 判定不成立或计划生成失败一律回退下面的逐行路径,语义不变、只是慢。
// 参见 S0DimSameDbStagingLoader:判定只依据源与库的客观属性,
// 不看 entity 名 / source_code 字面量 / 主机硬编码。
var samedb = await _sameDbLoader.TryPlanAsync(def, ct);
if (samedb is not null)
{
var written = await _sameDbLoader.LoadAsync(def, samedb, tenantId, batchId, ct);
// 集合式语句无「读了多少行」的独立计数,装载行数即贴源行数;
// 真正的把关在下面 [4][5] 的 stg == source 闸门,不依赖这里的自报数。
result.SourcePulled = written;
result.StagingWritten = written;
_logger.LogInformation("[S0Dim] {Key} 走同库快路径:{Reason},写入 {Rows} 行",
def.Key, samedb.Reason, written);
}
else
{
var pull = await _pullDispatcher.PullAllByEntityCodeAsync(def.EntityCode, pullCtx, ct);
result.SourcePulled = pull.RowsPulled;
result.StagingWritten = pull.RowsWritten;
}
// [4][5] 闸门:本批 staging 必须与源侧行数完全一致,否则不碰 dim
var stagingCount = await _db.Ado.GetIntAsync(S0DimSqlBuilder.BuildStagingCountSql(def), ps);
if (stagingCount != sourceCount)
throw new InvalidOperationException(
$"[{def.Key}] 贴源闸门未通过:source_count={sourceCount} 本批 stg_count={stagingCount}(dim 未改动)");
// [6] dim 单事务:DELETE 本租户 → INSERT → 事务内阻断级对账
result.DimRows = await MdpStdFullReplace.ReplaceAsync(
_db, def.DimTable, tenantId, extraWhere: null,
insertScopedAsync: async () =>
{
var now = DateTime.Now;
var insertPs = new List(ps) { new("@Now", now) };
var rows = await _db.Ado.ExecuteCommandAsync(S0DimSqlBuilder.BuildInsertSql(def), insertPs);
result.Reconcile = await _reconciler.AssertBlockingAsync(def, tenantId, batchId, ct);
return rows;
},
ct);
result.Status = "SUCCESS";
_logger.LogInformation(
"[S0Dim] {Key} tenant={Tenant} batch={Batch} purged={Purged} pulled={Pulled} stg={Stg} dim={Dim}",
def.Key, tenantId, batchId, result.StagingPurged, result.SourcePulled, stagingCount, result.DimRows);
}
catch (OperationCanceledException)
{
// 取消不是本维度的「失败」:吞掉它会把客户端断开记成 FAILED,
// 且外层 RefreshAsync 的 ct.ThrowIfCancellationRequested() 会在下一轮抛出,
// 导致 CompleteRunLogAsync 永不执行、run log 永久停在 RUNNING。
// 事务侧无需担心:异常已在 MdpStdFullReplace 的 catch 中回滚,dim 未改动。
throw;
}
catch (Exception ex)
{
result.Status = "FAILED";
result.Error = ex.Message;
_logger.LogError(ex, "[S0Dim] {Key} tenant={Tenant} batch={Batch} 物化失败(dim 未改动)",
def.Key, tenantId, batchId);
}
return result;
}
///
/// 物化区间的命令超时(秒)。取 900 与 InventoryMdpSyncService 一致:
/// 覆盖实测最慢的 ITEM(单维度最大 1169.8s 是整轮耗时,其中单条命令远小于此)并留足余量。
///
private const int MaterializeCommandTimeoutSeconds = 900;
/// 物化区间内临时放宽命令超时,Dispose 时还原原值(异常路径同样还原)。
private sealed class LongCommandTimeoutScope : IDisposable
{
private readonly ISqlSugarClient _db;
private readonly int _original;
public LongCommandTimeoutScope(ISqlSugarClient db, int seconds)
{
_db = db;
_original = db.Ado.CommandTimeOut;
db.Ado.CommandTimeOut = seconds;
}
public void Dispose() => _db.Ado.CommandTimeOut = _original;
}
///
/// 断言 mdp_entity 的运行时配置与 definition 完全一致,返回 batch_size。
/// 这一步把「配置漂移」变成显式失败 —— 否则 PullAll 可能悄悄拉到别的表 / 别的源。
///
private async Task AssertEntityContractAsync(S0DimDefinition def, CancellationToken ct)
{
var entity = await _db.Queryable()
.Where(x => x.EntityCode == def.EntityCode)
.FirstAsync(ct)
?? throw new InvalidOperationException($"[{def.Key}] mdp_entity 未登记:{def.EntityCode}(migration 未执行?)");
if (entity.Status != 1)
throw new InvalidOperationException($"[{def.Key}] mdp_entity.{def.EntityCode} 已停用(status={entity.Status})");
var source = await _db.Queryable().Where(x => x.Id == entity.SourceId).FirstAsync(ct)
?? throw new InvalidOperationException($"[{def.Key}] mdp_source id={entity.SourceId} 不存在");
void Expect(string what, string? actual, string expected)
{
if (!string.Equals(actual, expected, StringComparison.Ordinal))
throw new InvalidOperationException($"[{def.Key}] mdp_entity 配置漂移:{what} 实际='{actual}' 期望='{expected}'");
}
Expect("source_code", source.SourceCode, def.SourceSystem);
Expect("source_table_name", entity.SourceTableName, def.SourceTable);
Expect("target_table_name", entity.TargetTableName, def.StagingTable);
Expect("biz_key_expr", entity.BizKeyExpr, string.Join(",", def.SourceBizKeyColumns));
// ── 执行器选路的三个开关:不查会被静默改道 ──
// MdpSourcePullDispatcher 按
// !IsNullOrWhiteSpace(entity.SourceApiPath) || source.SourceType == "API" → _apiExecutor
// source.SourceType == "FILE_EXCEL" → Excel 分支
// 选择执行器。上面那几条 Expect 全部**照样通过**,但拉取已不再是「本库 SELECT」。
// 比较口径与 dispatcher 一致(OrdinalIgnoreCase),否则大小写差异会造成假通过。
if (!string.IsNullOrWhiteSpace(entity.SourceApiPath))
throw new InvalidOperationException(
$"[{def.Key}] mdp_entity.source_api_path 必须为空,实际='{entity.SourceApiPath}':" +
"非空会让 MdpSourcePullDispatcher 改走 API 执行器,绕开本库 DB 拉取契约");
if (!string.Equals(source.SourceType, "DB", StringComparison.OrdinalIgnoreCase))
throw new InvalidOperationException(
$"[{def.Key}] mdp_source.source_type 必须为 DB,实际='{source.SourceType}'");
if (!string.Equals(source.DbType, "MySQL", StringComparison.OrdinalIgnoreCase))
throw new InvalidOperationException(
$"[{def.Key}] mdp_source.db_type 必须为 MySQL,实际='{source.DbType}'");
// 本阶段只做 FULL:留了 incr_column 会让 BuildSelectSql 生成 "col > @cursor",
// 使源侧水位为 NULL 的行(如 DepartmentMaster 的 UATDEMO/未分配)永久不可达
if (!string.Equals(entity.SyncMode, "FULL", StringComparison.OrdinalIgnoreCase))
throw new InvalidOperationException($"[{def.Key}] sync_mode 必须为 FULL,实际='{entity.SyncMode}'");
if (!string.IsNullOrWhiteSpace(entity.IncrColumn))
throw new InvalidOperationException($"[{def.Key}] incr_column 必须为空(FULL 语义),实际='{entity.IncrColumn}'");
return entity.BatchSize > 0 ? entity.BatchSize : 1000;
}
///
/// 源侧业务键重复探针 + 单页容量检查。两者都在 purge 之前,失败时**任何数据都未被触碰**。
///
private async Task AssertNoSourceDuplicateAsync(
S0DimDefinition def, long tenantId, int sourceCount, int batchSize, List ps)
{
// 分页依赖 MdpDbPullExecutor 无 incr_column 时的 "ORDER BY 1"(按第 1 个物理列),
// 而 LocationMaster 的第 1 列是 Capacity(非唯一)→ 一旦真的翻页,OFFSET 结果不稳定。
// 因此要求单页装得下;超出时显式失败并提示调大 mdp_entity.batch_size。
if (sourceCount >= batchSize)
throw new InvalidOperationException(
$"[{def.Key}] 源行数 {sourceCount} 已达单页容量 {batchSize}:" +
"通用执行器在无 incr_column 时按 'ORDER BY 1' 分页,对本表非稳定序,请调大 mdp_entity.batch_size");
var q = await _db.Ado.SqlQueryAsync(
S0DimSqlBuilder.BuildBizKeyQualitySql(S0DimSqlBuilder.SourceBizKeySetSql(def)), ps);
var probe = q.FirstOrDefault();
if (probe is null) return;
if (probe.Blank_Cnt > 0)
throw new InvalidOperationException(
$"[{def.Key}] tenant={tenantId} 源侧有 {probe.Blank_Cnt} 行业务键为空,拒绝物化");
if (probe.Total != probe.Distinct_Cnt)
throw new InvalidOperationException(
$"[{def.Key}] tenant={tenantId} 源侧业务键重复(total={probe.Total} distinct={probe.Distinct_Cnt}):" +
"主数据唯一性已被破坏,transform 显式失败 —— 不做 first-wins / last-wins 静默取舍");
}
private sealed class SourceBizKeyProbe
{
public int Total { get; set; }
public int Distinct_Cnt { get; set; }
public int Blank_Cnt { get; set; }
}
}