using System.Text;
namespace Admin.NET.Plugin.AiDOP.DataPlatform.S0Dim;
///
/// 由 生成 SQL 的**唯一**位置(纯函数,不接触 DB,可单测)。
///
/// 生成的所有语句都必须能逐条证明租户范围:
/// staging 侧固定四段谓词 tenant_id / source_system / source_table / sync_batch_id;
/// dim 侧固定 tenant_id;source 侧固定 tenant_id。
///
public static class S0DimSqlBuilder
{
/// staging 别名。
public const string StgAlias = "s";
/// dim 别名。
public const string DimAlias = "d";
/// source 别名。
public const string SrcAlias = "a";
/// 业务键各段之间的分隔符,与 MdpStagingWriter.BuildBizKey 的 string.Join("#") 一致。
public const string BizKeySeparator = "#";
/// checksum 行内字段分隔(US,业务值不可能包含)。
private const string FieldSep = "CHAR(31 USING utf8mb4)";
/// checksum 的 NULL 哨兵(RS),与空串可区分。
private const string NullSentinel = "CHAR(30 USING utf8mb4)";
private static string Q(string ident)
{
if (!S0DimDefinition.IdentifierRe.IsMatch(ident))
throw new InvalidOperationException($"非法标识符:{ident}");
return $"`{ident}`";
}
/// 列取值表达式(作用于 staging 的 raw_data;TenantIdColumn 例外)。
private static string ValueExpr(S0DimColumn c) => c.Kind switch
{
S0DimValueKind.TenantIdColumn => $"{StgAlias}.{Q(S0DimDefinition.TenantColumn)}",
S0DimValueKind.Str => MdpJsonSql.Str(StgAlias, c.JsonPath!),
S0DimValueKind.Int => MdpJsonSql.Int(StgAlias, c.JsonPath!),
S0DimValueKind.Dec => MdpJsonSql.Dec(StgAlias, c.JsonPath!, c.Precision, c.Scale),
S0DimValueKind.DateTimeSec => MdpJsonSql.DateTimeSec(StgAlias, c.JsonPath!),
S0DimValueKind.BoolTrue => MdpJsonSql.BoolTrue(StgAlias, c.JsonPath!),
_ => throw new InvalidOperationException($"未支持的取值方式:{c.Kind}")
};
///
/// staging 四段固定谓词。 为 false 时用于「不限本批」的只读对账。
///
public static string StagingFilter(bool withBatch = true)
{
var sb = new StringBuilder();
sb.Append($"{StgAlias}.{Q(S0DimDefinition.TenantColumn)} = @TenantId");
sb.Append($" AND {StgAlias}.`source_system` = @SourceSystem");
sb.Append($" AND {StgAlias}.`source_table` = @SourceTable");
if (withBatch) sb.Append($" AND {StgAlias}.`sync_batch_id` = @BatchId");
return sb.ToString();
}
/// 行级过滤:Required 列非空 + 布尔列可辨识(不满足的行不进 dim,随后被行数对账捕获)。
private static IReadOnlyList RowGuards(S0DimDefinition def)
{
var guards = new List();
foreach (var c in def.Columns)
{
if (c.Kind == S0DimValueKind.TenantIdColumn) continue;
if (c.Required)
{
guards.Add($"({ValueExpr(c)}) IS NOT NULL");
// IS NOT NULL 挡不住空串。业务键走 CONCAT_WS 拼接,空段会被**保留**,
// 于是 Domain='' 会产出 '#LOC01' 这种「看起来合法」的键,而
// AssertNoSourceDuplicateAsync 的 blank_cnt 只捕获整串全空的情况。
// 当前 Catalog 中 Required 的 Str 列恰好就是全部业务键分量
// (domain_code / work_center_code / department_code / location_code / shelf_code),
// 故此守卫命中面 = 业务键,无附带影响。
if (c.Kind == S0DimValueKind.Str)
guards.Add($"({ValueExpr(c)}) <> ''");
}
if (c.Kind == S0DimValueKind.BoolTrue)
{
// MdpJsonSql.BoolTrue 对 null / 不可识别值静默返回 0;此处强制要求原始值可辨识,
// 否则整行被过滤 → dim_count < stg_count → 对账 FAIL(暴露而非吞掉)
var raw = MdpJsonSql.Raw(StgAlias, c.JsonPath!);
guards.Add($"LOWER({raw}) IN ('1','0','true','false')");
}
}
return guards;
}
///
/// dim 物化语句:INSERT INTO dim (...) SELECT ... FROM staging。
/// **刻意不带 ON DUPLICATE KEY UPDATE** —— 源侧业务键重复时直接撞 dim 唯一键报错,
/// 由外层事务回滚(transform fail),绝不 last-wins。
/// 参数:@TenantId @SourceSystem @SourceTable @BatchId @Now
///
public static string BuildInsertSql(S0DimDefinition def)
{
def.Validate();
var cols = new List();
var vals = new List();
foreach (var c in def.Columns)
{
cols.Add(Q(c.TargetColumn));
vals.Add(ValueExpr(c));
}
cols.AddRange(["`source_system`", "`source_biz_key`", "`sync_batch_id`", "`sync_time`"]);
vals.AddRange([$"{StgAlias}.`source_system`", $"{StgAlias}.`source_biz_key`", "@BatchId", "@Now"]);
var where = new List { StagingFilter() };
where.AddRange(RowGuards(def));
return $"""
INSERT INTO {Q(def.DimTable)}
({string.Join(", ", cols)})
SELECT
{string.Join(",\n ", vals)}
FROM {Q(def.StagingTable)} {StgAlias}
WHERE {string.Join("\n AND ", where)}
""";
}
/// staging 分区清理(purge)。参数:@TenantId @SourceSystem @SourceTable
public static string BuildPurgeStagingSql(S0DimDefinition def) =>
$"""
DELETE {StgAlias} FROM {Q(def.StagingTable)} {StgAlias}
WHERE {StagingFilter(withBatch: false)}
""";
/// 源表行数(按租户)。参数:@TenantId
public static string BuildSourceCountSql(S0DimDefinition def) =>
$"SELECT COUNT(*) FROM {Q(def.SourceTable)} {SrcAlias} " +
$"WHERE {SrcAlias}.{Q(S0DimDefinition.TenantColumn)} = @TenantId";
/// 源表中无法归属租户的行数(tenant_id IS NULL 或 <= 0)—— 这些行整条链路都看不见。
public static string BuildSourceUntenantedCountSql(S0DimDefinition def) =>
$"SELECT COUNT(*) FROM {Q(def.SourceTable)} {SrcAlias} " +
$"WHERE {SrcAlias}.{Q(S0DimDefinition.TenantColumn)} IS NULL " +
$"OR {SrcAlias}.{Q(S0DimDefinition.TenantColumn)} <= 0";
/// staging 行数。参数:@TenantId @SourceSystem @SourceTable [@BatchId]
public static string BuildStagingCountSql(S0DimDefinition def, bool withBatch = true) =>
$"SELECT COUNT(*) FROM {Q(def.StagingTable)} {StgAlias} WHERE {StagingFilter(withBatch)}";
/// dim 行数(按租户)。参数:@TenantId
public static string BuildDimCountSql(S0DimDefinition def) =>
$"SELECT COUNT(*) FROM {Q(def.DimTable)} {DimAlias} " +
$"WHERE {DimAlias}.{Q(S0DimDefinition.TenantColumn)} = @TenantId";
/// dim 中不属于本批的残留行数(FULL Replace 之后必须为 0)。参数:@TenantId @BatchId
public static string BuildDimBatchImpuritySql(S0DimDefinition def) =>
$"SELECT COUNT(*) FROM {Q(def.DimTable)} {DimAlias} " +
$"WHERE {DimAlias}.{Q(S0DimDefinition.TenantColumn)} = @TenantId " +
$"AND {DimAlias}.`sync_batch_id` <> @BatchId";
/// 源侧业务键表达式(物理列,与 mdp_entity.biz_key_expr 同序同分隔)。
public static string SourceBizKeyExpr(S0DimDefinition def) =>
$"CONCAT_WS('{BizKeySeparator}', {string.Join(", ", def.SourceBizKeyColumns.Select(c => $"{SrcAlias}.{Q(c)}"))})";
/// dim 侧业务键表达式(去掉 tenant_id 后的业务键列)。
public static string DimBizKeyExpr(S0DimDefinition def) =>
$"CONCAT_WS('{BizKeySeparator}', {string.Join(", ", def.BusinessKeyColumns.Skip(1).Select(c => $"{DimAlias}.{Q(c)}"))})";
/// 三层业务键集合来源子查询。
public static string SourceBizKeySetSql(S0DimDefinition def) =>
$"SELECT {SourceBizKeyExpr(def)} AS bk FROM {Q(def.SourceTable)} {SrcAlias} " +
$"WHERE {SrcAlias}.{Q(S0DimDefinition.TenantColumn)} = @TenantId";
///
public static string StagingBizKeySetSql(S0DimDefinition def, bool withBatch = true) =>
$"SELECT {StgAlias}.`source_biz_key` AS bk FROM {Q(def.StagingTable)} {StgAlias} WHERE {StagingFilter(withBatch)}";
///
public static string DimBizKeySetSql(S0DimDefinition def) =>
$"SELECT {DimBizKeyExpr(def)} AS bk FROM {Q(def.DimTable)} {DimAlias} " +
$"WHERE {DimAlias}.{Q(S0DimDefinition.TenantColumn)} = @TenantId";
/// 集合差集计数: \ 。
public static string BuildAntiJoinCountSql(string leftSql, string rightSql) =>
$"""
SELECT COUNT(*) FROM ({leftSql}) x
LEFT JOIN ({rightSql}) y ON y.bk <=> x.bk
WHERE y.bk IS NULL
""";
/// 集合内是否有重复 / 空业务键:返回 (总数, 去重数, 空值数)。
public static string BuildBizKeyQualitySql(string setSql) =>
$"""
SELECT COUNT(*) AS total, COUNT(DISTINCT x.bk) AS distinct_cnt,
SUM(CASE WHEN x.bk IS NULL OR x.bk = '' THEN 1 ELSE 0 END) AS blank_cnt
FROM ({setSql}) x
""";
/// NULL 归一(checksum 用)。
private static string Norm(string expr, S0DimValueKind kind) => kind switch
{
S0DimValueKind.DateTimeSec => $"IFNULL(DATE_FORMAT({expr}, '%Y-%m-%d %H:%i:%s'), {NullSentinel})",
S0DimValueKind.BoolTrue => $"IFNULL(CAST(CAST({expr} AS UNSIGNED) AS CHAR), {NullSentinel})",
_ => $"IFNULL(CAST({expr} AS CHAR), {NullSentinel})"
};
private static IReadOnlyList ChecksumColumns(S0DimDefinition def) =>
def.Columns.Where(c => c.Kind != S0DimValueKind.TenantIdColumn).ToList();
///
/// 单行哈希:MD5 前 15 个十六进制位 → 60 bit 无符号整数。
///
/// 🔴 CAST(... AS UNSIGNED) **不可省略**:CONV() 返回的是**字符串**,
/// 而 MySQL 的 SUM(字符串) 会按 DOUBLE 累加 —— 只有约 16 位有效数字,
/// 19 位的和末几位不可靠,且**两侧扫描顺序不同会产生不同舍入**,
/// 于是完全相同的数据也会算出不同校验和(2026-09-07 Batch 2 实测:
/// 同一租户 source=…980000 / dim=…980700,count 相同却误报不一致)。
/// 转成整数后 SUM 走 DECIMAL 精确累加,与顺序无关。
///
private static string RowHash(string rowStringExpr) =>
$"CAST(CONV(SUBSTRING(MD5({rowStringExpr}),1,15),16,10) AS UNSIGNED)";
///
/// 属性校验和:SUM()。
/// 用 SUM 而非 GROUP_CONCAT/BIT_XOR:交换律使**排序与结果无关**(无需 ORDER BY,也不受 group_concat_max_len 截断);
/// XOR 会让成对重复相互抵消从而掩盖重复。必须与 COUNT 成对使用。
///
public static string BuildSourceChecksumSql(S0DimDefinition def)
{
var parts = ChecksumColumns(def).Select(c => Norm($"{SrcAlias}.{Q(c.JsonPath!)}", c.Kind));
var rowStr = $"CONCAT_WS({FieldSep}, {Norm(SourceBizKeyExpr(def), S0DimValueKind.Str)}, {string.Join(", ", parts)})";
return $"SELECT COUNT(*) AS cnt, CAST(IFNULL(SUM({RowHash(rowStr)}),0) AS DECIMAL(40,0)) AS chk " +
$"FROM {Q(def.SourceTable)} {SrcAlias} WHERE {SrcAlias}.{Q(S0DimDefinition.TenantColumn)} = @TenantId";
}
///
public static string BuildDimChecksumSql(S0DimDefinition def)
{
var parts = ChecksumColumns(def).Select(c => Norm($"{DimAlias}.{Q(c.TargetColumn)}", c.Kind));
var rowStr = $"CONCAT_WS({FieldSep}, {Norm(DimBizKeyExpr(def), S0DimValueKind.Str)}, {string.Join(", ", parts)})";
return $"SELECT COUNT(*) AS cnt, CAST(IFNULL(SUM({RowHash(rowStr)}),0) AS DECIMAL(40,0)) AS chk " +
$"FROM {Q(def.DimTable)} {DimAlias} WHERE {DimAlias}.{Q(S0DimDefinition.TenantColumn)} = @TenantId";
}
/// dim 业务键重复计数(唯一键之外的兜底断言)。参数:@TenantId
public static string BuildDimDuplicateBizKeySql(S0DimDefinition def) =>
$"""
SELECT COUNT(*) FROM (
SELECT {DimBizKeyExpr(def)} AS bk
FROM {Q(def.DimTable)} {DimAlias}
WHERE {DimAlias}.{Q(S0DimDefinition.TenantColumn)} = @TenantId
GROUP BY bk HAVING COUNT(*) > 1
) z
""";
/// 镜像唯一性断言(如 dim_location 的 (tenant_id, location_code))。参数:@TenantId
public static string BuildMirrorUniqueViolationSql(S0DimDefinition def)
{
if (def.MirrorUniqueColumns is not { Count: > 0 }) throw new InvalidOperationException($"[{def.Key}] 未声明 MirrorUniqueColumns");
var cols = string.Join(", ", def.MirrorUniqueColumns.Select(c => $"{DimAlias}.{Q(c)}"));
return $"""
SELECT COUNT(*) FROM (
SELECT {cols}
FROM {Q(def.DimTable)} {DimAlias}
WHERE {DimAlias}.{Q(S0DimDefinition.TenantColumn)} = @TenantId
GROUP BY {cols} HAVING COUNT(*) > 1
) z
""";
}
/// 孤儿子行(父维度中找不到父行)。参数:@TenantId
///
/// 孤儿判定的公共 FROM/WHERE —— 样本查询与计数查询**共用同一份**,避免两处口径漂移。
///
/// 「未建立关系」不是孤儿:父键列为 NULL 表示该行压根没声明父(如员工未分配部门),
/// 与「声明了父但父不存在」(真悬挂)是两种语义。<=> 是 NULL 安全等值,
/// 而父维度的业务键列均为 NOT NULL,故不加 IS NOT NULL 过滤会把全部未分配行
/// 误报成孤儿,淹没真正的悬挂引用。对父键全部 Required 的维度(如 LocationShelf)本条件恒真,行为不变。
///
private static string OrphanFromWhere(S0DimDefinition def)
{
var on = string.Join(" AND ", def.ParentKeyColumns!.Select(c => $"p.{Q(c)} <=> {DimAlias}.{Q(c)}"));
var declared = string.Join(" AND ", def.ParentKeyColumns!.Select(c => $"{DimAlias}.{Q(c)} IS NOT NULL"));
return $"""
FROM {Q(def.DimTable)} {DimAlias}
LEFT JOIN {Q(def.ParentDimTable!)} p
ON p.{Q(S0DimDefinition.TenantColumn)} = {DimAlias}.{Q(S0DimDefinition.TenantColumn)} AND {on}
WHERE {DimAlias}.{Q(S0DimDefinition.TenantColumn)} = @TenantId AND {declared} AND p.`id` IS NULL
""";
}
/// 孤儿子行**样本**(受 limit 截断)。参数:@TenantId
public static string BuildOrphanChildSql(S0DimDefinition def, int limit)
{
AssertParentDeclared(def);
var sel = string.Join(", ", def.BusinessKeyColumns.Skip(1).Select(c => $"{DimAlias}.{Q(c)}"));
return $"SELECT {sel} {OrphanFromWhere(def)} LIMIT {limit}";
}
/// 孤儿子行**真实总数**(不截断)。参数:@TenantId
public static string BuildOrphanChildCountSql(S0DimDefinition def)
{
AssertParentDeclared(def);
return $"SELECT COUNT(*) {OrphanFromWhere(def)}";
}
private static void AssertParentDeclared(S0DimDefinition def)
{
if (def.ParentDimTable is null || def.ParentKeyColumns is not { Count: > 0 })
throw new InvalidOperationException($"[{def.Key}] 未声明父维度");
}
}