S0DimResults.cs 9.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244
  1. using System.Text.Json.Serialization;
  2. namespace Admin.NET.Plugin.AiDOP.DataPlatform.S0Dim;
  3. /// <summary>
  4. /// source / dim 两侧属性校验和的一次读数(<c>count</c> 与 <c>checksum</c> **成对**,缺一不可)。
  5. ///
  6. /// <para>由 <see cref="S0DimReconciler.ReadChecksumPairAsync"/> 唯一产出,
  7. /// 供两处共用:① <c>ReconcileAsync</c> 的 Layer 3 写后对账;
  8. /// ② <see cref="S0DimMaterializer"/> 的写前闸门。两者共用同一读取路径,
  9. /// 保证「写前判等」与「写后对账」不会各跑一套口径。</para>
  10. /// </summary>
  11. public sealed class S0DimChecksumPair
  12. {
  13. /// <summary>源侧参与校验和的行数(按租户)。</summary>
  14. public int SourceCount { get; set; }
  15. /// <summary>dim 侧参与校验和的行数(按租户)。</summary>
  16. public int DimCount { get; set; }
  17. /// <summary>
  18. /// 源侧校验和原值。<c>DECIMAL(40,0)</c> 的精确整数,**不外化为 JSON 数字** ——
  19. /// 20+ 位整数经 JSON/MySQL JSON 取出时会退化成 DOUBLE 丢精度,读者会看到一个错的数字。
  20. /// 对外一律走 <see cref="SourceChecksum"/> 字符串。
  21. /// </summary>
  22. [JsonIgnore]
  23. public decimal SourceChecksumValue { get; set; }
  24. /// <inheritdoc cref="SourceChecksumValue"/>
  25. [JsonIgnore]
  26. public decimal DimChecksumValue { get; set; }
  27. /// <summary>源侧校验和(十进制字符串,与 <see cref="S0DimReconcileResult.SourceChecksum"/> 同格式)。</summary>
  28. public string SourceChecksum => SourceChecksumValue.ToString("F0");
  29. /// <summary>dim 侧校验和(十进制字符串,与 <see cref="S0DimReconcileResult.DimChecksum"/> 同格式)。</summary>
  30. public string DimChecksum => DimChecksumValue.ToString("F0");
  31. /// <summary>
  32. /// 两侧完全一致。<b>count 与 checksum 必须同时相等</b> —— 判据与
  33. /// <c>S0DimReconciler</c> Layer 3 的失败条件逐字互补,不是「只比 checksum」。
  34. /// </summary>
  35. public bool IsMatch => IsMatchOf(SourceCount, SourceChecksumValue, DimCount, DimChecksumValue);
  36. /// <summary>
  37. /// 校验和判等的**唯一**判据(纯函数,可单测)。
  38. ///
  39. /// <para>🔴 <b>count 不可省略</b>:<c>SUM(RowHash)</c> 是可交换的加法,
  40. /// 理论上不同行集可能求和相同;<c>S0DimSqlBuilder.BuildSourceChecksumSql</c> 的注释
  41. /// 明确写了「必须与 COUNT 成对使用」。只比 checksum 会让闸门在极端情况下误判「无变化」
  42. /// 而跳过一次本该执行的刷新。</para>
  43. /// </summary>
  44. public static bool IsMatchOf(int sourceCount, decimal sourceChecksum, int dimCount, decimal dimChecksum)
  45. => sourceCount == dimCount && sourceChecksum == dimChecksum;
  46. }
  47. /// <summary>业务键集合三层等价的差集计数。全部为 0 才算通过。</summary>
  48. public sealed class S0DimBizKeyDiff
  49. {
  50. /// <summary>source 有而 stg 没有。</summary>
  51. public int SourceMinusStaging { get; set; }
  52. /// <summary>stg 有而 source 没有。</summary>
  53. public int StagingMinusSource { get; set; }
  54. /// <summary>stg 有而 dim 没有。</summary>
  55. public int StagingMinusDim { get; set; }
  56. /// <summary>dim 有而 stg 没有。</summary>
  57. public int DimMinusStaging { get; set; }
  58. /// <summary>四个方向是否全为 0。</summary>
  59. public bool IsEqual => SourceMinusStaging == 0 && StagingMinusSource == 0
  60. && StagingMinusDim == 0 && DimMinusStaging == 0;
  61. }
  62. /// <summary>某一层业务键的质量(重复 / 空值)。</summary>
  63. public sealed class S0DimBizKeyQuality
  64. {
  65. /// <summary>层名:source / staging / dim。</summary>
  66. public string Layer { get; set; } = "";
  67. /// <summary>总行数。</summary>
  68. public int Total { get; set; }
  69. /// <summary>去重后行数。</summary>
  70. public int Distinct { get; set; }
  71. /// <summary>空业务键行数(stg 侧出现即意味着 biz_key_expr 回落到了 source_row_id)。</summary>
  72. public int Blank { get; set; }
  73. /// <summary>无重复且无空值。</summary>
  74. public bool IsClean => Total == Distinct && Blank == 0;
  75. }
  76. /// <summary>单个维度的对账结果。</summary>
  77. public sealed class S0DimReconcileResult
  78. {
  79. /// <summary>维度 Key。</summary>
  80. public string Key { get; set; } = "";
  81. /// <summary>租户。</summary>
  82. public long TenantId { get; set; }
  83. /// <summary>本轮批次(只读对账时可能为 null)。</summary>
  84. public string? BatchId { get; set; }
  85. /// <summary>源表行数(按租户)。</summary>
  86. public int SourceCount { get; set; }
  87. /// <summary>staging 行数(四段谓词)。</summary>
  88. public int StagingCount { get; set; }
  89. /// <summary>dim 行数(按租户)。</summary>
  90. public int DimCount { get; set; }
  91. /// <summary>源表中无法归属租户的行(整条链路不可见,WARN)。</summary>
  92. public int SourceUntenanted { get; set; }
  93. /// <summary>dim 中不属于本批的残留行(FULL Replace 后必须为 0)。</summary>
  94. public int DimBatchImpurity { get; set; }
  95. /// <summary>业务键集合差集。</summary>
  96. public S0DimBizKeyDiff BizKeyDiff { get; set; } = new();
  97. /// <summary>三层业务键质量。</summary>
  98. public List<S0DimBizKeyQuality> BizKeyQuality { get; set; } = [];
  99. /// <summary>源侧属性校验和。</summary>
  100. public string SourceChecksum { get; set; } = "0";
  101. /// <summary>dim 侧属性校验和。</summary>
  102. public string DimChecksum { get; set; } = "0";
  103. /// <summary>dim 业务键重复组数。</summary>
  104. public int DimDuplicateBizKeys { get; set; }
  105. /// <summary>镜像唯一性违例组数(未声明时为 -1 表示不适用)。</summary>
  106. public int MirrorUniqueViolations { get; set; } = -1;
  107. /// <summary>
  108. /// 孤儿子行**真实总数**(WARN,不阻断;未声明父维度时为 0)。
  109. /// 与 <see cref="OrphanChildren"/> 不同:后者受 <c>OrphanSampleLimit</c> 截断,只是样本。
  110. /// </summary>
  111. public int OrphanChildCount { get; set; }
  112. /// <summary>孤儿子行业务键**样本**(最多 <c>OrphanSampleLimit</c> 条)。总数看 <see cref="OrphanChildCount"/>。</summary>
  113. public List<string> OrphanChildren { get; set; } = [];
  114. /// <summary>阻断级失败原因(为空表示通过)。</summary>
  115. public List<string> Failures { get; set; } = [];
  116. /// <summary>告警(不阻断)。</summary>
  117. public List<string> Warnings { get; set; } = [];
  118. /// <summary>无阻断级失败。</summary>
  119. public bool IsPass => Failures.Count == 0;
  120. }
  121. /// <summary>单个维度的物化结果。</summary>
  122. public sealed class S0DimMaterializeResult
  123. {
  124. /// <summary>本轮实际执行过刷新且写后对账通过。</summary>
  125. public const string StatusSuccess = "SUCCESS";
  126. /// <summary>本轮失败,dim 停在上一轮的完整快照。</summary>
  127. public const string StatusFailed = "FAILED";
  128. /// <summary>
  129. /// 本轮**验证过** source 与 dim 完全一致,因此**没有执行**刷新。
  130. ///
  131. /// <para>属「非失败」,但**不是** <see cref="StatusSuccess"/>:
  132. /// SUCCESS 表示「跑过并验证成功」,本状态表示「验证后确认无需跑」。
  133. /// 两者混同会让「自动刷新是否真的在工作」不可观测。</para>
  134. /// </summary>
  135. public const string StatusSkippedNoChange = "SKIPPED_NO_CHANGE";
  136. /// <summary>跳过原因:源与 dim 的 (count, checksum) 完全一致。</summary>
  137. public const string SkipReasonNoChange = "NO_CHANGE";
  138. /// <summary>维度 Key。</summary>
  139. public string Key { get; set; } = "";
  140. /// <summary>租户。</summary>
  141. public long TenantId { get; set; }
  142. /// <summary>批次。</summary>
  143. public string BatchId { get; set; } = "";
  144. /// <summary>purge 掉的 staging 行数。</summary>
  145. public int StagingPurged { get; set; }
  146. /// <summary>本批拉取到的源行数。</summary>
  147. public int SourcePulled { get; set; }
  148. /// <summary>本批写入 staging 的行数。</summary>
  149. public int StagingWritten { get; set; }
  150. /// <summary>写入 dim 的行数。</summary>
  151. public int DimRows { get; set; }
  152. /// <summary>对账结果。跳过时为 null —— 本轮没有写入,也就没有写后对账。</summary>
  153. public S0DimReconcileResult? Reconcile { get; set; }
  154. /// <summary>
  155. /// 写前闸门读到的 source / dim 校验和读数。闸门执行过即非 null(无论是否跳过),
  156. /// 外化进 <c>summary_json</c> 供事后复核「当时凭什么判定无变化」。
  157. /// </summary>
  158. public S0DimChecksumPair? Checksum { get; set; }
  159. /// <summary>跳过原因(仅 <see cref="StatusSkippedNoChange"/> 时非 null)。</summary>
  160. public string? SkipReason { get; set; }
  161. /// <summary><see cref="StatusSuccess"/> / <see cref="StatusSkippedNoChange"/> / <see cref="StatusFailed"/>。</summary>
  162. public string Status { get; set; } = StatusSuccess;
  163. /// <summary>失败信息。</summary>
  164. public string? Error { get; set; }
  165. }
  166. /// <summary>一次刷新(可含多个维度)的整体结果。</summary>
  167. public sealed class S0DimRefreshResult
  168. {
  169. /// <summary>批次(同时是 stg/dim 的 sync_batch_id 与 run log 的 batch_id)。</summary>
  170. public string BatchId { get; set; } = "";
  171. /// <summary>租户。</summary>
  172. public long TenantId { get; set; }
  173. /// <summary>
  174. /// SUCCESS / SUCCESS_WITH_WARNING / SKIPPED_NO_CHANGE / FAILED / CANCELED。
  175. /// <c>SKIPPED_NO_CHANGE</c> 仅在**全部**维度都被写前闸门跳过时出现。
  176. /// </summary>
  177. public string Status { get; set; } = "SUCCESS";
  178. /// <summary>逐维度结果。</summary>
  179. public List<S0DimMaterializeResult> Items { get; set; } = [];
  180. /// <summary>耗时(毫秒)。</summary>
  181. public int DurationMs { get; set; }
  182. /// <summary>run log 主键。</summary>
  183. public long RunLogId { get; set; }
  184. }