S8AuthorityHealthGateTests.cs 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358
  1. using System.Reflection;
  2. using Admin.NET.Plugin.AiDOP.Service.S8.Rules.Health;
  3. using Xunit;
  4. namespace Admin.NET.Plugin.AiDOP.Tests.S8;
  5. /// <summary>
  6. /// S8-AUTHORITY-HEALTH-GATE:恢复放行门禁的判定逻辑。
  7. ///
  8. /// <para><b>本文件同时守两种相反的失效,两边都会静默</b>:</para>
  9. /// <list type="number">
  10. /// <item><b>门禁太松</b> —— 中台断供时取数变空,恢复判定会把全部在办异常判成「已恢复」。
  11. /// 没有报错、没有通知,异常就消失了。</item>
  12. /// <item><b>门禁太紧</b> —— 若有人把门禁写成「取数为空就拦」,真实恢复将永不发生,
  13. /// 异常在列表里堆到天荒地老,而调度器每轮都报成功。
  14. /// <see cref="B_HealthyEmpty_StillAllowsRecovery"/> 就是为这一侧准备的反向守卫。</item>
  15. /// </list>
  16. ///
  17. /// <para>判定是纯函数(<see cref="S8AuthorityHealthEvaluator.Evaluate"/>),
  18. /// 不接数据库、不接 DI,因此 A–H 全部可在普通单测里跑。
  19. /// 「怎么把运行日志查出来」是 <c>S8MdpAuthorityHealthResolver</c> 的事,不在此处断言。</para>
  20. /// </summary>
  21. public class S8AuthorityHealthGateTests
  22. {
  23. private const long TenantA = 838257186181189L;
  24. private const string Dataset = "ORDER_DELIVERY_RISK";
  25. /// <summary>固定「现在」,让所有时间边界可逐分钟锁定。</summary>
  26. private static readonly DateTime Now = new(2026, 9, 8, 12, 0, 0);
  27. private static readonly TimeSpan Stale = TimeSpan.FromHours(1);
  28. private static S8AuthorityObservation Obs(
  29. DateTime? lastSuccessEndAt = null,
  30. bool specFound = true,
  31. bool producerTrusted = true,
  32. bool newerFailure = false,
  33. string? newerFailureStatus = null,
  34. DateTime? inFlightStartAt = null,
  35. string snapshot = S8SnapshotInvariant.NotApplicable,
  36. string? snapshotDetail = null) => new()
  37. {
  38. DatasetCode = Dataset,
  39. TenantId = TenantA,
  40. ObservedAt = Now,
  41. SpecFound = specFound,
  42. ProducerTrusted = producerTrusted,
  43. LastSuccessEndAt = lastSuccessEndAt,
  44. HasNewerTerminalFailure = newerFailure,
  45. NewerTerminalFailureStatus = newerFailureStatus,
  46. OldestNewerInFlightStartAt = inFlightStartAt,
  47. SnapshotInvariant = snapshot,
  48. SnapshotDetail = snapshotDetail,
  49. StaleWindow = Stale
  50. };
  51. /// <summary>一次完全健康的观测:30 分钟前成功,早于 1 小时的新鲜度窗口。</summary>
  52. private static S8AuthorityObservation Healthy() => Obs(lastSuccessEndAt: Now.AddMinutes(-30));
  53. // ══════════════════ CASE A / B:放行侧 ══════════════════
  54. /// <summary>CASE A:Authority 可信 → 恢复正常放行。</summary>
  55. [Fact]
  56. public void A_Healthy_AllowsRecovery()
  57. {
  58. var r = S8AuthorityHealthEvaluator.Evaluate(Healthy());
  59. Assert.Equal(S8AuthorityHealthState.Healthy, r.State);
  60. Assert.Equal(S8AuthorityHealthReason.Healthy, r.ReasonCode);
  61. Assert.Equal(Now.AddMinutes(-30), r.LastSuccessAt);
  62. Assert.Equal(Now.AddMinutes(-30) + Stale, r.StaleAfter);
  63. Assert.True(r.AllowsRecovery);
  64. }
  65. /// <summary>
  66. /// CASE B(<b>本文件最重要的一条</b>):Authority 可信但本轮取数为空 → <b>恢复仍然放行</b>。
  67. ///
  68. /// <para>「空集」是恢复的<b>正常输入</b>而非异常信号 —— 全部业务对象都合法退场时,
  69. /// 本来就该看到 0 行。把门禁写成「空集就拦」,恢复将永不发生且没有任何报错。</para>
  70. ///
  71. /// <para>断言分两层:行为上放行;<b>结构上根本拿不到行数</b> ——
  72. /// 判定的入参类型里不存在任何与行数/命中数相关的字段,
  73. /// 于是「顺手加个空集判断」在编译期就做不到。这一层比行为断言更耐改。</para>
  74. /// </summary>
  75. [Fact]
  76. public void B_HealthyEmpty_StillAllowsRecovery()
  77. {
  78. // 行为层:与 CASE A 走的是同一组判据 —— 判定里根本没有「空/非空」这个分支。
  79. Assert.True(S8AuthorityHealthEvaluator.Evaluate(Healthy()).AllowsRecovery);
  80. // 结构层一:判定入参里不得出现任何行数概念。
  81. var banned = new[] { "hit", "row", "count", "empty", "any", "size", "length" };
  82. foreach (var p in typeof(S8AuthorityObservation).GetProperties())
  83. {
  84. foreach (var b in banned)
  85. {
  86. Assert.False(p.Name.Contains(b, StringComparison.OrdinalIgnoreCase),
  87. $"S8AuthorityObservation.{p.Name} 疑似把行数引入了健康判定");
  88. }
  89. }
  90. // 结构层二:放行判定是结果对象上的无参属性,没有任何可供传入行数的入口。
  91. var allows = typeof(S8AuthorityHealthResult).GetProperty(nameof(S8AuthorityHealthResult.AllowsRecovery));
  92. Assert.NotNull(allows);
  93. Assert.Empty(allows!.GetMethod!.GetParameters());
  94. // 结构层三:Evaluate 只有一个入参,就是观测值本身。
  95. var evaluate = typeof(S8AuthorityHealthEvaluator).GetMethod(
  96. nameof(S8AuthorityHealthEvaluator.Evaluate), BindingFlags.Public | BindingFlags.Static);
  97. Assert.NotNull(evaluate);
  98. Assert.Single(evaluate!.GetParameters());
  99. Assert.Equal(typeof(S8AuthorityObservation), evaluate.GetParameters()[0].ParameterType);
  100. }
  101. // ══════════════════ CASE C–H:拦截侧 ══════════════════
  102. /// <summary>CASE C:回溯窗口内一次成功都没有 → UNKNOWN。「查不到」不等于「没问题」。</summary>
  103. [Fact]
  104. public void C_NoSuccessfulRun_IsUnknownAndBlocks()
  105. {
  106. var r = S8AuthorityHealthEvaluator.Evaluate(Obs(lastSuccessEndAt: null));
  107. Assert.Equal(S8AuthorityHealthState.Unknown, r.State);
  108. Assert.Equal(S8AuthorityHealthReason.NoSuccessfulRun, r.ReasonCode);
  109. Assert.Null(r.LastSuccessAt);
  110. Assert.Null(r.StaleAfter);
  111. Assert.False(r.AllowsRecovery);
  112. }
  113. /// <summary>CASE C':数据集压根没声明 AuthoritySpec → UNKNOWN,绝不默认健康。</summary>
  114. [Fact]
  115. public void C2_NoAuthoritySpec_IsUnknownAndBlocks()
  116. {
  117. var r = S8AuthorityHealthEvaluator.Evaluate(Obs(specFound: false, lastSuccessEndAt: Now.AddMinutes(-1)));
  118. Assert.Equal(S8AuthorityHealthState.Unknown, r.State);
  119. Assert.Equal(S8AuthorityHealthReason.NoAuthoritySpec, r.ReasonCode);
  120. Assert.False(r.AllowsRecovery);
  121. }
  122. /// <summary>CASE D:最近一次成功已超过新鲜度窗口 → DEGRADED / STALE。</summary>
  123. [Fact]
  124. public void D_SuccessTooOld_IsStaleAndBlocks()
  125. {
  126. var r = S8AuthorityHealthEvaluator.Evaluate(Obs(lastSuccessEndAt: Now.AddMinutes(-90)));
  127. Assert.Equal(S8AuthorityHealthState.Degraded, r.State);
  128. Assert.Equal(S8AuthorityHealthReason.Stale, r.ReasonCode);
  129. Assert.False(r.AllowsRecovery);
  130. }
  131. /// <summary>CASE D':恰好落在窗口边界上仍算健康 —— 阈值是「超过」才降级。</summary>
  132. [Fact]
  133. public void D2_ExactlyAtStaleBoundary_IsStillHealthy()
  134. {
  135. var r = S8AuthorityHealthEvaluator.Evaluate(Obs(lastSuccessEndAt: Now - Stale));
  136. Assert.Equal(S8AuthorityHealthState.Healthy, r.State);
  137. Assert.True(r.AllowsRecovery);
  138. }
  139. /// <summary>
  140. /// CASE E:成功之后又开了一轮、还没跑完 → IN_FLIGHT。
  141. /// 此刻 Authority 可能只写了一半,恢复判定不能在这个瞬间做。
  142. /// </summary>
  143. [Fact]
  144. public void E_NewerRunInFlight_Blocks()
  145. {
  146. var r = S8AuthorityHealthEvaluator.Evaluate(
  147. Obs(lastSuccessEndAt: Now.AddMinutes(-40), inFlightStartAt: Now.AddMinutes(-5)));
  148. Assert.Equal(S8AuthorityHealthState.Degraded, r.State);
  149. Assert.Equal(S8AuthorityHealthReason.InFlight, r.ReasonCode);
  150. Assert.False(r.AllowsRecovery);
  151. }
  152. /// <summary>
  153. /// CASE E':在途运行的年龄已超过新鲜度窗口 → STUCK_RUN,与 IN_FLIGHT 分开报。
  154. ///
  155. /// <para>运行日志里的 RUNNING 行没有任何应用侧回收机制(失败路径会吞掉自己的写入异常,
  156. /// 注释里写的是「遗留 RUNNING 行可由运维手动清理」,而并不存在这样的清理作业)。
  157. /// 实测该表有 298 条永不终态的 RUNNING 行,S1 占 42 条、最老已 570 小时。
  158. /// 若与「正在正常跑」混为一谈,运维分不清该等还是该查 —— 两者都拦,但排障动作不同。</para>
  159. /// </summary>
  160. [Fact]
  161. public void E2_StuckRun_IsReportedSeparately()
  162. {
  163. var r = S8AuthorityHealthEvaluator.Evaluate(
  164. Obs(lastSuccessEndAt: Now.AddHours(-5), inFlightStartAt: Now.AddHours(-4)));
  165. Assert.Equal(S8AuthorityHealthState.Degraded, r.State);
  166. Assert.Equal(S8AuthorityHealthReason.StuckRun, r.ReasonCode);
  167. Assert.False(r.AllowsRecovery);
  168. }
  169. /// <summary>CASE F:成功之后的那一轮失败了 → LATEST_FAILED。</summary>
  170. [Fact]
  171. public void F_NewerRunFailed_Blocks()
  172. {
  173. var r = S8AuthorityHealthEvaluator.Evaluate(
  174. Obs(lastSuccessEndAt: Now.AddMinutes(-40), newerFailure: true, newerFailureStatus: "FAILED"));
  175. Assert.Equal(S8AuthorityHealthState.Degraded, r.State);
  176. Assert.Equal(S8AuthorityHealthReason.LatestFailed, r.ReasonCode);
  177. Assert.False(r.AllowsRecovery);
  178. }
  179. /// <summary>
  180. /// CASE F':失败发生在最近一次成功<b>之前</b> → 不影响判定。
  181. /// 历史失败是常态;若一律降级,门禁会永久拦死。
  182. /// (观测层已按「锚点之后」过滤,此处等价于 newerFailure = false。)
  183. /// </summary>
  184. [Fact]
  185. public void F2_OlderFailure_DoesNotDegrade()
  186. {
  187. var r = S8AuthorityHealthEvaluator.Evaluate(
  188. Obs(lastSuccessEndAt: Now.AddMinutes(-20), newerFailure: false));
  189. Assert.Equal(S8AuthorityHealthState.Healthy, r.State);
  190. Assert.True(r.AllowsRecovery);
  191. }
  192. /// <summary>CASE G:成功之后出现终态 PARTIAL → LATEST_PARTIAL。部分成功的快照不足以支撑恢复判定。</summary>
  193. [Fact]
  194. public void G_PartialRun_Blocks()
  195. {
  196. var r = S8AuthorityHealthEvaluator.Evaluate(
  197. Obs(lastSuccessEndAt: Now.AddMinutes(-40), newerFailure: true, newerFailureStatus: "PARTIAL"));
  198. Assert.Equal(S8AuthorityHealthState.Degraded, r.State);
  199. Assert.Equal(S8AuthorityHealthReason.LatestPartial, r.ReasonCode);
  200. Assert.False(r.AllowsRecovery);
  201. }
  202. /// <summary>
  203. /// CASE G':未知的终态状态一律按失败处置,不得被当成健康放行。
  204. /// <para>该表的 status 值域实测还包含 <c>SUCCESS_WITH_WARNING</c> 与 <c>CANCELED</c>
  205. /// (由 S0 维度刷新写出,当前不出现在 S1 上)。采集层用的是
  206. /// <c>status &lt;&gt; 'SUCCESS'</c> 而非白名单,正是为了让将来新增的终态状态
  207. /// 默认落到「失败」而不是「健康」这一侧。</para>
  208. /// </summary>
  209. [Fact]
  210. public void G2_UnknownTerminalStatus_IsTreatedAsFailure()
  211. {
  212. var r = S8AuthorityHealthEvaluator.Evaluate(
  213. Obs(lastSuccessEndAt: Now.AddMinutes(-40), newerFailure: true, newerFailureStatus: "SUCCESS_WITH_WARNING"));
  214. Assert.Equal(S8AuthorityHealthState.Degraded, r.State);
  215. Assert.Equal(S8AuthorityHealthReason.LatestFailed, r.ReasonCode);
  216. Assert.False(r.AllowsRecovery);
  217. }
  218. /// <summary>CASE G'':快照不变量被破坏 → 即使生产运行成功也降级。</summary>
  219. [Fact]
  220. public void G3_SnapshotInvariantViolated_Blocks()
  221. {
  222. var r = S8AuthorityHealthEvaluator.Evaluate(Obs(
  223. lastSuccessEndAt: Now.AddMinutes(-10),
  224. snapshot: S8SnapshotInvariant.Violated,
  225. snapshotDetail: "同时存在 2 个当前批次"));
  226. Assert.Equal(S8AuthorityHealthState.Degraded, r.State);
  227. Assert.Equal(S8AuthorityHealthReason.SnapshotInvariantFailed, r.ReasonCode);
  228. Assert.Contains("2 个当前批次", r.Reason);
  229. Assert.False(r.AllowsRecovery);
  230. }
  231. /// <summary>
  232. /// CASE G''':快照发布无法确认 → UNKNOWN,<b>不猜</b>。
  233. ///
  234. /// <para>这是 Rule03 今天的真实处境:生产侧的发布语句在本轮零行时命中 0 行、
  235. /// 在库里不留任何批次标记,于是「源侧本轮确实没有数据」与「发布压根没跑成」字面等同。
  236. /// 猜成健康的代价是把真实缺料预警整体判成已恢复,因此保守拦截。
  237. /// 消除它需要生产侧补一个发布标记 —— 那是下一批的工作,不在本轮范围。</para>
  238. /// </summary>
  239. [Fact]
  240. public void G4_SnapshotPublicationUnconfirmed_IsUnknownAndBlocks()
  241. {
  242. var r = S8AuthorityHealthEvaluator.Evaluate(Obs(
  243. lastSuccessEndAt: Now.AddMinutes(-10),
  244. snapshot: S8SnapshotInvariant.Unconfirmed,
  245. snapshotDetail: "当前批次数为 0"));
  246. Assert.Equal(S8AuthorityHealthState.Unknown, r.State);
  247. Assert.Equal(S8AuthorityHealthReason.SnapshotPublicationUnconfirmed, r.ReasonCode);
  248. Assert.False(r.AllowsRecovery);
  249. }
  250. /// <summary>
  251. /// CASE H:Authority 有多个写入方且部分不留运行日志 → UNKNOWN,
  252. /// 且必须<b>压过一切健康信号</b>。
  253. ///
  254. /// <para>本用例刻意喂进一次「一分钟前刚成功、快照不变量也成立」的完美观测,
  255. /// 它不得把结论洗白 —— 因为问题不在于生产跑没跑成,
  256. /// 而在于运行日志根本证明不了表里最终的数据出自谁。</para>
  257. /// </summary>
  258. [Fact]
  259. public void H_MultiWriterAuthority_IsUnknownRegardlessOfRunHealth()
  260. {
  261. var r = S8AuthorityHealthEvaluator.Evaluate(Obs(
  262. producerTrusted: false,
  263. lastSuccessEndAt: Now.AddMinutes(-1),
  264. snapshot: S8SnapshotInvariant.Satisfied));
  265. Assert.Equal(S8AuthorityHealthState.Unknown, r.State);
  266. Assert.Equal(S8AuthorityHealthReason.Rule01MultiWriterUntrusted, r.ReasonCode);
  267. Assert.False(r.AllowsRecovery);
  268. }
  269. // ══════════════════ 穷尽性与词表封闭 ══════════════════
  270. public static IEnumerable<object[]> AllNonHealthyReasonCodes() =>
  271. typeof(S8AuthorityHealthReason)
  272. .GetFields(BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy)
  273. .Where(f => f.IsLiteral && !f.IsInitOnly && f.FieldType == typeof(string))
  274. .Select(f => (string)f.GetRawConstantValue()!)
  275. .Where(v => v != S8AuthorityHealthReason.Healthy)
  276. .Select(v => new object[] { v });
  277. /// <summary>
  278. /// 放行只有一个出口:<c>State == HEALTHY</c>。
  279. /// <para>用反射枚举<b>全部</b>原因码 —— 将来有人新增一个却忘了想清楚放不放行时,
  280. /// 这条会自动变红。默认必须是拦。</para>
  281. /// </summary>
  282. [Theory]
  283. [MemberData(nameof(AllNonHealthyReasonCodes))]
  284. public void EveryNonHealthyReason_Blocks(string reasonCode)
  285. {
  286. foreach (var state in new[] { S8AuthorityHealthState.Degraded, S8AuthorityHealthState.Unknown })
  287. {
  288. var r = new S8AuthorityHealthResult { State = state, ReasonCode = reasonCode };
  289. Assert.False(r.AllowsRecovery, $"state={state} reason={reasonCode} 不应放行");
  290. }
  291. }
  292. /// <summary>状态是封闭三元集合,不得出现第四种(尤其不得出现 HEALTHY_EMPTY)。</summary>
  293. [Fact]
  294. public void StateVocabularyIsClosed_AndHasNoHealthyEmpty()
  295. {
  296. var states = typeof(S8AuthorityHealthState)
  297. .GetFields(BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy)
  298. .Where(f => f.IsLiteral && f.FieldType == typeof(string))
  299. .Select(f => (string)f.GetRawConstantValue()!)
  300. .OrderBy(v => v, StringComparer.Ordinal)
  301. .ToArray();
  302. Assert.Equal(new[] { "DEGRADED", "HEALTHY", "UNKNOWN" }, states);
  303. }
  304. /// <summary>端口是接口且实现可被自动注册,调度器才能换实现而不改自身。</summary>
  305. [Fact]
  306. public void Resolver_IsASeamInterface()
  307. {
  308. Assert.True(typeof(IS8AuthorityHealthResolver).IsInterface);
  309. Assert.True(typeof(IS8AuthorityHealthResolver)
  310. .IsAssignableFrom(typeof(S8MdpAuthorityHealthResolver)));
  311. }
  312. }