|
|
@@ -0,0 +1,358 @@
|
|
|
+using System.Reflection;
|
|
|
+using Admin.NET.Plugin.AiDOP.Service.S8.Rules.Health;
|
|
|
+using Xunit;
|
|
|
+
|
|
|
+namespace Admin.NET.Plugin.AiDOP.Tests.S8;
|
|
|
+
|
|
|
+/// <summary>
|
|
|
+/// S8-AUTHORITY-HEALTH-GATE:恢复放行门禁的判定逻辑。
|
|
|
+///
|
|
|
+/// <para><b>本文件同时守两种相反的失效,两边都会静默</b>:</para>
|
|
|
+/// <list type="number">
|
|
|
+/// <item><b>门禁太松</b> —— 中台断供时取数变空,恢复判定会把全部在办异常判成「已恢复」。
|
|
|
+/// 没有报错、没有通知,异常就消失了。</item>
|
|
|
+/// <item><b>门禁太紧</b> —— 若有人把门禁写成「取数为空就拦」,真实恢复将永不发生,
|
|
|
+/// 异常在列表里堆到天荒地老,而调度器每轮都报成功。
|
|
|
+/// <see cref="B_HealthyEmpty_StillAllowsRecovery"/> 就是为这一侧准备的反向守卫。</item>
|
|
|
+/// </list>
|
|
|
+///
|
|
|
+/// <para>判定是纯函数(<see cref="S8AuthorityHealthEvaluator.Evaluate"/>),
|
|
|
+/// 不接数据库、不接 DI,因此 A–H 全部可在普通单测里跑。
|
|
|
+/// 「怎么把运行日志查出来」是 <c>S8MdpAuthorityHealthResolver</c> 的事,不在此处断言。</para>
|
|
|
+/// </summary>
|
|
|
+public class S8AuthorityHealthGateTests
|
|
|
+{
|
|
|
+ private const long TenantA = 838257186181189L;
|
|
|
+ private const string Dataset = "ORDER_DELIVERY_RISK";
|
|
|
+
|
|
|
+ /// <summary>固定「现在」,让所有时间边界可逐分钟锁定。</summary>
|
|
|
+ private static readonly DateTime Now = new(2026, 9, 8, 12, 0, 0);
|
|
|
+
|
|
|
+ private static readonly TimeSpan Stale = TimeSpan.FromHours(1);
|
|
|
+
|
|
|
+ private static S8AuthorityObservation Obs(
|
|
|
+ DateTime? lastSuccessEndAt = null,
|
|
|
+ bool specFound = true,
|
|
|
+ bool producerTrusted = true,
|
|
|
+ bool newerFailure = false,
|
|
|
+ string? newerFailureStatus = null,
|
|
|
+ DateTime? inFlightStartAt = null,
|
|
|
+ string snapshot = S8SnapshotInvariant.NotApplicable,
|
|
|
+ string? snapshotDetail = null) => new()
|
|
|
+ {
|
|
|
+ DatasetCode = Dataset,
|
|
|
+ TenantId = TenantA,
|
|
|
+ ObservedAt = Now,
|
|
|
+ SpecFound = specFound,
|
|
|
+ ProducerTrusted = producerTrusted,
|
|
|
+ LastSuccessEndAt = lastSuccessEndAt,
|
|
|
+ HasNewerTerminalFailure = newerFailure,
|
|
|
+ NewerTerminalFailureStatus = newerFailureStatus,
|
|
|
+ OldestNewerInFlightStartAt = inFlightStartAt,
|
|
|
+ SnapshotInvariant = snapshot,
|
|
|
+ SnapshotDetail = snapshotDetail,
|
|
|
+ StaleWindow = Stale
|
|
|
+ };
|
|
|
+
|
|
|
+ /// <summary>一次完全健康的观测:30 分钟前成功,早于 1 小时的新鲜度窗口。</summary>
|
|
|
+ private static S8AuthorityObservation Healthy() => Obs(lastSuccessEndAt: Now.AddMinutes(-30));
|
|
|
+
|
|
|
+ // ══════════════════ CASE A / B:放行侧 ══════════════════
|
|
|
+
|
|
|
+ /// <summary>CASE A:Authority 可信 → 恢复正常放行。</summary>
|
|
|
+ [Fact]
|
|
|
+ public void A_Healthy_AllowsRecovery()
|
|
|
+ {
|
|
|
+ var r = S8AuthorityHealthEvaluator.Evaluate(Healthy());
|
|
|
+
|
|
|
+ Assert.Equal(S8AuthorityHealthState.Healthy, r.State);
|
|
|
+ Assert.Equal(S8AuthorityHealthReason.Healthy, r.ReasonCode);
|
|
|
+ Assert.Equal(Now.AddMinutes(-30), r.LastSuccessAt);
|
|
|
+ Assert.Equal(Now.AddMinutes(-30) + Stale, r.StaleAfter);
|
|
|
+ Assert.True(r.AllowsRecovery);
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>
|
|
|
+ /// CASE B(<b>本文件最重要的一条</b>):Authority 可信但本轮取数为空 → <b>恢复仍然放行</b>。
|
|
|
+ ///
|
|
|
+ /// <para>「空集」是恢复的<b>正常输入</b>而非异常信号 —— 全部业务对象都合法退场时,
|
|
|
+ /// 本来就该看到 0 行。把门禁写成「空集就拦」,恢复将永不发生且没有任何报错。</para>
|
|
|
+ ///
|
|
|
+ /// <para>断言分两层:行为上放行;<b>结构上根本拿不到行数</b> ——
|
|
|
+ /// 判定的入参类型里不存在任何与行数/命中数相关的字段,
|
|
|
+ /// 于是「顺手加个空集判断」在编译期就做不到。这一层比行为断言更耐改。</para>
|
|
|
+ /// </summary>
|
|
|
+ [Fact]
|
|
|
+ public void B_HealthyEmpty_StillAllowsRecovery()
|
|
|
+ {
|
|
|
+ // 行为层:与 CASE A 走的是同一组判据 —— 判定里根本没有「空/非空」这个分支。
|
|
|
+ Assert.True(S8AuthorityHealthEvaluator.Evaluate(Healthy()).AllowsRecovery);
|
|
|
+
|
|
|
+ // 结构层一:判定入参里不得出现任何行数概念。
|
|
|
+ var banned = new[] { "hit", "row", "count", "empty", "any", "size", "length" };
|
|
|
+ foreach (var p in typeof(S8AuthorityObservation).GetProperties())
|
|
|
+ {
|
|
|
+ foreach (var b in banned)
|
|
|
+ {
|
|
|
+ Assert.False(p.Name.Contains(b, StringComparison.OrdinalIgnoreCase),
|
|
|
+ $"S8AuthorityObservation.{p.Name} 疑似把行数引入了健康判定");
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 结构层二:放行判定是结果对象上的无参属性,没有任何可供传入行数的入口。
|
|
|
+ var allows = typeof(S8AuthorityHealthResult).GetProperty(nameof(S8AuthorityHealthResult.AllowsRecovery));
|
|
|
+ Assert.NotNull(allows);
|
|
|
+ Assert.Empty(allows!.GetMethod!.GetParameters());
|
|
|
+
|
|
|
+ // 结构层三:Evaluate 只有一个入参,就是观测值本身。
|
|
|
+ var evaluate = typeof(S8AuthorityHealthEvaluator).GetMethod(
|
|
|
+ nameof(S8AuthorityHealthEvaluator.Evaluate), BindingFlags.Public | BindingFlags.Static);
|
|
|
+ Assert.NotNull(evaluate);
|
|
|
+ Assert.Single(evaluate!.GetParameters());
|
|
|
+ Assert.Equal(typeof(S8AuthorityObservation), evaluate.GetParameters()[0].ParameterType);
|
|
|
+ }
|
|
|
+
|
|
|
+ // ══════════════════ CASE C–H:拦截侧 ══════════════════
|
|
|
+
|
|
|
+ /// <summary>CASE C:回溯窗口内一次成功都没有 → UNKNOWN。「查不到」不等于「没问题」。</summary>
|
|
|
+ [Fact]
|
|
|
+ public void C_NoSuccessfulRun_IsUnknownAndBlocks()
|
|
|
+ {
|
|
|
+ var r = S8AuthorityHealthEvaluator.Evaluate(Obs(lastSuccessEndAt: null));
|
|
|
+
|
|
|
+ Assert.Equal(S8AuthorityHealthState.Unknown, r.State);
|
|
|
+ Assert.Equal(S8AuthorityHealthReason.NoSuccessfulRun, r.ReasonCode);
|
|
|
+ Assert.Null(r.LastSuccessAt);
|
|
|
+ Assert.Null(r.StaleAfter);
|
|
|
+ Assert.False(r.AllowsRecovery);
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>CASE C':数据集压根没声明 AuthoritySpec → UNKNOWN,绝不默认健康。</summary>
|
|
|
+ [Fact]
|
|
|
+ public void C2_NoAuthoritySpec_IsUnknownAndBlocks()
|
|
|
+ {
|
|
|
+ var r = S8AuthorityHealthEvaluator.Evaluate(Obs(specFound: false, lastSuccessEndAt: Now.AddMinutes(-1)));
|
|
|
+
|
|
|
+ Assert.Equal(S8AuthorityHealthState.Unknown, r.State);
|
|
|
+ Assert.Equal(S8AuthorityHealthReason.NoAuthoritySpec, r.ReasonCode);
|
|
|
+ Assert.False(r.AllowsRecovery);
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>CASE D:最近一次成功已超过新鲜度窗口 → DEGRADED / STALE。</summary>
|
|
|
+ [Fact]
|
|
|
+ public void D_SuccessTooOld_IsStaleAndBlocks()
|
|
|
+ {
|
|
|
+ var r = S8AuthorityHealthEvaluator.Evaluate(Obs(lastSuccessEndAt: Now.AddMinutes(-90)));
|
|
|
+
|
|
|
+ Assert.Equal(S8AuthorityHealthState.Degraded, r.State);
|
|
|
+ Assert.Equal(S8AuthorityHealthReason.Stale, r.ReasonCode);
|
|
|
+ Assert.False(r.AllowsRecovery);
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>CASE D':恰好落在窗口边界上仍算健康 —— 阈值是「超过」才降级。</summary>
|
|
|
+ [Fact]
|
|
|
+ public void D2_ExactlyAtStaleBoundary_IsStillHealthy()
|
|
|
+ {
|
|
|
+ var r = S8AuthorityHealthEvaluator.Evaluate(Obs(lastSuccessEndAt: Now - Stale));
|
|
|
+
|
|
|
+ Assert.Equal(S8AuthorityHealthState.Healthy, r.State);
|
|
|
+ Assert.True(r.AllowsRecovery);
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>
|
|
|
+ /// CASE E:成功之后又开了一轮、还没跑完 → IN_FLIGHT。
|
|
|
+ /// 此刻 Authority 可能只写了一半,恢复判定不能在这个瞬间做。
|
|
|
+ /// </summary>
|
|
|
+ [Fact]
|
|
|
+ public void E_NewerRunInFlight_Blocks()
|
|
|
+ {
|
|
|
+ var r = S8AuthorityHealthEvaluator.Evaluate(
|
|
|
+ Obs(lastSuccessEndAt: Now.AddMinutes(-40), inFlightStartAt: Now.AddMinutes(-5)));
|
|
|
+
|
|
|
+ Assert.Equal(S8AuthorityHealthState.Degraded, r.State);
|
|
|
+ Assert.Equal(S8AuthorityHealthReason.InFlight, r.ReasonCode);
|
|
|
+ Assert.False(r.AllowsRecovery);
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>
|
|
|
+ /// CASE E':在途运行的年龄已超过新鲜度窗口 → STUCK_RUN,与 IN_FLIGHT 分开报。
|
|
|
+ ///
|
|
|
+ /// <para>运行日志里的 RUNNING 行没有任何应用侧回收机制(失败路径会吞掉自己的写入异常,
|
|
|
+ /// 注释里写的是「遗留 RUNNING 行可由运维手动清理」,而并不存在这样的清理作业)。
|
|
|
+ /// 实测该表有 298 条永不终态的 RUNNING 行,S1 占 42 条、最老已 570 小时。
|
|
|
+ /// 若与「正在正常跑」混为一谈,运维分不清该等还是该查 —— 两者都拦,但排障动作不同。</para>
|
|
|
+ /// </summary>
|
|
|
+ [Fact]
|
|
|
+ public void E2_StuckRun_IsReportedSeparately()
|
|
|
+ {
|
|
|
+ var r = S8AuthorityHealthEvaluator.Evaluate(
|
|
|
+ Obs(lastSuccessEndAt: Now.AddHours(-5), inFlightStartAt: Now.AddHours(-4)));
|
|
|
+
|
|
|
+ Assert.Equal(S8AuthorityHealthState.Degraded, r.State);
|
|
|
+ Assert.Equal(S8AuthorityHealthReason.StuckRun, r.ReasonCode);
|
|
|
+ Assert.False(r.AllowsRecovery);
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>CASE F:成功之后的那一轮失败了 → LATEST_FAILED。</summary>
|
|
|
+ [Fact]
|
|
|
+ public void F_NewerRunFailed_Blocks()
|
|
|
+ {
|
|
|
+ var r = S8AuthorityHealthEvaluator.Evaluate(
|
|
|
+ Obs(lastSuccessEndAt: Now.AddMinutes(-40), newerFailure: true, newerFailureStatus: "FAILED"));
|
|
|
+
|
|
|
+ Assert.Equal(S8AuthorityHealthState.Degraded, r.State);
|
|
|
+ Assert.Equal(S8AuthorityHealthReason.LatestFailed, r.ReasonCode);
|
|
|
+ Assert.False(r.AllowsRecovery);
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>
|
|
|
+ /// CASE F':失败发生在最近一次成功<b>之前</b> → 不影响判定。
|
|
|
+ /// 历史失败是常态;若一律降级,门禁会永久拦死。
|
|
|
+ /// (观测层已按「锚点之后」过滤,此处等价于 newerFailure = false。)
|
|
|
+ /// </summary>
|
|
|
+ [Fact]
|
|
|
+ public void F2_OlderFailure_DoesNotDegrade()
|
|
|
+ {
|
|
|
+ var r = S8AuthorityHealthEvaluator.Evaluate(
|
|
|
+ Obs(lastSuccessEndAt: Now.AddMinutes(-20), newerFailure: false));
|
|
|
+
|
|
|
+ Assert.Equal(S8AuthorityHealthState.Healthy, r.State);
|
|
|
+ Assert.True(r.AllowsRecovery);
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>CASE G:成功之后出现终态 PARTIAL → LATEST_PARTIAL。部分成功的快照不足以支撑恢复判定。</summary>
|
|
|
+ [Fact]
|
|
|
+ public void G_PartialRun_Blocks()
|
|
|
+ {
|
|
|
+ var r = S8AuthorityHealthEvaluator.Evaluate(
|
|
|
+ Obs(lastSuccessEndAt: Now.AddMinutes(-40), newerFailure: true, newerFailureStatus: "PARTIAL"));
|
|
|
+
|
|
|
+ Assert.Equal(S8AuthorityHealthState.Degraded, r.State);
|
|
|
+ Assert.Equal(S8AuthorityHealthReason.LatestPartial, r.ReasonCode);
|
|
|
+ Assert.False(r.AllowsRecovery);
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>
|
|
|
+ /// CASE G':未知的终态状态一律按失败处置,不得被当成健康放行。
|
|
|
+ /// <para>该表的 status 值域实测还包含 <c>SUCCESS_WITH_WARNING</c> 与 <c>CANCELED</c>
|
|
|
+ /// (由 S0 维度刷新写出,当前不出现在 S1 上)。采集层用的是
|
|
|
+ /// <c>status <> 'SUCCESS'</c> 而非白名单,正是为了让将来新增的终态状态
|
|
|
+ /// 默认落到「失败」而不是「健康」这一侧。</para>
|
|
|
+ /// </summary>
|
|
|
+ [Fact]
|
|
|
+ public void G2_UnknownTerminalStatus_IsTreatedAsFailure()
|
|
|
+ {
|
|
|
+ var r = S8AuthorityHealthEvaluator.Evaluate(
|
|
|
+ Obs(lastSuccessEndAt: Now.AddMinutes(-40), newerFailure: true, newerFailureStatus: "SUCCESS_WITH_WARNING"));
|
|
|
+
|
|
|
+ Assert.Equal(S8AuthorityHealthState.Degraded, r.State);
|
|
|
+ Assert.Equal(S8AuthorityHealthReason.LatestFailed, r.ReasonCode);
|
|
|
+ Assert.False(r.AllowsRecovery);
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>CASE G'':快照不变量被破坏 → 即使生产运行成功也降级。</summary>
|
|
|
+ [Fact]
|
|
|
+ public void G3_SnapshotInvariantViolated_Blocks()
|
|
|
+ {
|
|
|
+ var r = S8AuthorityHealthEvaluator.Evaluate(Obs(
|
|
|
+ lastSuccessEndAt: Now.AddMinutes(-10),
|
|
|
+ snapshot: S8SnapshotInvariant.Violated,
|
|
|
+ snapshotDetail: "同时存在 2 个当前批次"));
|
|
|
+
|
|
|
+ Assert.Equal(S8AuthorityHealthState.Degraded, r.State);
|
|
|
+ Assert.Equal(S8AuthorityHealthReason.SnapshotInvariantFailed, r.ReasonCode);
|
|
|
+ Assert.Contains("2 个当前批次", r.Reason);
|
|
|
+ Assert.False(r.AllowsRecovery);
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>
|
|
|
+ /// CASE G''':快照发布无法确认 → UNKNOWN,<b>不猜</b>。
|
|
|
+ ///
|
|
|
+ /// <para>这是 Rule03 今天的真实处境:生产侧的发布语句在本轮零行时命中 0 行、
|
|
|
+ /// 在库里不留任何批次标记,于是「源侧本轮确实没有数据」与「发布压根没跑成」字面等同。
|
|
|
+ /// 猜成健康的代价是把真实缺料预警整体判成已恢复,因此保守拦截。
|
|
|
+ /// 消除它需要生产侧补一个发布标记 —— 那是下一批的工作,不在本轮范围。</para>
|
|
|
+ /// </summary>
|
|
|
+ [Fact]
|
|
|
+ public void G4_SnapshotPublicationUnconfirmed_IsUnknownAndBlocks()
|
|
|
+ {
|
|
|
+ var r = S8AuthorityHealthEvaluator.Evaluate(Obs(
|
|
|
+ lastSuccessEndAt: Now.AddMinutes(-10),
|
|
|
+ snapshot: S8SnapshotInvariant.Unconfirmed,
|
|
|
+ snapshotDetail: "当前批次数为 0"));
|
|
|
+
|
|
|
+ Assert.Equal(S8AuthorityHealthState.Unknown, r.State);
|
|
|
+ Assert.Equal(S8AuthorityHealthReason.SnapshotPublicationUnconfirmed, r.ReasonCode);
|
|
|
+ Assert.False(r.AllowsRecovery);
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>
|
|
|
+ /// CASE H:Authority 有多个写入方且部分不留运行日志 → UNKNOWN,
|
|
|
+ /// 且必须<b>压过一切健康信号</b>。
|
|
|
+ ///
|
|
|
+ /// <para>本用例刻意喂进一次「一分钟前刚成功、快照不变量也成立」的完美观测,
|
|
|
+ /// 它不得把结论洗白 —— 因为问题不在于生产跑没跑成,
|
|
|
+ /// 而在于运行日志根本证明不了表里最终的数据出自谁。</para>
|
|
|
+ /// </summary>
|
|
|
+ [Fact]
|
|
|
+ public void H_MultiWriterAuthority_IsUnknownRegardlessOfRunHealth()
|
|
|
+ {
|
|
|
+ var r = S8AuthorityHealthEvaluator.Evaluate(Obs(
|
|
|
+ producerTrusted: false,
|
|
|
+ lastSuccessEndAt: Now.AddMinutes(-1),
|
|
|
+ snapshot: S8SnapshotInvariant.Satisfied));
|
|
|
+
|
|
|
+ Assert.Equal(S8AuthorityHealthState.Unknown, r.State);
|
|
|
+ Assert.Equal(S8AuthorityHealthReason.Rule01MultiWriterUntrusted, r.ReasonCode);
|
|
|
+ Assert.False(r.AllowsRecovery);
|
|
|
+ }
|
|
|
+
|
|
|
+ // ══════════════════ 穷尽性与词表封闭 ══════════════════
|
|
|
+
|
|
|
+ public static IEnumerable<object[]> AllNonHealthyReasonCodes() =>
|
|
|
+ typeof(S8AuthorityHealthReason)
|
|
|
+ .GetFields(BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy)
|
|
|
+ .Where(f => f.IsLiteral && !f.IsInitOnly && f.FieldType == typeof(string))
|
|
|
+ .Select(f => (string)f.GetRawConstantValue()!)
|
|
|
+ .Where(v => v != S8AuthorityHealthReason.Healthy)
|
|
|
+ .Select(v => new object[] { v });
|
|
|
+
|
|
|
+ /// <summary>
|
|
|
+ /// 放行只有一个出口:<c>State == HEALTHY</c>。
|
|
|
+ /// <para>用反射枚举<b>全部</b>原因码 —— 将来有人新增一个却忘了想清楚放不放行时,
|
|
|
+ /// 这条会自动变红。默认必须是拦。</para>
|
|
|
+ /// </summary>
|
|
|
+ [Theory]
|
|
|
+ [MemberData(nameof(AllNonHealthyReasonCodes))]
|
|
|
+ public void EveryNonHealthyReason_Blocks(string reasonCode)
|
|
|
+ {
|
|
|
+ foreach (var state in new[] { S8AuthorityHealthState.Degraded, S8AuthorityHealthState.Unknown })
|
|
|
+ {
|
|
|
+ var r = new S8AuthorityHealthResult { State = state, ReasonCode = reasonCode };
|
|
|
+ Assert.False(r.AllowsRecovery, $"state={state} reason={reasonCode} 不应放行");
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>状态是封闭三元集合,不得出现第四种(尤其不得出现 HEALTHY_EMPTY)。</summary>
|
|
|
+ [Fact]
|
|
|
+ public void StateVocabularyIsClosed_AndHasNoHealthyEmpty()
|
|
|
+ {
|
|
|
+ var states = typeof(S8AuthorityHealthState)
|
|
|
+ .GetFields(BindingFlags.Public | BindingFlags.Static | BindingFlags.FlattenHierarchy)
|
|
|
+ .Where(f => f.IsLiteral && f.FieldType == typeof(string))
|
|
|
+ .Select(f => (string)f.GetRawConstantValue()!)
|
|
|
+ .OrderBy(v => v, StringComparer.Ordinal)
|
|
|
+ .ToArray();
|
|
|
+
|
|
|
+ Assert.Equal(new[] { "DEGRADED", "HEALTHY", "UNKNOWN" }, states);
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>端口是接口且实现可被自动注册,调度器才能换实现而不改自身。</summary>
|
|
|
+ [Fact]
|
|
|
+ public void Resolver_IsASeamInterface()
|
|
|
+ {
|
|
|
+ Assert.True(typeof(IS8AuthorityHealthResolver).IsInterface);
|
|
|
+ Assert.True(typeof(IS8AuthorityHealthResolver)
|
|
|
+ .IsAssignableFrom(typeof(S8MdpAuthorityHealthResolver)));
|
|
|
+ }
|
|
|
+}
|