|
|
@@ -0,0 +1,247 @@
|
|
|
+using System.Text.RegularExpressions;
|
|
|
+using Admin.NET.Plugin.AiDOP.Service.S8.Rules.Health;
|
|
|
+using Xunit;
|
|
|
+
|
|
|
+namespace Admin.NET.Plugin.AiDOP.Tests.S8;
|
|
|
+
|
|
|
+/// <summary>
|
|
|
+/// S8 Authority 健康判定的「in-flight 判据」契约。
|
|
|
+///
|
|
|
+/// <para><b>要守的是什么</b>：生产侧写 <c>mdp_transform_run_log</c> 的形态是
|
|
|
+/// 「<c>INSERT RUNNING</c> → <c>try { UPDATE SUCCESS } catch { UPDATE FAILED }</c>」，<b>没有 finally</b>。
|
|
|
+/// 进程被 SIGTERM / kill / 宿主终止时没有异常投递到托管层，那一行就永远停在 <c>end_time IS NULL</c>。
|
|
|
+/// 实测全库近 400 条这样的历史残留、最早回溯一个多月、横跨八个 job family。
|
|
|
+/// 若健康判定把它们全当「当前尚在执行」，Authority 会被长期误判为 Degraded。</para>
|
|
|
+///
|
|
|
+/// <para><b>但反方向同样致命</b>：如果为了让健康变绿就把所有 <c>RUNNING</c> 都过滤掉，
|
|
|
+/// 一次真正卡住的生产运行将再也不会被拦截 —— 而现有 schema 没有
|
|
|
+/// instance_id / pid / heartbeat，<b>根本无法证明进程已死</b>。
|
|
|
+/// 所以判据只能是「已被同 job+tenant 的更晚终态执行覆盖」这一条**可证明**的事实，
|
|
|
+/// 绝不能退化成按时间猜（近 30 天正常长任务实测可达 7583 秒）。</para>
|
|
|
+///
|
|
|
+/// <para><b>覆盖边界说明</b>：抑制判据本身写在 <see cref="S8MdpAuthorityHealthResolver"/> 的裸 SQL 里，
|
|
|
+/// 无库不可执行，故本文件分两段守：
|
|
|
+/// ① SQL 的<b>结构不变量</b>（tenant 必须参与关联、必须严格 <c>></c>、终态口径必须是 <c>end_time IS NOT NULL</c>）——
|
|
|
+/// 针对的正是最容易写错、且写错后静默的三处；
|
|
|
+/// ② 判据产出之后的<b>下游后果</b>（有/无 in-flight 分别得到什么 State 与 ReasonCode）——
|
|
|
+/// 走纯函数 <see cref="S8AuthorityHealthEvaluator"/>，真断言最终判定，不是断言字符串。</para>
|
|
|
+/// </summary>
|
|
|
+public class S8AuthorityInFlightOrphanFilterTests
|
|
|
+{
|
|
|
+ private const long TenantA = 838257186181189L;
|
|
|
+ private const long TenantB = 797403760988229L;
|
|
|
+ private const string Dataset = "ORDER_DELIVERY_RISK";
|
|
|
+
|
|
|
+ private static readonly DateTime Now = new(2026, 9, 18, 12, 0, 0);
|
|
|
+ private static readonly TimeSpan Stale = TimeSpan.FromHours(1);
|
|
|
+
|
|
|
+ // ══════════════════ 段①：抑制判据的 SQL 结构不变量 ══════════════════
|
|
|
+
|
|
|
+ private static string ResolverSource()
|
|
|
+ {
|
|
|
+ var dir = new DirectoryInfo(AppContext.BaseDirectory);
|
|
|
+ while (dir != null && !Directory.Exists(Path.Combine(dir.FullName, "Admin.NET.Plugin.AiDOP")))
|
|
|
+ dir = dir.Parent;
|
|
|
+ Assert.NotNull(dir);
|
|
|
+ var path = Path.Combine(dir!.FullName, "Admin.NET.Plugin.AiDOP",
|
|
|
+ "Service", "S8", "Rules", "Health", "S8MdpAuthorityHealthResolver.cs");
|
|
|
+ Assert.True(File.Exists(path), $"源码路径需同步更新：{path}");
|
|
|
+ return File.ReadAllText(path);
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>取 QueryNewerInFlightAsync 的 SQL 字面量，去掉 XML 文档注释（注释要点名被禁的写法，那是说明不是实现）。</summary>
|
|
|
+ private static string InFlightSql()
|
|
|
+ {
|
|
|
+ var code = string.Join('\n', ResolverSource().Split('\n')
|
|
|
+ .Where(l => !l.TrimStart().StartsWith("///", StringComparison.Ordinal)));
|
|
|
+ // 必须定位到「方法定义」而不是调用点 —— 文件里先出现的是 Evaluate 里的调用行。
|
|
|
+ var i = code.IndexOf("private async Task<DateTime?> QueryNewerInFlightAsync", StringComparison.Ordinal);
|
|
|
+ Assert.True(i >= 0, "未找到 QueryNewerInFlightAsync 的方法定义（签名若变更需同步本测试）");
|
|
|
+ var j = code.IndexOf("SqlQueryAsync", i, StringComparison.Ordinal);
|
|
|
+ Assert.True(j > i, "未找到该方法的 SQL 结束位置");
|
|
|
+ return code[i..j];
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>抑制子查询必须存在 —— 否则历史残留会全数回流成 in-flight。</summary>
|
|
|
+ [Fact]
|
|
|
+ public void 抑制子查询存在()
|
|
|
+ {
|
|
|
+ var sql = InFlightSql();
|
|
|
+ Assert.Contains("NOT EXISTS", sql, StringComparison.Ordinal);
|
|
|
+ Assert.Contains("mdp_transform_run_log later", sql, StringComparison.Ordinal);
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>
|
|
|
+ /// tenant_id 必须参与关联。
|
|
|
+ /// 只按 job_code 关联会让「A 租户跑完」压掉「B 租户真的卡住」—— 跨租户静默放行，
|
|
|
+ /// 这是本判据最危险的写错方式。
|
|
|
+ /// </summary>
|
|
|
+ [Fact]
|
|
|
+ public void 抑制判据必须同时匹配job与tenant()
|
|
|
+ {
|
|
|
+ var sql = InFlightSql();
|
|
|
+ Assert.Matches(new Regex(@"later\.job_code\s*=\s*r\.job_code"), sql);
|
|
|
+ Assert.Matches(new Regex(@"later\.tenant_id\s*=\s*r\.tenant_id"), sql);
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>
|
|
|
+ /// 必须严格 <c>></c>。用 <c>>=</c> 会让同一时刻开始的另一条终态行把本行压掉，
|
|
|
+ /// 而「同时开始」恰恰不构成「更晚跑完」的证据。
|
|
|
+ /// </summary>
|
|
|
+ [Fact]
|
|
|
+ public void 更晚判定必须严格大于不得用大于等于()
|
|
|
+ {
|
|
|
+ var sql = InFlightSql();
|
|
|
+ Assert.Matches(new Regex(@"later\.start_time\s*>\s*r\.start_time"), sql);
|
|
|
+ Assert.DoesNotMatch(new Regex(@"later\.start_time\s*>=\s*r\.start_time"), sql);
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>
|
|
|
+ /// 「更晚那条」必须是<b>终态</b>。若漏掉这一条，另一条同样悬着的 RUNNING
|
|
|
+ /// 就能把更早的 RUNNING 压掉 —— 两条都没结束，却互相证明对方结束了。
|
|
|
+ /// 终态口径沿用本文件既有的 end_time IS NOT NULL（与 QueryNewerTerminalFailureAsync 一致）。
|
|
|
+ /// </summary>
|
|
|
+ [Fact]
|
|
|
+ public void 更晚那条必须已终态()
|
|
|
+ {
|
|
|
+ var sql = InFlightSql();
|
|
|
+ Assert.Matches(new Regex(@"later\.end_time\s+IS\s+NOT\s+NULL"), sql);
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>候选侧的原有条件一条都不能丢：未终态 + 晚于锚点 + 本 job/tenant。</summary>
|
|
|
+ [Fact]
|
|
|
+ public void 候选侧原有条件保持不变()
|
|
|
+ {
|
|
|
+ var sql = InFlightSql();
|
|
|
+ Assert.Matches(new Regex(@"r\.end_time\s+IS\s+NULL"), sql);
|
|
|
+ Assert.Matches(new Regex(@"r\.start_time\s*>\s*@Anchor"), sql);
|
|
|
+ Assert.Matches(new Regex(@"r\.job_code\s*=\s*@JobCode"), sql);
|
|
|
+ Assert.Matches(new Regex(@"r\.tenant_id\s*=\s*@TenantId"), sql);
|
|
|
+ Assert.Contains("MIN(r.start_time)", sql, StringComparison.Ordinal);
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>不得借本批塞入按时间猜 orphan 的逻辑（固定 timeout 会误伤 7583 秒的合法长任务）。</summary>
|
|
|
+ [Fact]
|
|
|
+ public void 不得引入时间阈值猜测()
|
|
|
+ {
|
|
|
+ var sql = InFlightSql();
|
|
|
+ foreach (var forbidden in new[] { "INTERVAL", "NOW()", "TIMESTAMPDIFF", "DATE_SUB" })
|
|
|
+ Assert.DoesNotContain(forbidden, sql, StringComparison.OrdinalIgnoreCase);
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>本批是只读判定修正：抑制逻辑里不得出现任何写操作。</summary>
|
|
|
+ [Fact]
|
|
|
+ public void 判据保持只读()
|
|
|
+ {
|
|
|
+ var sql = InFlightSql();
|
|
|
+ foreach (var w in new[] { "UPDATE ", "DELETE ", "INSERT ", "TRUNCATE" })
|
|
|
+ Assert.DoesNotContain(w, sql, StringComparison.OrdinalIgnoreCase);
|
|
|
+ }
|
|
|
+
|
|
|
+ // ══════════════════ 段②：判据产出之后的下游后果（纯函数真断言）══════════════════
|
|
|
+
|
|
|
+ private static S8AuthorityObservation Obs(DateTime? lastSuccessEndAt, DateTime? inFlightStartAt) => new()
|
|
|
+ {
|
|
|
+ DatasetCode = Dataset,
|
|
|
+ TenantId = TenantA,
|
|
|
+ ObservedAt = Now,
|
|
|
+ SpecFound = true,
|
|
|
+ ProducerTrusted = true,
|
|
|
+ LastSuccessEndAt = lastSuccessEndAt,
|
|
|
+ HasNewerTerminalFailure = false,
|
|
|
+ NewerTerminalFailureStatus = null,
|
|
|
+ OldestNewerInFlightStartAt = inFlightStartAt,
|
|
|
+ SnapshotInvariant = S8SnapshotInvariant.NotApplicable,
|
|
|
+ SnapshotDetail = null,
|
|
|
+ StaleWindow = Stale
|
|
|
+ };
|
|
|
+
|
|
|
+ /// <summary>
|
|
|
+ /// 历史残留被抑制后（判据返回 null），健康判定不得再出现 IN_FLIGHT / STUCK_RUN。
|
|
|
+ /// 这是本批要消掉的那个误判。
|
|
|
+ /// </summary>
|
|
|
+ [Fact]
|
|
|
+ public void 被抑制后不再判InFlight()
|
|
|
+ {
|
|
|
+ var r = S8AuthorityHealthEvaluator.Evaluate(
|
|
|
+ Obs(lastSuccessEndAt: Now.AddMinutes(-30), inFlightStartAt: null));
|
|
|
+
|
|
|
+ Assert.Equal(S8AuthorityHealthState.Healthy, r.State);
|
|
|
+ Assert.NotEqual(S8AuthorityHealthReason.InFlight, r.ReasonCode);
|
|
|
+ Assert.NotEqual(S8AuthorityHealthReason.StuckRun, r.ReasonCode);
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>
|
|
|
+ /// 尚无更晚终态的 RUNNING 必须继续拦截 —— 反向守卫。
|
|
|
+ /// 若这条挂了，说明有人为了"健康变绿"把真正在飞/卡住的运行也过滤掉了。
|
|
|
+ /// </summary>
|
|
|
+ [Fact]
|
|
|
+ public void 未被抑制时仍判InFlight()
|
|
|
+ {
|
|
|
+ var r = S8AuthorityHealthEvaluator.Evaluate(
|
|
|
+ Obs(lastSuccessEndAt: Now.AddMinutes(-30), inFlightStartAt: Now.AddMinutes(-10)));
|
|
|
+
|
|
|
+ Assert.Equal(S8AuthorityHealthState.Degraded, r.State);
|
|
|
+ Assert.Equal(S8AuthorityHealthReason.InFlight, r.ReasonCode);
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>StuckWindow 语义不得被本批改动：超过 StaleWindow 仍应升级为 STUCK_RUN。</summary>
|
|
|
+ [Fact]
|
|
|
+ public void Stuck窗口语义不变()
|
|
|
+ {
|
|
|
+ var r = S8AuthorityHealthEvaluator.Evaluate(
|
|
|
+ Obs(lastSuccessEndAt: Now.AddHours(-5), inFlightStartAt: Now.AddHours(-3)));
|
|
|
+
|
|
|
+ Assert.Equal(S8AuthorityHealthState.Degraded, r.State);
|
|
|
+ Assert.Equal(S8AuthorityHealthReason.StuckRun, r.ReasonCode);
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>
|
|
|
+ /// 锚点语义不得被本批改动。
|
|
|
+ ///
|
|
|
+ /// <para>注意这里的真实语义（写测试时我先猜错过）：<b>没有任何成功运行时，评估器在分支③
|
|
|
+ /// 就短路成 <c>Unknown / NO_SUCCESSFUL_RUN</c>，根本走不到 in-flight 的分支⑤</b>。
|
|
|
+ /// 也就是说 in-flight 判据只在「已有成功、且之后出现未终态运行」时才有意义 ——
|
|
|
+ /// resolver 里 <c>anchor = lastSuccess ?? lookbackFrom</c> 的回退分支，
|
|
|
+ /// 对最终健康判定并不产生 in-flight 后果。</para>
|
|
|
+ ///
|
|
|
+ /// <para>本测试锁死这一点：无论 in-flight 传不传值，无成功时结论都必须一致。</para>
|
|
|
+ /// </summary>
|
|
|
+ [Fact]
|
|
|
+ public void 锚点语义不变_无成功时短路为NoSuccessfulRun()
|
|
|
+ {
|
|
|
+ var withInFlight = S8AuthorityHealthEvaluator.Evaluate(
|
|
|
+ Obs(lastSuccessEndAt: null, inFlightStartAt: Now.AddMinutes(-10)));
|
|
|
+ var withoutInFlight = S8AuthorityHealthEvaluator.Evaluate(
|
|
|
+ Obs(lastSuccessEndAt: null, inFlightStartAt: null));
|
|
|
+
|
|
|
+ Assert.Equal(S8AuthorityHealthState.Unknown, withInFlight.State);
|
|
|
+ Assert.Equal(S8AuthorityHealthReason.NoSuccessfulRun, withInFlight.ReasonCode);
|
|
|
+ Assert.Equal(withoutInFlight.State, withInFlight.State);
|
|
|
+ Assert.Equal(withoutInFlight.ReasonCode, withInFlight.ReasonCode);
|
|
|
+ }
|
|
|
+
|
|
|
+ /// <summary>抑制只作用于 in-flight 这一路；更晚终态失败的拦截不受影响。</summary>
|
|
|
+ [Fact]
|
|
|
+ public void 终态失败拦截不受影响()
|
|
|
+ {
|
|
|
+ var o = Obs(lastSuccessEndAt: Now.AddMinutes(-30), inFlightStartAt: null);
|
|
|
+ var r = S8AuthorityHealthEvaluator.Evaluate(new S8AuthorityObservation
|
|
|
+ {
|
|
|
+ DatasetCode = o.DatasetCode,
|
|
|
+ TenantId = TenantB,
|
|
|
+ ObservedAt = o.ObservedAt,
|
|
|
+ SpecFound = true,
|
|
|
+ ProducerTrusted = true,
|
|
|
+ LastSuccessEndAt = o.LastSuccessEndAt,
|
|
|
+ HasNewerTerminalFailure = true,
|
|
|
+ NewerTerminalFailureStatus = "FAILED",
|
|
|
+ OldestNewerInFlightStartAt = null,
|
|
|
+ SnapshotInvariant = S8SnapshotInvariant.NotApplicable,
|
|
|
+ StaleWindow = Stale
|
|
|
+ });
|
|
|
+
|
|
|
+ Assert.Equal(S8AuthorityHealthState.Degraded, r.State);
|
|
|
+ Assert.NotEqual(S8AuthorityHealthReason.InFlight, r.ReasonCode);
|
|
|
+ }
|
|
|
+}
|
YY968XX