S8RecoveryGateWiringTests.cs 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304
  1. using System.Reflection;
  2. using Admin.NET.Plugin.AiDOP.Service.S8;
  3. using Admin.NET.Plugin.AiDOP.Service.S8.Rules.Health;
  4. using Xunit;
  5. namespace Admin.NET.Plugin.AiDOP.Tests.S8;
  6. /// <summary>
  7. /// S8-AUTHORITY-HEALTH-GATE:门禁在调度器里的<b>接线位置与爆炸半径</b>。
  8. ///
  9. /// <para><b>为什么是源码扫描而不是行为测试</b>:<see cref="S8WatchSchedulerService"/>
  10. /// 的构造参数里有 8 个 <c>SqlSugarRepository&lt;T&gt;</c>,其无参构造读取
  11. /// <c>SqlSugarSetup.ITenant</c> 静态字段,没有 Furion 启动就是 NRE ——
  12. /// 本仓从未、也无法在测试里实例化调度器(<c>grep "new S8WatchSchedulerService"</c> 全仓 0 命中)。
  13. /// 既有的 <c>S8SchedulerScopeGuardTests</c> / <c>S8SystemRecoveredTimelineTests</c>
  14. /// 走的都是同一条路。</para>
  15. ///
  16. /// <para><b>判定逻辑不在这里</b>,在 <see cref="S8AuthorityHealthGateTests"/>(纯函数、可断言)。
  17. /// 本文件只回答一个问题:门禁挡住的到底是<b>哪一段</b>。
  18. /// 挡多了,正面观测停摆;挡少了,中台停更时照样假恢复。</para>
  19. /// </summary>
  20. public class S8RecoveryGateWiringTests
  21. {
  22. private const string SchedulerSrc = "Service/S8/S8WatchSchedulerService.cs";
  23. // ───────────────── helpers ─────────────────
  24. /// <summary>读取源码并剥掉注释行 —— 断言不应被注释里的字面量满足或破坏。</summary>
  25. private static string CodeOnly(string relative)
  26. {
  27. var dir = new DirectoryInfo(AppContext.BaseDirectory);
  28. while (dir != null && !Directory.Exists(Path.Combine(dir.FullName, "Admin.NET.Plugin.AiDOP")))
  29. dir = dir.Parent;
  30. Assert.NotNull(dir);
  31. var full = Path.Combine(dir!.FullName, "Admin.NET.Plugin.AiDOP",
  32. relative.Replace('/', Path.DirectorySeparatorChar));
  33. Assert.True(File.Exists(full), $"源码文件不存在,路径需同步更新:{full}");
  34. return string.Join('\n', File.ReadAllLines(full)
  35. .Where(l =>
  36. {
  37. var t = l.TrimStart();
  38. return !t.StartsWith("///", StringComparison.Ordinal)
  39. && !t.StartsWith("//", StringComparison.Ordinal);
  40. }));
  41. }
  42. private static string Scheduler() => CodeOnly(SchedulerSrc);
  43. private static int At(string code, string anchor)
  44. {
  45. var i = code.IndexOf(anchor, StringComparison.Ordinal);
  46. Assert.True(i >= 0, $"未找到锚点:{anchor}");
  47. return i;
  48. }
  49. private static string Slice(string code, string from, string to)
  50. {
  51. var a = At(code, from);
  52. var b = code.IndexOf(to, a, StringComparison.Ordinal);
  53. return b > a ? code[a..b] : code[a..];
  54. }
  55. private static string ProcessBody() => Slice(Scheduler(),
  56. "private async Task<List<S8WatchCreationResult>> ProcessSingleRuleAsync",
  57. "public async Task OnRuleCompletedAsync");
  58. private static string ReconcileBody() => Slice(Scheduler(),
  59. "private async Task<List<long>> ReconcileRecoveriesForRuleAsync",
  60. "private static AdoS8DetectionLog BuildHitLog");
  61. private static string RunSingleBody() => Slice(Scheduler(),
  62. "public async Task<S8RuleRunResult> RunSingleRuleAsync",
  63. "private async Task<List<S8WatchCreationResult>> ProcessSingleRuleAsync");
  64. /// <summary>
  65. /// 计算 <paramref name="index"/> 处的花括号嵌套深度。
  66. /// <para>只在<b>单个方法体切片</b>内使用:日志模板里的 <c>{RuleCode}</c> 之类占位符成对出现、
  67. /// 净增为 0,不影响相对深度比较。若将来该方法引入 raw string SQL,本助手需先剥离字符串字面量。</para>
  68. /// </summary>
  69. private static int DepthAt(string code, int index)
  70. {
  71. var d = 0;
  72. for (var i = 0; i < index; i++)
  73. {
  74. if (code[i] == '{') d++;
  75. else if (code[i] == '}') d--;
  76. }
  77. return d;
  78. }
  79. private static int Count(string code, string needle)
  80. {
  81. int n = 0, i = 0;
  82. while ((i = code.IndexOf(needle, i, StringComparison.Ordinal)) >= 0) { n++; i += needle.Length; }
  83. return n;
  84. }
  85. // ───────────────── 接线基本形态 ─────────────────
  86. /// <summary>调度器必须依赖端口接口,且不得自己知道运行日志表名。</summary>
  87. [Fact]
  88. public void Scheduler_DependsOnTheResolverSeamInterface()
  89. {
  90. var ctor = typeof(S8WatchSchedulerService).GetConstructors().Single();
  91. Assert.Contains(ctor.GetParameters(), p => p.ParameterType == typeof(IS8AuthorityHealthResolver));
  92. var code = Scheduler();
  93. Assert.DoesNotContain("new S8MdpAuthorityHealthResolver", code);
  94. // 物理表名属适配层的知识;调度器只传 DatasetCode。
  95. Assert.DoesNotContain("mdp_transform_run_log", code);
  96. Assert.DoesNotContain("dwd_requirement_examine_detail", code);
  97. Assert.DoesNotContain("mdp_std_so", code);
  98. }
  99. /// <summary>门禁必须在 reconcile <b>之前</b>,且位于 ProcessSingleRuleAsync 内。</summary>
  100. [Fact]
  101. public void Gate_RunsBeforeRecoveryReconcile()
  102. {
  103. var body = ProcessBody();
  104. Assert.True(At(body, "_authorityHealthResolver.ResolveAsync") < At(body, "health.AllowsRecovery"));
  105. Assert.True(At(body, "health.AllowsRecovery") < At(body, "ReconcileRecoveriesForRuleAsync("));
  106. }
  107. // ───────────────── CASE B(结构侧):空集不得成为拦截理由 ─────────────────
  108. /// <summary>
  109. /// CASE B:门禁与「本轮有没有命中」<b>完全无关</b>。
  110. /// <para>取数与门禁之间不得出现任何对结果规模的判断 ——
  111. /// 一旦有人补上 <c>if (hits.Count == 0)</c>,真实恢复会永远不再发生,
  112. /// 而日志上每一轮都是成功。</para>
  113. /// </summary>
  114. [Fact]
  115. public void B_GateIsNotConditionedOnHitCount()
  116. {
  117. var body = ProcessBody();
  118. var window = body[At(body, "await evaluator.EvaluateAsync")..At(body, "ReconcileRecoveriesForRuleAsync(")];
  119. Assert.DoesNotContain("hits.Count ==", window);
  120. Assert.DoesNotContain("hits.Count >", window);
  121. Assert.DoesNotContain("hits.Any(", window);
  122. Assert.DoesNotContain("!hits.Any", window);
  123. // reconcile 自身也不得新增空集早退 —— 空 hits 正是「全部恢复」的正常输入。
  124. var reconcile = ReconcileBody();
  125. Assert.DoesNotContain("hits.Count == 0", reconcile);
  126. Assert.DoesNotContain("!hits.Any()", reconcile);
  127. }
  128. // ───────────────── CASE I:抗抖计数不得被拦截路径触碰 ─────────────────
  129. /// <summary>
  130. /// CASE I:门禁拦截时 <c>ConsecutiveMissCount</c> <b>既不加也不减</b>。
  131. ///
  132. /// <para>实现方式必须是「整段跳过 reconcile」,而<b>不是</b>在 reconcile 内部某处 return ——
  133. /// 后者会让候选查询与 miss 自增的相对顺序变成定时炸弹:今天写在自增之前,
  134. /// 明天挪到之后,就变成「拦截期间照样累计 miss,中台一恢复立刻集体假恢复」。</para>
  135. /// </summary>
  136. [Fact]
  137. public void I_BlockedPath_LeavesConsecutiveMissCountUntouched()
  138. {
  139. var code = Scheduler();
  140. var reconcile = ReconcileBody();
  141. const string inc = "ConsecutiveMissCount = x.ConsecutiveMissCount + 1";
  142. Assert.Equal(2, Count(code, inc));
  143. Assert.Equal(2, Count(reconcile, inc)); // 两处自增必须全部落在 reconcile 内
  144. // 门禁绝不能被搬进 reconcile —— 那样自增与拦截的先后无法再被静态保证。
  145. Assert.DoesNotContain("_authorityHealthResolver", reconcile);
  146. Assert.DoesNotContain("AllowsRecovery", reconcile);
  147. Assert.DoesNotContain("S8AuthorityHealth", reconcile);
  148. // 拦截分支自身不碰任何抗抖计数与恢复标记。
  149. var body = ProcessBody();
  150. var blocked = body[At(body, "if (!health.AllowsRecovery)")..At(body, "ReconcileRecoveriesForRuleAsync(")];
  151. Assert.DoesNotContain("ConsecutiveMissCount", blocked);
  152. Assert.DoesNotContain("ConsecutiveHitCount", blocked);
  153. Assert.DoesNotContain("RecoveredAt", blocked);
  154. }
  155. // ───────────────── CASE J:正面观测不受影响 ─────────────────
  156. /// <summary>
  157. /// CASE J:门禁拦住的<b>只有恢复</b>,建单与刷新照常。
  158. ///
  159. /// <para>用花括号深度证明 <c>foreach (var hit in hits)</c> 与方法首行同级 ——
  160. /// 它没有被裹进门禁的 if/else 里。这比 <c>Assert.Contains</c> 强:
  161. /// 把 foreach 缩进到 else 分支内,字符串照样能匹配到,深度不会。</para>
  162. ///
  163. /// <para>正面观测停摆是比假恢复更早被发现、但同样严重的失效:
  164. /// 中台停更期间新异常一条也建不出来,而页面上一片安静。</para>
  165. /// </summary>
  166. [Fact]
  167. public void J_BlockedPath_StillRunsCreateAndRefresh()
  168. {
  169. var body = ProcessBody();
  170. var baseline = At(body, "var results = new List<S8WatchCreationResult>();");
  171. var gate = At(body, "if (!health.AllowsRecovery)");
  172. var loop = At(body, "foreach (var hit in hits)");
  173. Assert.True(loop > gate, "hit 循环必须在门禁之后");
  174. Assert.Equal(DepthAt(body, baseline), DepthAt(body, loop));
  175. }
  176. // ───────────────── CASE K:不判失败、不触发自动暂停 ─────────────────
  177. /// <summary>
  178. /// CASE K:门禁拦截<b>不是一次失败的 run</b>。
  179. ///
  180. /// <para>若拦截走 <c>Success = false</c>,规则完成回调会累计 <c>ConsecutiveFailureCount</c>,
  181. /// 达到阈值即写 <c>paused_until</c> 把规则自动暂停一小时。
  182. /// 于是中台跑一次批,监控就睡一小时 —— 而页面上只显示「最近结果:失败」,
  183. /// 没人知道是数据源的问题。</para>
  184. /// </summary>
  185. [Fact]
  186. public void K_BlockedPath_DoesNotFailTheRunNorAutoPause()
  187. {
  188. var body = ProcessBody();
  189. var blocked = body[At(body, "if (!health.AllowsRecovery)")..At(body, "foreach (var hit in hits)")];
  190. Assert.DoesNotContain("throw", blocked);
  191. Assert.DoesNotContain("Success = false", blocked);
  192. Assert.DoesNotContain("S8RuleRunResult", blocked);
  193. Assert.DoesNotContain("Enabled = false", blocked);
  194. Assert.DoesNotContain("PausedUntil", blocked);
  195. // 门禁不得上移到 RunSingleRuleAsync —— 那一层的失败语义就是整条 run 失败。
  196. Assert.DoesNotContain("_authorityHealthResolver", RunSingleBody());
  197. }
  198. /// <summary>
  199. /// CASE K':探测自身失败必须<b>降级为 UNKNOWN</b>,不得让异常穿到上层变成整条 run 失败 ——
  200. /// 那等于绕过本条守卫达成同一个后果。该兜底落在适配实现里,此处断言它确实存在。
  201. /// </summary>
  202. [Fact]
  203. public void K2_ResolverFailure_DegradesToUnknown_NotRunFailure()
  204. {
  205. var resolver = CodeOnly("Service/S8/Rules/Health/S8MdpAuthorityHealthResolver.cs");
  206. var probe = resolver[At(resolver, "catch (Exception ex)")..];
  207. Assert.Contains("S8AuthorityHealthState.Unknown", probe);
  208. Assert.Contains("S8AuthorityHealthReason.ResolverFailed", probe);
  209. Assert.DoesNotContain("throw;", probe);
  210. }
  211. // ───────────────── 观测性:拦截必须留痕 ─────────────────
  212. /// <summary>
  213. /// 拦截期间异常只增不减,这件事<b>必须能被查到</b>。
  214. /// <para>只写日志不够 —— 日志会滚走,而 detection_log 是可查的。
  215. /// 原因码要落 <c>failure_reason</c>,否则只知道「被拦了」、不知道为什么。</para>
  216. /// </summary>
  217. [Fact]
  218. public void BlockedPath_WritesRecoveryBlockedDetectionLog()
  219. {
  220. var body = ProcessBody();
  221. var blocked = body[At(body, "if (!health.AllowsRecovery)")..At(body, "foreach (var hit in hits)")];
  222. Assert.Contains("WriteDetectionLogAsync", blocked);
  223. Assert.Contains("DetectResult = DetectResultRecoveryBlocked", blocked);
  224. Assert.Contains("FailureReason = health.ReasonCode", blocked);
  225. Assert.Contains("PayloadSnapshot", blocked);
  226. // detect_result 列是 varchar(32),常量不得超长。
  227. var f = typeof(S8WatchSchedulerService).GetField("DetectResultRecoveryBlocked",
  228. BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Public);
  229. Assert.NotNull(f);
  230. var v = (string)f!.GetRawConstantValue()!;
  231. Assert.Equal("RECOVERY_BLOCKED", v);
  232. Assert.True(v.Length <= 32);
  233. }
  234. // ───────────────── 适配层的租户隔离 ─────────────────
  235. /// <summary>
  236. /// 适配层的租户谓词必须是严格相等。
  237. ///
  238. /// <para>仓内 <c>MdpMonitorService.BuildMdpRunLogTenantWhere</c> 用的是
  239. /// <c>(tenant_id = @TenantId OR tenant_id = 0)</c> —— <b>本类不得复用</b>:
  240. /// 该表确实存在 <c>tenant_id = 0</c> 的平台行(实测 S1 有 111 条,
  241. /// 且 <c>S5_PURCHASE_RECEIPT_MDP_SYNC</c> 有 325 条平台级 SUCCESS),
  242. /// 一条这样的行会让所有租户同时被判成健康。</para>
  243. /// </summary>
  244. [Fact]
  245. public void Resolver_UsesStrictTenantEquality()
  246. {
  247. var resolver = CodeOnly("Service/S8/Rules/Health/S8MdpAuthorityHealthResolver.cs");
  248. Assert.DoesNotContain("OR tenant_id = 0", resolver);
  249. Assert.DoesNotContain("BuildMdpRunLogTenantWhere", resolver);
  250. Assert.Equal(3, Count(resolver, "r.tenant_id = @TenantId")); // 三条运行日志查询各一次
  251. // 必须显式过滤终态,否则会取到永不回收的孤儿 RUNNING 行。
  252. Assert.Contains("r.status = 'SUCCESS'", resolver);
  253. Assert.Contains("r.end_time IS NOT NULL", resolver);
  254. // 回溯窗口不是可选优化:无界回溯在长期停摆的租户上实测单次 146ms。
  255. Assert.Contains("r.start_time >= @LookbackFrom", resolver);
  256. }
  257. }