浏览代码

fix(mdp): 人工重算可取代未开跑的自动任务,解开重算队列死锁

页面「数据重算」原来被一条永远领不到的 QUEUED 自动任务永久挡住:
ClaimNextQueuedAsync 对非执行机只放行 MANUAL,而 EnqueueAsync 见到
任何 active 任务就回 409,两条各自正确的规则互相死锁。今天 UAT 租户
S5/S6/S7 就卡在 03:40 入队、执行机已退出的 AUTO_NIGHTLY 上。

- ModuleRebuildStore: 新增 TrySupersedeQueuedAsync,条件更新
  (WHERE status='QUEUED') 保证不会掀掉已被 Worker 抢成 RUNNING 的任务
- ModuleRebuildService: 人工请求可把同 scope 未开跑的自动任务收口为
  CANCELLED 后重新入队;MANUAL 之间、以及 RUNNING 仍然回 409
- 补 3 条守卫测试(取代成功 / RUNNING 不取代 / 自动之间不取代)

chore: bump version server 1.0.588
Co-authored-by: Cursor <cursoragent@cursor.com>
YY968XX 2 天之前
父节点
当前提交
fb4874a3c6

+ 3 - 3
server/Admin.NET.Web.Entry/Admin.NET.Web.Entry.csproj

@@ -11,9 +11,9 @@
     <GenerateSatelliteAssembliesForCore>true</GenerateSatelliteAssembliesForCore>
     <Copyright>Admin.NET</Copyright>
     <Description>Admin.NET 通用权限开发平台</Description>
-    <AssemblyVersion>1.0.587</AssemblyVersion>
-    <FileVersion>1.0.587</FileVersion>
-    <Version>1.0.587</Version>
+    <AssemblyVersion>1.0.588</AssemblyVersion>
+    <FileVersion>1.0.588</FileVersion>
+    <Version>1.0.588</Version>
   </PropertyGroup>
 
   <ItemGroup>

+ 64 - 0
server/Plugins/Admin.NET.Plugin.AiDOP.Tests/DataPlatform/ModuleRebuildServiceTests.cs

@@ -122,6 +122,56 @@ public class ModuleRebuildServiceTests
         Assert.Equal(ModuleRebuildStatus.Queued, result.Body.Status);
     }
 
+    /// <summary>
+    /// 夜间兜底任务在非执行机上永远领不到(<c>ClaimNextQueuedAsync</c> 的白名单),
+    /// 若它还能挡住人工入队,页面「数据重算」按钮就被永久卡死。人工请求必须能取代它。
+    /// </summary>
+    [Fact]
+    public async Task ManualEnqueue_SupersedesQueuedAutoJob()
+    {
+        var store = new MemoryJobStore();
+        var svc = new ModuleRebuildService(store, new ModuleRebuildQueue(), new AlwaysOnModuleRebuildCapability());
+        var nightly = await svc.EnqueueAsync("S2", 9, 1, null, "AUTO_NIGHTLY");
+
+        var manual = await svc.EnqueueAsync("S2", 9, 1, 7);
+
+        Assert.Equal(202, manual.StatusCode);
+        Assert.NotEqual(nightly.Body.JobId, manual.Body.JobId);
+        var superseded = store.Rows.Single(x => x.Id == nightly.Body.JobId);
+        Assert.Equal(ModuleRebuildStatus.Cancelled, superseded.Status);
+        Assert.Contains("AUTO_NIGHTLY", superseded.ErrorMessage);
+    }
+
+    /// <summary>已经开跑的任务不得被掀掉:取代只针对 QUEUED。</summary>
+    [Fact]
+    public async Task ManualEnqueue_DoesNotSupersedeRunningAutoJob()
+    {
+        var store = new MemoryJobStore();
+        var svc = new ModuleRebuildService(store, new ModuleRebuildQueue(), new AlwaysOnModuleRebuildCapability());
+        await svc.EnqueueAsync("S2", 9, 1, null, "AUTO_NIGHTLY");
+        var running = await store.ClaimNextQueuedAsync(new[] { "S2" });
+
+        var manual = await svc.EnqueueAsync("S2", 9, 1, 7);
+
+        Assert.Equal(409, manual.StatusCode);
+        Assert.Equal(ModuleRebuildStatus.Running, store.Rows.Single(x => x.Id == running.Id).Status);
+    }
+
+    /// <summary>取代资格只给人工请求;自动触发器之间仍按原样互斥,避免夜间批次自相残杀。</summary>
+    [Fact]
+    public async Task AutoEnqueue_DoesNotSupersedeQueuedAutoJob()
+    {
+        var store = new MemoryJobStore();
+        var svc = new ModuleRebuildService(store, new ModuleRebuildQueue(), new ConfigurableCapability { AutoMinIntervalHours = 0 });
+        var nightly = await svc.EnqueueAsync("S2", 9, 1, null, "AUTO_NIGHTLY");
+
+        var auto = await svc.EnqueueAsync("S2", 9, 1, null, "AUTO");
+
+        Assert.Equal(409, auto.StatusCode);
+        Assert.Equal(nightly.Body.JobId, auto.Body.JobId);
+        Assert.Equal(ModuleRebuildStatus.Queued, store.Rows.Single(x => x.Id == nightly.Body.JobId).Status);
+    }
+
     [Fact]
     public void S1Service_DelegatesToUnifiedPath()
     {
@@ -620,6 +670,20 @@ public class ModuleRebuildServiceTests
                     .OrderBy(x => x.Id).FirstOrDefault());
         }
 
+        public Task<bool> TrySupersedeQueuedAsync(long jobId, string reason, CancellationToken ct = default)
+        {
+            lock (_gate)
+            {
+                var row = Rows.FirstOrDefault(x => x.Id == jobId && x.Status == ModuleRebuildStatus.Queued);
+                if (row == null) return Task.FromResult(false);
+                row.Status = ModuleRebuildStatus.Cancelled;
+                row.CurrentStage = ModuleRebuildStages.Cancelled;
+                row.FinishedAt = DateTime.Now;
+                row.ErrorMessage = reason;
+                return Task.FromResult(true);
+            }
+        }
+
         public Task<AdoModuleDashboardRebuildJob> GetByIdAsync(string moduleCode, long id, long tenantId, long factoryId, CancellationToken ct = default)
         {
             lock (_gate)

+ 19 - 4
server/Plugins/Admin.NET.Plugin.AiDOP/DataPlatform/MdpRebuild/ModuleRebuildService.cs

@@ -64,17 +64,32 @@ public sealed class ModuleRebuildService : ITransient
             });
         }
 
+        var normalizedTrigger = string.IsNullOrWhiteSpace(triggerType) ? "MANUAL" : triggerType.Trim().ToUpperInvariant();
+        var manualRequest = ModuleRebuildTriggerType.IsManualTrigger(normalizedTrigger, requestedBy);
+
         var active = await _jobs.FindActiveAsync(scope.ModuleCode, scope.TenantId, scope.FactoryId, ct);
         if (active != null)
         {
-            return (409, Conflict(scope.ModuleCode, active));
+            // 人工重算可以取代同 scope 里「还没开跑的自动任务」:二者都是同 scope 全量,取代不丢工作量。
+            // 不让位则会死锁——ClaimNextQueuedAsync 只让执行机领自动任务,非执行机上这条 QUEUED 永远
+            // 不会消失,页面「数据重算」按钮就被它永久挡住(执行机指派还会随进程重启失效)。
+            var supersedable = manualRequest
+                && active.Status == ModuleRebuildStatus.Queued
+                && !ModuleRebuildTriggerType.IsManualTrigger(active.TriggerType, active.RequestedBy);
+            if (!supersedable
+                || !await _jobs.TrySupersedeQueuedAsync(active.Id, $"已被人工重算取代(原 {active.TriggerType})", ct))
+            {
+                return (409, Conflict(scope.ModuleCode, active));
+            }
+            _logger.LogInformation(
+                "[ModuleRebuild] {Module} tenant={Tenant} factory={Factory} 人工重算取代待执行任务 job={JobId} trigger={Trigger}",
+                scope.ModuleCode, scope.TenantId, scope.FactoryId, active.Id, active.TriggerType);
         }
 
         // AUTO / BOOTSTRAP 才冷却。MANUAL 是页面「数据重算」按钮,AUTO_NIGHTLY 是全量兜底,两者都必须放行。
-        var normalizedTrigger = string.IsNullOrWhiteSpace(triggerType) ? "MANUAL" : triggerType.Trim().ToUpperInvariant();
         var cooldownHours = _capability.AutoMinIntervalHours;
         if (cooldownHours > 0
-            && !ModuleRebuildTriggerType.IsManualTrigger(normalizedTrigger, requestedBy)
+            && !manualRequest
             && normalizedTrigger != ModuleRebuildTriggerType.AutoNightly)
         {
             var lastSuccess = await _jobs.FindLastSuccessAsync(scope.ModuleCode, scope.TenantId, scope.FactoryId, ct);
@@ -105,7 +120,7 @@ public sealed class ModuleRebuildService : ITransient
             ProgressPercent = 0,
             ProgressMessage = "已入队",
             LastProgressAt = now,
-            TriggerType = string.IsNullOrWhiteSpace(triggerType) ? "MANUAL" : triggerType.Trim().ToUpperInvariant(),
+            TriggerType = normalizedTrigger,
             RequestedBy = requestedBy,
             SubmittedAt = now,
             CreateTime = now,

+ 24 - 0
server/Plugins/Admin.NET.Plugin.AiDOP/DataPlatform/MdpRebuild/ModuleRebuildStore.cs

@@ -61,6 +61,13 @@ public interface IModuleRebuildJobStore
 {
     Task<AdoModuleDashboardRebuildJob> InsertQueuedAsync(AdoModuleDashboardRebuildJob row, CancellationToken ct = default);
     Task<AdoModuleDashboardRebuildJob> FindActiveAsync(string moduleCode, long tenantId, long factoryId, CancellationToken ct = default);
+    /// <summary>
+    /// 把一条仍在 QUEUED 的任务收口为 CANCELLED,给人工重算腾位。
+    ///
+    /// <para>条件更新(<c>WHERE status='QUEUED'</c>):若 Worker 已把它抢成 RUNNING 则返回 <c>false</c>,
+    /// 调用方必须据此退回 409,不得把正在跑的任务掀掉。</para>
+    /// </summary>
+    Task<bool> TrySupersedeQueuedAsync(long jobId, string reason, CancellationToken ct = default);
     /// <summary>该 scope 最近一条 SUCCESS,用于 AUTO 冷却判定。没有则返回 null。</summary>
     Task<AdoModuleDashboardRebuildJob> FindLastSuccessAsync(string moduleCode, long tenantId, long factoryId, CancellationToken ct = default);
     Task<AdoModuleDashboardRebuildJob> GetByIdAsync(string moduleCode, long id, long tenantId, long factoryId, CancellationToken ct = default);
@@ -119,6 +126,23 @@ public sealed class ModuleRebuildJobStore : IModuleRebuildJobStore, ITransient
             .OrderBy(x => x.Id)
             .FirstAsync(ct);
 
+    public async Task<bool> TrySupersedeQueuedAsync(long jobId, string reason, CancellationToken ct = default)
+    {
+        var now = DateTime.Now;
+        var n = await _db.Updateable<AdoModuleDashboardRebuildJob>()
+            .SetColumns(x => new AdoModuleDashboardRebuildJob
+            {
+                Status = ModuleRebuildStatus.Cancelled,
+                CurrentStage = ModuleRebuildStages.Cancelled,
+                FinishedAt = now,
+                ErrorMessage = reason,
+                UpdateTime = now
+            })
+            .Where(x => x.Id == jobId && x.Status == ModuleRebuildStatus.Queued)
+            .ExecuteCommandAsync(ct);
+        return n > 0;
+    }
+
     public Task<AdoModuleDashboardRebuildJob> FindLastSuccessAsync(string moduleCode, long tenantId, long factoryId, CancellationToken ct = default) =>
         _db.Queryable<AdoModuleDashboardRebuildJob>()
             .Where(x => x.ModuleCode == moduleCode && x.TenantId == tenantId && x.FactoryId == factoryId