Browse Source

chore(s8): add log retention cleanup job

- Add S8LogRetentionCleanupJob with [Daily] (midnight) cadence:
  - ado_s8_detection_log: 30-day retention (detected_at cutoff)
  - ado_s8_notification_log: 90-day retention (created_at cutoff)
  - SysJobTriggerRecord: 14-day retention (CreatedTime cutoff)
- Default OFF; opt-in via AIDOP_S8_LOG_CLEANUP_ENABLED=true env (same
  AIDOP_* convention as SqlSugarSetup / S8WatchSchedulerJob).
- Batched physical DELETE via Queryable.Take(1000) + Deleteable.Where(ids)
  two-step (SqlSugar Deleteable does not support Take); per-Job/per-table
  cap is 50 batches = 50,000 rows, residual swept on next day.
- Single-fire disabled log via Interlocked.Exchange (_firstDisabledLogged);
  per-run summary always logged at LogInformation.
- Does not touch SysLogEx / SysLogOp / Admin.NET.Core LogJob; runs
  alongside [Daily] @00:00 LogJob with no table overlap.

Bump csproj 1.0.136 -> 1.0.137.
YY968XX 1 month ago
parent
commit
b4beb673a5

+ 3 - 3
server/Admin.NET.Web.Entry/Admin.NET.Web.Entry.csproj

@@ -11,9 +11,9 @@
     <GenerateSatelliteAssembliesForCore>true</GenerateSatelliteAssembliesForCore>
     <Copyright>Admin.NET</Copyright>
     <Description>Admin.NET 通用权限开发平台</Description>
-    <AssemblyVersion>1.0.136</AssemblyVersion>
-    <FileVersion>1.0.136</FileVersion>
-    <Version>1.0.136</Version>
+    <AssemblyVersion>1.0.137</AssemblyVersion>
+    <FileVersion>1.0.137</FileVersion>
+    <Version>1.0.137</Version>
   </PropertyGroup>
 
   <ItemGroup>

+ 172 - 0
server/Plugins/Admin.NET.Plugin.AiDOP/Job/S8LogRetentionCleanupJob.cs

@@ -0,0 +1,172 @@
+using Admin.NET.Core;
+using Admin.NET.Plugin.AiDOP.Entity.S8;
+using Furion.Schedule;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using SqlSugar;
+
+namespace Admin.NET.Plugin.AiDOP.Job;
+
+/// <summary>
+/// S8-LOG-RETENTION-CLEANUP-P1-2:S8 日志保留清理。
+/// 每日凌晨分批物理 DELETE 三张日志表过期数据:
+///  - ado_s8_detection_log:30 天保留(detected_at &lt; now - 30d)
+///  - ado_s8_notification_log:90 天保留(created_at &lt; now - 90d)
+///  - SysJobTriggerRecord:14 天保留(CreatedTime &lt; now - 14d)
+/// 默认 OFF;env override AIDOP_S8_LOG_CLEANUP_ENABLED=true 强制开启。
+/// 分批:单批 CleanupBatchSize 行;单表单次 Job 最多 MaxCleanupBatchesPerTable 批;
+/// 达到单次上限后剩余过期数据在下一日继续清理,避免单次 Job 长时间占用 DB。
+/// SqlSugar Deleteable 不支持 Take,采用 Queryable.Take(N).Select(Id) + Deleteable.Where(ids.Contains) 两步删除。
+/// 与 Admin.NET.Core/Job/LogJob.cs 同走 [Daily](00:00),两 Job 删除不同表无锁冲突。
+/// AdoS8DetectionLog.is_deleted 字段对 Queryable 不做默认过滤,本批一并物理删除软删行与活跃过期行。
+/// </summary>
+[JobDetail("job_s8_log_retention_cleanup", Description = "S8 日志保留清理",
+    GroupName = "default", Concurrent = false)]
+[Daily(TriggerId = "trigger_s8_log_retention_cleanup", Description = "每日凌晨执行")]
+public class S8LogRetentionCleanupJob : IJob
+{
+    private readonly IServiceScopeFactory _scopeFactory;
+    private readonly ILogger _logger;
+
+    // 禁用日志单进程内只输出一次,避免每次触发都刷日志。
+    private static int _firstDisabledLogged;
+
+    // env override 命名沿用 P0 批次的 AIDOP_* 前缀(与 S8WatchSchedulerJob 等一致)。
+    private const string EnvCleanupEnabled = "AIDOP_S8_LOG_CLEANUP_ENABLED";
+
+    // 已拍板分级保留天数。
+    private const int DetectionLogRetentionDays = 30;
+    private const int NotificationLogRetentionDays = 90;
+    private const int JobTriggerRecordRetentionDays = 14;
+
+    // 分批参数:单批 1000 行;单表单次 Job 最多 50 批(5w 行/表/日上限),残余下一日继续。
+    private const int CleanupBatchSize = 1000;
+    private const int MaxCleanupBatchesPerTable = 50;
+
+    public S8LogRetentionCleanupJob(IServiceScopeFactory scopeFactory, ILoggerFactory loggerFactory)
+    {
+        _scopeFactory = scopeFactory;
+        _logger = loggerFactory.CreateLogger(nameof(S8LogRetentionCleanupJob));
+    }
+
+    public async Task ExecuteAsync(JobExecutingContext context, CancellationToken stoppingToken)
+    {
+        // 默认 OFF;env override AIDOP_S8_LOG_CLEANUP_ENABLED=true 强制开启。
+        // 与 P0 Scheduler:Enabled / S8:Scheduler:Enabled 独立,本批不复用既有 gate。
+        var envRaw = Environment.GetEnvironmentVariable(EnvCleanupEnabled);
+        if (!bool.TryParse(envRaw, out var enabled) || !enabled)
+        {
+            if (Interlocked.Exchange(ref _firstDisabledLogged, 1) == 0)
+            {
+                _logger.LogInformation(
+                    "S8LogRetentionCleanupJob 被配置禁用:{EnvName} 未设置或非 true",
+                    EnvCleanupEnabled);
+            }
+            return;
+        }
+
+        using var scope = _scopeFactory.CreateScope();
+        var db = scope.ServiceProvider.GetRequiredService<ISqlSugarClient>().CopyNew();
+
+        var totalDetection = 0;
+        var totalNotification = 0;
+        var totalTrigger = 0;
+
+        try
+        {
+            totalDetection = await CleanupDetectionLogAsync(db, stoppingToken);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "ado_s8_detection_log 清理失败");
+        }
+
+        try
+        {
+            totalNotification = await CleanupNotificationLogAsync(db, stoppingToken);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "ado_s8_notification_log 清理失败");
+        }
+
+        try
+        {
+            totalTrigger = await CleanupJobTriggerRecordAsync(db, stoppingToken);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "SysJobTriggerRecord 清理失败");
+        }
+
+        _logger.LogInformation(
+            "S8LogRetentionCleanupJob 本轮删除摘要:detection_log={DetectionCount} notification_log={NotificationCount} sys_job_trigger_record={TriggerCount}",
+            totalDetection, totalNotification, totalTrigger);
+    }
+
+    private async Task<int> CleanupDetectionLogAsync(ISqlSugarClient db, CancellationToken token)
+    {
+        var cutoff = DateTime.Now.AddDays(-DetectionLogRetentionDays);
+        var total = 0;
+        for (var i = 0; i < MaxCleanupBatchesPerTable; i++)
+        {
+            if (token.IsCancellationRequested) break;
+            var ids = await db.Queryable<AdoS8DetectionLog>()
+                .Where(x => x.DetectedAt < cutoff)
+                .Take(CleanupBatchSize)
+                .Select(x => x.Id)
+                .ToListAsync();
+            if (ids.Count == 0) break;
+            var affected = await db.Deleteable<AdoS8DetectionLog>()
+                .Where(x => ids.Contains(x.Id))
+                .ExecuteCommandAsync(token);
+            total += affected;
+            if (ids.Count < CleanupBatchSize) break;
+        }
+        return total;
+    }
+
+    private async Task<int> CleanupNotificationLogAsync(ISqlSugarClient db, CancellationToken token)
+    {
+        var cutoff = DateTime.Now.AddDays(-NotificationLogRetentionDays);
+        var total = 0;
+        for (var i = 0; i < MaxCleanupBatchesPerTable; i++)
+        {
+            if (token.IsCancellationRequested) break;
+            var ids = await db.Queryable<AdoS8NotificationLog>()
+                .Where(x => x.CreatedAt < cutoff)
+                .Take(CleanupBatchSize)
+                .Select(x => x.Id)
+                .ToListAsync();
+            if (ids.Count == 0) break;
+            var affected = await db.Deleteable<AdoS8NotificationLog>()
+                .Where(x => ids.Contains(x.Id))
+                .ExecuteCommandAsync(token);
+            total += affected;
+            if (ids.Count < CleanupBatchSize) break;
+        }
+        return total;
+    }
+
+    private async Task<int> CleanupJobTriggerRecordAsync(ISqlSugarClient db, CancellationToken token)
+    {
+        var cutoff = DateTime.Now.AddDays(-JobTriggerRecordRetentionDays);
+        var total = 0;
+        for (var i = 0; i < MaxCleanupBatchesPerTable; i++)
+        {
+            if (token.IsCancellationRequested) break;
+            var ids = await db.Queryable<SysJobTriggerRecord>()
+                .Where(x => x.CreatedTime != null && x.CreatedTime < cutoff)
+                .Take(CleanupBatchSize)
+                .Select(x => x.Id)
+                .ToListAsync();
+            if (ids.Count == 0) break;
+            var affected = await db.Deleteable<SysJobTriggerRecord>()
+                .Where(x => ids.Contains(x.Id))
+                .ExecuteCommandAsync(token);
+            total += affected;
+            if (ids.Count < CleanupBatchSize) break;
+        }
+        return total;
+    }
+}