using Admin.NET.Plugin.AiDOP.Entity.DataPlatform;
namespace Admin.NET.Plugin.AiDOP.DataPlatform.Executors;
/// 统一入站抽数执行器(方式甲 DB / 方式乙 API)。
public interface IMdpSourcePullExecutor
{
/// DB_SYNC / API_PULL
string SupportedType { get; }
Task PullAsync(MdpSource source, MdpEntity entity, MdpPullContext ctx, CancellationToken cancellationToken = default);
}
public sealed class MdpPullContext
{
public long TenantId { get; set; } = 1300000000001L;
public string BatchId { get; set; } = "";
/// 强制全量(忽略水位与滚动下界)。
public bool FullRefresh { get; set; }
/// 关联 mdp_sync_task.task_code,用于读取调度上的同步窗口。
public string? TaskCode { get; set; }
/// FULL / INCR / ROLLING(可由调度表覆盖)。
public string? SyncWindowType { get; set; }
/// 如 7d / 24h / 2026-01-01。
public string? SyncWindowValue { get; set; }
/// ROLLING 解析后的下界(含)。
public DateTime? WindowFrom { get; set; }
/// 全量分页偏移(仅 FullRefresh / 无 incr 时由 PullAll 推进)。
public int Offset { get; set; }
}
public sealed class MdpPullResult
{
public int RowsPulled { get; set; }
public int RowsWritten { get; set; }
public string? NewCursor { get; set; }
public string? Message { get; set; }
}
/// 按实体/源类型选择 DB 或 API 执行器。
public sealed class MdpSourcePullDispatcher : ITransient
{
private readonly MdpDbPullExecutor _dbExecutor;
private readonly MdpApiPullExecutor _apiExecutor;
private readonly ISqlSugarClient _db;
public MdpSourcePullDispatcher(
MdpDbPullExecutor dbExecutor,
MdpApiPullExecutor apiExecutor,
ISqlSugarClient db)
{
_dbExecutor = dbExecutor;
_apiExecutor = apiExecutor;
_db = db;
}
public async Task PullByEntityCodeAsync(string entityCode, MdpPullContext ctx, CancellationToken cancellationToken = default)
{
var entity = await _db.Queryable()
.Where(x => x.EntityCode == entityCode && x.Status == 1)
.FirstAsync(cancellationToken)
?? throw new InvalidOperationException($"mdp_entity 未找到启用实体:{entityCode}");
var source = await _db.Queryable()
.Where(x => x.Id == entity.SourceId && x.Status == 1)
.FirstAsync(cancellationToken)
?? throw new InvalidOperationException($"mdp_source id={entity.SourceId} 未找到或未启用");
if (string.IsNullOrWhiteSpace(ctx.BatchId))
ctx.BatchId = $"MDP_PULL_{DateTime.Now:yyyyMMddHHmmss}";
await ApplyScheduleWindowAsync(ctx, cancellationToken);
MdpSyncWindowResolver.Apply(ctx);
IMdpSourcePullExecutor executor =
!string.IsNullOrWhiteSpace(entity.SourceApiPath)
|| string.Equals(source.SourceType, "API", StringComparison.OrdinalIgnoreCase)
? _apiExecutor
: _dbExecutor;
return await executor.PullAsync(source, entity, ctx, cancellationToken);
}
///
/// 分页抽尽:FullRefresh 用 OFFSET;增量依赖实体 LastCursor 在页间推进。
///
public async Task PullAllByEntityCodeAsync(
string entityCode,
MdpPullContext ctx,
CancellationToken cancellationToken = default,
int maxPages = 500)
{
var totalPulled = 0;
var totalWritten = 0;
string? lastCursor = null;
string? lastMessage = null;
ctx.Offset = 0;
for (var page = 0; page < maxPages; page++)
{
cancellationToken.ThrowIfCancellationRequested();
var pageResult = await PullByEntityCodeAsync(entityCode, ctx, cancellationToken);
totalPulled += pageResult.RowsPulled;
totalWritten += pageResult.RowsWritten;
lastCursor = pageResult.NewCursor ?? lastCursor;
lastMessage = pageResult.Message;
var entity = await _db.Queryable()
.Where(x => x.EntityCode == entityCode && x.Status == 1)
.FirstAsync(cancellationToken);
var batchSize = entity?.BatchSize > 0 ? entity.BatchSize : 1000;
if (pageResult.RowsPulled <= 0 || pageResult.RowsPulled < batchSize)
break;
if (ctx.FullRefresh)
ctx.Offset += pageResult.RowsPulled;
// 增量:实体 LastCursor 已在执行器内更新,下一页自动收窄
}
return new MdpPullResult
{
RowsPulled = totalPulled,
RowsWritten = totalWritten,
NewCursor = lastCursor,
Message = $"OK pages pulled={totalPulled} written={totalWritten}; {lastMessage}"
};
}
/// 若上下文带 TaskCode 且未显式指定窗口,则从 mdp_sync_task_schedule 读取。
private async Task ApplyScheduleWindowAsync(MdpPullContext ctx, CancellationToken cancellationToken)
{
if (ctx.FullRefresh) return;
if (string.IsNullOrWhiteSpace(ctx.TaskCode)) return;
if (!string.IsNullOrWhiteSpace(ctx.SyncWindowType)) return;
var schedules = await _db.Queryable()
.Where(x => x.TaskCode == ctx.TaskCode && (x.TenantId == ctx.TenantId || x.TenantId == 0))
.OrderByDescending(x => x.TenantId)
.Take(1)
.ToListAsync(cancellationToken);
var schedule = schedules.FirstOrDefault();
if (schedule == null) return;
ctx.SyncWindowType = schedule.SyncWindowType;
ctx.SyncWindowValue = schedule.SyncWindowValue;
}
}