using System.Data; using System.Text.Json; using Admin.NET.Plugin.AiDOP.Entity.DataPlatform; using SqlSugar; namespace Admin.NET.Plugin.AiDOP.DataPlatform.Executors; /// /// 方式甲:从 mdp_source DB 源按 mdp_entity.source_table_name 增量抽数 → target_table_name(贴源)。 /// public sealed class MdpDbPullExecutor : IMdpSourcePullExecutor, ITransient { public string SupportedType => "DB_SYNC"; private readonly MdpSourceScopeFactory _scopeFactory; private readonly ISqlSugarClient _db; private readonly MdpStagingWriter _writer; public MdpDbPullExecutor(MdpSourceScopeFactory scopeFactory, ISqlSugarClient db, MdpStagingWriter writer) { _scopeFactory = scopeFactory; _db = db; _writer = writer; } public async Task PullAsync(MdpSource source, MdpEntity entity, MdpPullContext ctx, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(entity.SourceTableName)) throw new InvalidOperationException($"实体 {entity.EntityCode} 未配置 source_table_name"); if (string.IsNullOrWhiteSpace(entity.TargetTableName)) throw new InvalidOperationException($"实体 {entity.EntityCode} 未配置 target_table_name"); var scope = await _scopeFactory.GetScopeAsync(source.SourceCode, cancellationToken); var batchSize = entity.BatchSize > 0 ? entity.BatchSize : 1000; var isSqlServer = string.Equals(source.DbType, "SQLSERVER", StringComparison.OrdinalIgnoreCase) || string.Equals(source.DbType, "MSSQL", StringComparison.OrdinalIgnoreCase); var sql = BuildSelectSql(entity, isSqlServer, batchSize, ctx); var parameters = new List(); if (ctx.WindowFrom.HasValue) parameters.Add(new SugarParameter("@windowFrom", ctx.WindowFrom.Value)); if (!ctx.FullRefresh && !string.Equals(ctx.SyncWindowType, "FULL", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(entity.IncrColumn) && !string.IsNullOrWhiteSpace(entity.LastCursor) && !string.Equals(ctx.SyncWindowType, "ROLLING", StringComparison.OrdinalIgnoreCase)) parameters.Add(new SugarParameter("@cursor", entity.LastCursor)); // ROLLING:用窗口下界;若已有水位且水位晚于下界,则进一步用水位收窄 if (!ctx.FullRefresh && string.Equals(ctx.SyncWindowType, "ROLLING", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(entity.IncrColumn) && !string.IsNullOrWhiteSpace(entity.LastCursor) && ctx.WindowFrom.HasValue && DateTime.TryParse(entity.LastCursor, out var cursorDt) && cursorDt > ctx.WindowFrom.Value) parameters.Add(new SugarParameter("@cursor", entity.LastCursor)); var table = await scope.Ado.GetDataTableAsync(sql, parameters); cancellationToken.ThrowIfCancellationRequested(); var written = 0; string? maxCursor = entity.LastCursor; var now = DateTime.Now; foreach (DataRow row in table.Rows) { cancellationToken.ThrowIfCancellationRequested(); var dict = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (DataColumn col in table.Columns) dict[col.ColumnName] = row[col] == DBNull.Value ? null : row[col]; var sourceRowId = ResolveSourceRowId(dict); var rawJson = JsonSerializer.Serialize(dict); if (!string.IsNullOrWhiteSpace(entity.IncrColumn) && dict.TryGetValue(entity.IncrColumn, out var incrVal) && incrVal != null) { var cursor = incrVal is DateTime dt ? dt.ToString("yyyy-MM-dd HH:mm:ss.fff") : incrVal.ToString(); if (!string.IsNullOrEmpty(cursor) && (maxCursor == null || string.CompareOrdinal(cursor, maxCursor) > 0)) maxCursor = cursor; } written += await _writer.UpsertAsync( source, entity, entity.SourceTableName!, dict, rawJson, sourceRowId, ctx); } if (!string.IsNullOrEmpty(maxCursor) && maxCursor != entity.LastCursor) { await _db.Updateable() .SetColumns(x => new MdpEntity { LastCursor = maxCursor, LastSyncTo = now, UpdateTime = now }) .Where(x => x.Id == entity.Id) .ExecuteCommandAsync(cancellationToken); } await WriteSyncLogAsync(entity, ctx, table.Rows.Count, written, null); var windowHint = ctx.SyncWindowType ?? (ctx.FullRefresh ? "FULL" : "INCR"); return new MdpPullResult { RowsPulled = table.Rows.Count, RowsWritten = written, NewCursor = maxCursor, Message = $"OK window={windowHint}" + (ctx.WindowFrom.HasValue ? $" from={ctx.WindowFrom:yyyy-MM-dd HH:mm:ss}" : "") }; } private static string BuildSelectSql(MdpEntity entity, bool isSqlServer, int batchSize, MdpPullContext ctx) { var table = entity.SourceTableName!.Trim(); // 仅允许简单标识符/schema.table,防注入 if (!System.Text.RegularExpressions.Regex.IsMatch(table, @"^[A-Za-z0-9_\.\[\]]+$")) throw new InvalidOperationException($"非法 source_table_name:{table}"); var predicates = new List(); string orderBy; if (!string.IsNullOrWhiteSpace(entity.IncrColumn)) { var incr = entity.IncrColumn.Trim(); if (!System.Text.RegularExpressions.Regex.IsMatch(incr, @"^[A-Za-z0-9_]+$")) throw new InvalidOperationException($"非法 incr_column:{incr}"); var incrExpr = isSqlServer ? incr : $"`{incr}`"; orderBy = incrExpr; var isRolling = string.Equals(ctx.SyncWindowType, "ROLLING", StringComparison.OrdinalIgnoreCase); if (!ctx.FullRefresh && isRolling && ctx.WindowFrom.HasValue) predicates.Add($"{incrExpr} >= @windowFrom"); var useCursor = !ctx.FullRefresh && !string.IsNullOrWhiteSpace(entity.LastCursor) && (!isRolling || (ctx.WindowFrom.HasValue && DateTime.TryParse(entity.LastCursor, out var cursorDt) && cursorDt > ctx.WindowFrom.Value)); if (useCursor) predicates.Add($"{incrExpr} > @cursor"); } else { orderBy = isSqlServer ? "(SELECT NULL)" : "1"; } var where = predicates.Count > 0 ? " WHERE " + string.Join(" AND ", predicates) : ""; var offset = ctx.Offset > 0 ? ctx.Offset : 0; if (isSqlServer) { // SQL Server:OFFSET 需 ORDER BY;无 incr 时用稳定键兜底 if (string.IsNullOrWhiteSpace(entity.IncrColumn)) orderBy = "(SELECT NULL)"; return $"SELECT * FROM {table}{where} ORDER BY {orderBy} OFFSET {offset} ROWS FETCH NEXT {batchSize} ROWS ONLY"; } return offset > 0 ? $"SELECT * FROM {table}{where} ORDER BY {orderBy} LIMIT {batchSize} OFFSET {offset}" : $"SELECT * FROM {table}{where} ORDER BY {orderBy} LIMIT {batchSize}"; } private static string ResolveSourceRowId(Dictionary dict) { foreach (var key in new[] { "id", "Id", "ID", "RecID", "RecId", "recid", "Recid", "noid", "billno", "BillNo", "djbh", "Djbh" }) { if (dict.TryGetValue(key, out var v) && v != null) return v.ToString() ?? Guid.NewGuid().ToString("N"); } return Guid.NewGuid().ToString("N"); } private async Task WriteSyncLogAsync(MdpEntity entity, MdpPullContext ctx, int pulled, int written, string? error) { try { await _db.Ado.ExecuteCommandAsync(@" INSERT INTO mdp_sync_log (tenant_id, entity_id, entity_code, sync_batch_id, status, source_rows, target_rows, error_message, create_time, update_time) VALUES (@tenantId, @entityId, @entityCode, @batchId, @status, @sourceRows, @targetRows, @error, @now, @now)", new SugarParameter("@tenantId", ctx.TenantId), new SugarParameter("@entityId", entity.Id), new SugarParameter("@entityCode", entity.EntityCode), new SugarParameter("@batchId", ctx.BatchId), new SugarParameter("@status", string.IsNullOrEmpty(error) ? "SUCCESS" : "FAILED"), new SugarParameter("@sourceRows", pulled), new SugarParameter("@targetRows", written), new SugarParameter("@error", error), new SugarParameter("@now", DateTime.Now)); } catch { // 日志表结构可能与种子不一致时不阻断主流程 } } }