Quellcode durchsuchen

feat(mdp): 按实体立即拉取端点与前端按钮; fix(simulator): mock-api 启动端口占用误报 500; chore: bump web 2.4.416 / server 1.0.598

后端
- MdpSyncEntityConfigService 新增 POST api/DataPlatform/sync-tasks/{taskCode}/entities/{entityCode}/pull。
  配置层自定义实体此前没有运行时入口:MdpModuleStagingPuller 的实体清单全部由调用方显式传入,
  各模块只传自己硬编码的那一份,登记完也拉不动。
- 端点不透传 entityCode,先按「归属本任务 + 当前租户可见 + 已启用」三条件校验,收敛越权面;
  tenantId<=0 提前拦截,避免贴源租户解析回落 null 导致整行静默跳过、对外表现为「拉取成功但 0 行」。
- 回传 effectiveWindowType / effectiveFullRefresh 实际生效值:调度窗口为 FULL 时后端会强制全量,
  不按请求值谎报本次是增量还是全量。
- 契约测试新增 ManualEntityPull_IsTaskScoped_AndTenantGuarded,在源码层钉住上述收口条件。

前端
- syncTasks.ts 新增 pullEntityNow 与 MdpEntityPullResult。
- SyncTaskEntityDrawer.vue 操作列加「立即拉取」:@click.stop 阻断表格行点击冒泡;
  pullingId 单行 loading 防并发重复打出多个 MANUAL_ 批次;源表名与 API 路径皆空时置灰
  (后端标识符硬校验必然失败);结果弹窗独立 try,用户按 ESC 关闭不被误报为拉取失败。

模拟器
- app.py 修复 POST /api/mock-api/start 在端口被占用时返回 HTTP 500:uvicorn 0.54.0 的 Server
  已无 startup_error 属性,访问即 AttributeError。改为真实端口探测 _port_in_use() +
  轮询 server.started / should_exit,占用时如实返回 running=false 与错误原因。
- 新增 tests/test_mock_api_lifecycle.py(2 用例,真实端口启停生命周期回归守卫;
  既有测试全走进程内 TestClient,覆盖不到绑定路径),README §7 测试清单同步。

联调取证
DB_SYNC 通道已全链路打通:实体 SIM_ITEM_DB 经该端点拉取返回 200,rowsPulled=1 / rowsWritten=1,
贴源表 mdp_stg_sim_sim_item 落行字段与源表 sim_item 逐字段一致(含中文未损坏),
重复拉取行数恒为 1、id 不变,幂等成立。
Pengxy vor 2 Tagen
Ursprung
Commit
01ae6913dd

+ 1 - 1
Web/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "admin.net",
 	"type": "module",
-	"version": "2.4.415",
+	"version": "2.4.416",
 	"packageManager": "pnpm@10.32.1",
 	"lastBuildTime": "2026.03.15",
 	"description": "Admin.NET 站在巨人肩膀上的 .NET 通用权限开发框架",

+ 29 - 0
Web/src/views/aidop/data-platform/api/syncTasks.ts

@@ -206,6 +206,35 @@ export function deleteSyncEntity(id: number) {
 	return service.delete<{ id: number }>(`/api/DataPlatform/entities/${id}`).then((r) => r.data);
 }
 
+/** 立即拉取一次的返回。requested* 是请求值,effective* 才是后端实际生效值。 */
+export interface MdpEntityPullResult {
+	entityCode: string;
+	targetTable?: string | null;
+	batchId: string;
+	requestedFullRefresh: boolean;
+	effectiveWindowType?: string | null;
+	effectiveFullRefresh: boolean;
+	rowsPulled: number;
+	rowsWritten: number;
+	newCursor?: string | null;
+	message?: string | null;
+}
+
+/**
+ * 按实体立即拉取一次(源 → 贴源)。
+ *
+ * fullRefresh 只是「请求值」:任务调度窗口为 FULL 时后端会强制全量,
+ * 因此界面必须以返回的 effectiveFullRefresh / effectiveWindowType 为准展示,
+ * 不能按请求值宣称本次是增量还是全量。
+ */
+export function pullEntityNow(taskCode: string, entityCode: string, fullRefresh = false) {
+	return service
+		.post<MdpEntityPullResult>(
+			`/api/DataPlatform/sync-tasks/${encodeURIComponent(taskCode)}/entities/${encodeURIComponent(entityCode)}/pull?fullRefresh=${fullRefresh}`
+		)
+		.then((r) => r.data);
+}
+
 export function fetchEntityFieldMappings(entityId: number) {
 	return service.get<MdpFieldMappingRow[]>(`/api/DataPlatform/entities/${entityId}/field-mappings`).then((r) => r.data);
 }

+ 52 - 2
Web/src/views/aidop/data-platform/components/SyncTaskEntityDrawer.vue

@@ -17,8 +17,9 @@
 					<el-tag :type="row.status === 1 ? 'success' : 'info'" size="small">{{ row.status === 1 ? '启用' : '停用' }}</el-tag>
 				</template>
 			</el-table-column>
-			<el-table-column label="操作" width="200" fixed="right" align="center">
+			<el-table-column label="操作" width="260" fixed="right" align="center">
 				<template #default="{ row }">
+					<el-button link type="primary" :loading="pullingId === row.id" :disabled="!canPull(row)" @click.stop="onPull(row)">立即拉取</el-button>
 					<el-button link type="primary" @click.stop="openMappings(row)">字段映射</el-button>
 					<el-button link type="primary" @click.stop="openEdit(row)">编辑</el-button>
 					<el-button link type="danger" :disabled="isProtected(row)" @click.stop="confirmDelete(row)">删除</el-button>
@@ -74,14 +75,16 @@
 </template>
 
 <script setup lang="ts">
-import { computed, reactive, ref, watch } from 'vue';
+import { computed, h, reactive, ref, watch } from 'vue';
 import { ElMessage, ElMessageBox, type FormInstance, type FormRules } from 'element-plus';
 import FieldMappingEditor from './FieldMappingEditor.vue';
 import {
 	createSyncTaskEntity,
 	deleteSyncEntity,
 	fetchSyncTaskEntities,
+	pullEntityNow,
 	updateSyncEntity,
+	type MdpEntityPullResult,
 	type MdpEntityRow,
 	type MdpEntityUpsertInput,
 	type MdpSyncTaskRow,
@@ -99,6 +102,8 @@ const emit = defineEmits<{
 
 const loading = ref(false);
 const saving = ref(false);
+// 单行级 loading:只让被点的那一行按钮转圈,不禁用整个表格
+const pullingId = ref<number | null>(null);
 const rows = ref<MdpEntityRow[]>([]);
 const formVisible = ref(false);
 const formMode = ref<'create' | 'edit'>('create');
@@ -232,6 +237,51 @@ function onMappingSaved() {
 	emit('saved');
 }
 
+function canPull(row: MdpEntityRow) {
+	// 源表名与 API 路径都为空时,后端 MdpDbPullExecutor.PullAsync 的标识符硬校验必然失败,
+	// 与其让用户点了才报错,不如提前置灰。
+	return Boolean(row.sourceTableName?.trim() || row.sourceApiPath?.trim());
+}
+
+async function onPull(row: MdpEntityRow) {
+	if (!props.task || pullingId.value != null) return;
+	pullingId.value = row.id;
+	let res: MdpEntityPullResult | null = null;
+	try {
+		res = await pullEntityNow(props.task.taskCode, row.entityCode);
+	} catch (e: unknown) {
+		ElMessage.error((e as Error)?.message || '拉取失败');
+	} finally {
+		pullingId.value = null;
+	}
+	if (!res) return;
+
+	// 拉取行数与「实际是否全量」分开呈现:调度窗口为 FULL 时后端会把请求的增量强制成全量,
+	// 只报请求值会让用户误以为本次是增量。
+	const lines = [
+		`拉取行数:${res.rowsPulled}`,
+		`写入行数:${res.rowsWritten}`,
+		`请求全量:${res.requestedFullRefresh ? '是' : '否'}`,
+		`实际全量:${res.effectiveFullRefresh ? '是' : '否'}(窗口 ${res.effectiveWindowType || '—'})`,
+		`目标表:${res.targetTable || '—'}`,
+		`批次号:${res.batchId}`,
+		`新游标:${res.newCursor || '—'}`,
+	];
+	if (res.message) lines.push(`后端消息:${res.message}`);
+
+	// 弹窗单独 try:用户按 ESC 关掉弹窗产生的 reject 不能被误报成「拉取失败」
+	try {
+		await ElMessageBox.alert(
+			h('div', { style: 'line-height:1.8;white-space:pre-wrap;word-break:break-all;' }, lines.join('\n')),
+			`实体 ${res.entityCode} 拉取完成`,
+			{ type: 'success', confirmButtonText: '知道了' }
+		);
+	} catch {
+		/* 用户关闭弹窗 */
+	}
+	await loadEntities();
+}
+
 watch(
 	() => [props.visible, props.task?.taskCode] as const,
 	([vis, code]) => {

+ 3 - 3
server/Admin.NET.Web.Entry/Admin.NET.Web.Entry.csproj

@@ -11,9 +11,9 @@
     <GenerateSatelliteAssembliesForCore>true</GenerateSatelliteAssembliesForCore>
     <Copyright>Admin.NET</Copyright>
     <Description>Admin.NET 通用权限开发平台</Description>
-    <AssemblyVersion>1.0.597</AssemblyVersion>
-    <FileVersion>1.0.597</FileVersion>
-    <Version>1.0.597</Version>
+    <AssemblyVersion>1.0.598</AssemblyVersion>
+    <FileVersion>1.0.598</FileVersion>
+    <Version>1.0.598</Version>
   </PropertyGroup>
 
   <ItemGroup>

+ 25 - 0
server/Plugins/Admin.NET.Plugin.AiDOP.Tests/DataPlatform/MdpConfigApiTenantIsolationTests.cs

@@ -58,6 +58,31 @@ public class MdpConfigApiTenantIsolationTests
         Assert.DoesNotContain("FUNC-S9-020", menu, StringComparison.Ordinal);
     }
 
+    /// <summary>
+    /// 手动拉取端点是配置层自定义实体唯一的运行时入口,也是唯一能把 entityCode
+    /// 递到 dispatcher 的口子,因此它的收口条件必须在源码层被钉住。
+    /// </summary>
+    [Fact]
+    public void ManualEntityPull_IsTaskScoped_AndTenantGuarded()
+    {
+        var source = ReadPluginFile("DataPlatform", "MdpSyncEntityConfigService.cs");
+        Assert.Contains("sync-tasks/{taskCode}/entities/{entityCode}/pull", source, StringComparison.Ordinal);
+
+        var method = source
+            .Split("[DisplayName(\"按实体立即拉取一次\")]")[1]
+            .Split("[DisplayName(\"更新同步实体\")]")[0];
+
+        // 无租户上下文必须硬失败:否则贴源写入的租户解析回落到 null,整行被静默跳过,
+        // 对外表现为「拉取成功但 0 行」。
+        Assert.Contains("tenantId <= 0", method, StringComparison.Ordinal);
+        Assert.Contains("无法解析当前租户", method, StringComparison.Ordinal);
+
+        // entityCode 不得直接透传:必须先按「归属本任务 + 当前租户可见 + 已启用」收口
+        Assert.Contains("u.JobId == jobCode", method, StringComparison.Ordinal);
+        Assert.Contains("(u.TenantId == tenantId || u.TenantId == 0)", method, StringComparison.Ordinal);
+        Assert.Contains("u.Status == 1", method, StringComparison.Ordinal);
+    }
+
     private static string ReadPluginFile(string folder, string file) =>
         ReadRepoFile("server", "Plugins", "Admin.NET.Plugin.AiDOP", folder, file);
 

+ 79 - 1
server/Plugins/Admin.NET.Plugin.AiDOP/DataPlatform/MdpSyncEntityConfigService.cs

@@ -1,3 +1,4 @@
+using Admin.NET.Plugin.AiDOP.DataPlatform.Executors;
 using Admin.NET.Plugin.AiDOP.Dto.DataPlatform;
 using Admin.NET.Plugin.AiDOP.Entity.DataPlatform;
 
@@ -18,11 +19,16 @@ public class MdpSyncEntityConfigService : IDynamicApiController, ITransient
 
     private readonly ISqlSugarClient _db;
     private readonly UserManager _userManager;
+    private readonly MdpSourcePullDispatcher _dispatcher;
 
-    public MdpSyncEntityConfigService(ISqlSugarClient db, UserManager userManager)
+    public MdpSyncEntityConfigService(
+        ISqlSugarClient db,
+        UserManager userManager,
+        MdpSourcePullDispatcher dispatcher)
     {
         _db = db;
         _userManager = userManager;
+        _dispatcher = dispatcher;
     }
 
     [DisplayName("同步任务实体列表")]
@@ -117,6 +123,78 @@ public class MdpSyncEntityConfigService : IDynamicApiController, ITransient
         return new { id };
     }
 
+    /// <summary>
+    /// 按实体立即拉取一次(源 → 贴源)。
+    ///
+    /// <para><b>为什么需要它</b>:<see cref="Executors.MdpModuleStagingPuller"/> 的实体清单全部由调用方
+    /// 显式传入,各模块只传自己硬编码的那一份(<c>S1MdpEntityConfig.All</c> 等),配置层新增的
+    /// 自定义实体没有任何运行时入口,登记完也拉不动。这里补一个受控的通用触发口。</para>
+    ///
+    /// <para><b>越权面收敛</b>:不把 entityCode 直接透传给 dispatcher,而是先按「归属本任务 +
+    /// 当前租户可见 + 已启用」三条件校验,避免借该端点拉取任意实体。</para>
+    /// </summary>
+    [DisplayName("按实体立即拉取一次")]
+    [HttpPost("sync-tasks/{taskCode}/entities/{entityCode}/pull")]
+    public async Task<object> PullEntityNow(
+        string taskCode,
+        string entityCode,
+        [FromQuery] bool fullRefresh = false,
+        CancellationToken cancellationToken = default)
+    {
+        var tenantId = _userManager.TenantId;
+        // tenantId<=0 时贴源写入的租户解析会回落到 null,整行被静默跳过(只打 Warning),
+        // 对外表现为「拉取成功但 0 行」。提前挡住,不给静默失败留口子。
+        if (tenantId <= 0)
+            throw Oops.Oh("无法解析当前租户,请在具体租户下执行拉取");
+
+        var task = await FindTaskAsync(taskCode, tenantId);
+        var jobCode = string.IsNullOrWhiteSpace(task.JobCode) ? task.TaskCode : task.JobCode;
+        var code = entityCode.Trim();
+
+        var target = await _db.Queryable<MdpEntity>()
+            .Where(u => u.EntityCode == code
+                        && u.Status == 1
+                        && u.JobId == jobCode
+                        && (u.TenantId == tenantId || u.TenantId == 0))
+            .FirstAsync(cancellationToken)
+            ?? throw Oops.Oh($"实体 {code} 不存在、未启用,或不归属任务 {taskCode}");
+
+        // sync_batch_id 是 varchar(64):固定段 22 字符 + 实体编码最多截 42 字符,
+        // 截尾只落在实体编码上,不动时间戳,避免批次号互相撞。
+        var codePart = code.Length > 42 ? code[..42] : code;
+        var ctx = new MdpPullContext
+        {
+            TenantId = tenantId,
+            FullRefresh = fullRefresh,
+            BatchId = $"MANUAL_{DateTime.Now:yyyyMMddHHmmss}_{codePart}",
+            TaskCode = task.TaskCode
+        };
+
+        try
+        {
+            var result = await _dispatcher.PullAllByEntityCodeAsync(code, ctx, cancellationToken);
+            return new
+            {
+                entityCode = code,
+                targetTable = target.TargetTableName,
+                batchId = ctx.BatchId,
+                requestedFullRefresh = fullRefresh,
+                // 调度窗口可能把 FULL 落成 ctx.FullRefresh=true,回传实际生效值,不做黑盒
+                effectiveWindowType = ctx.SyncWindowType,
+                effectiveFullRefresh = ctx.FullRefresh,
+                rowsPulled = result.RowsPulled,
+                rowsWritten = result.RowsWritten,
+                newCursor = result.NewCursor,
+                message = result.Message
+            };
+        }
+        catch (InvalidOperationException ex)
+        {
+            // 贴源契约缺列、源表标识符非法、源不可达等都走这里;转业务异常把原因如实带回前端
+            throw Oops.Oh(ex.Message);
+        }
+    }
+
     [DisplayName("更新同步实体")]
     [HttpPut("entities/{id:long}")]
     public async Task<object> UpdateEntity(long id, [FromBody] MdpEntityUpsertInput input)

+ 2 - 1
tools/integration-simulator/README.md

@@ -103,11 +103,12 @@ $env:AIDOP_SIM_ADMIN_TOKEN        = "<可选:只读管理 API 用>"
 | `tests/test_inbound_signature.py` | 签名金标向量(与独立实现交叉验证)、SHA256/HMAC、签名头不外泄 |
 | `tests/test_sample_adapters.py` | 目录三通道完整性、26 契约覆盖、样例路径存在、`SIM-{runId}-` 业务键、契约字段名、缺必填如实上报 |
 | `tests/test_mock_api.py` | cursor 过滤、四种鉴权、401/403/429/500 注入、响应路径切换、样例挂载、审计脱敏 |
+| `tests/test_mock_api_lifecycle.py` | 真实端口启停生命周期:绑定成功且可连通、重复调用幂等、端口占用如实报错(回归 uvicorn 0.30+ 无 `startup_error` 导致的 500) |
 | `tests/test_mock_db.py` | DDL 与类型推断、幂等 upsert、库名硬阻断、配置清单;真实 MySQL 用例无密码时跳过并标注未实测 |
 | `tests/test_guard.py` | 非白名单目标阻断、预检状态分类、运行记录脱敏与容量 |
 | `tests/test_redaction.py` | Secret/连接串/Bearer/AccessKey 后四位脱敏 |
 
-> `tests/test_guard.py` 是任务书初始清单外新增的一份守卫测试(安全闸门与三层判定),其余文件名与任务书一致。
+> `tests/test_guard.py`(安全闸门与三层判定)与 `tests/test_mock_api_lifecycle.py`(真实端口启停)是任务书初始清单外新增的守卫测试,其余文件名与任务书一致。
 
 ## 8. 目录
 

+ 39 - 7
tools/integration-simulator/app.py

@@ -13,6 +13,7 @@ from __future__ import annotations
 
 import asyncio
 import os
+import socket
 import sys
 import time
 import uuid
@@ -121,18 +122,49 @@ class _MockServerHandle:
 MOCK_HANDLE = _MockServerHandle()
 
 
+def _port_in_use(port: int) -> bool:
+    """真实探测本机端口是否已被占用。
+
+    不依赖 uvicorn 内部属性:0.30+ 的 Server 已无 startup_error,
+    读它会抛 AttributeError 并把已启动的服务误报成 500。
+    """
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
+        probe.settimeout(0.3)
+        return probe.connect_ex(("127.0.0.1", int(port))) == 0
+
+
 async def _start_mock_api() -> dict:
+    port = int(CONFIG["mockApiPort"])
     if MOCK_HANDLE.task and not MOCK_HANDLE.task.done():
-        return {"running": True, "port": CONFIG["mockApiPort"], "alreadyRunning": True}
-    config = uvicorn.Config(MOCK_APP, host="127.0.0.1", port=int(CONFIG["mockApiPort"]), log_level="warning")
+        return {"running": True, "port": port, "alreadyRunning": True}
+
+    # 端口预检必须在建 task 之前:否则失败路径会留下一个已赋值的句柄,
+    # 使 _mock_running() 与实际监听状态不一致。
+    if _port_in_use(port):
+        MOCK_HANDLE.last_error = f"端口 {port} 已被占用(可能是另一个模拟器实例)"
+        return {"running": False, "error": MOCK_HANDLE.last_error}
+
+    config = uvicorn.Config(MOCK_APP, host="127.0.0.1", port=port, log_level="warning")
     server = uvicorn.Server(config)
     MOCK_HANDLE.server = server
     MOCK_HANDLE.task = asyncio.get_running_loop().create_task(server.serve())
-    await asyncio.sleep(0.4)
-    if server.startup_error:
-        MOCK_HANDLE.last_error = str(server.startup_error)
-        return {"running": False, "error": MOCK_HANDLE.last_error}
-    return {"running": True, "port": CONFIG["mockApiPort"]}
+
+    # 轮询库的公开状态位,上限 ~2s:比固定 sleep 既不会过早返回未就绪,也不会白等。
+    for _ in range(40):
+        await asyncio.sleep(0.05)
+        if server.started:
+            MOCK_HANDLE.last_error = ""
+            return {"running": True, "port": port}
+        if server.should_exit or MOCK_HANDLE.task.done():
+            break
+
+    MOCK_HANDLE.last_error = f"Mock API 在端口 {port} 上启动失败"
+    failed_task = MOCK_HANDLE.task
+    MOCK_HANDLE.server = None
+    MOCK_HANDLE.task = None
+    if failed_task and not failed_task.done():
+        failed_task.cancel()
+    return {"running": False, "error": MOCK_HANDLE.last_error}
 
 
 async def _stop_mock_api() -> dict:

+ 73 - 0
tools/integration-simulator/tests/test_mock_api_lifecycle.py

@@ -0,0 +1,73 @@
+"""Mock API 内嵌启停的真实端口生命周期回归(缺陷回归守卫)。
+
+背景:uvicorn 0.30+ 的 Server 已无 startup_error 属性,旧实现访问它会抛 AttributeError,
+使 POST /api/mock-api/start 恒定返回 500(而服务其实已启动),端口被占用时也无法如实报错。
+既有测试全部走进程内 TestClient,覆盖不到真实绑定路径——本文件用真实端口补齐该盲区。
+"""
+from __future__ import annotations
+
+import asyncio
+import socket
+
+import pytest
+
+import app  # noqa: E402  conftest 已把模拟器根目录加入 sys.path
+
+
+def _free_port() -> int:
+    """取一个当前空闲的端口(绑定 0 由 OS 分配后立即释放)。"""
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(("127.0.0.1", 0))
+        return s.getsockname()[1]
+
+
+@pytest.fixture()
+def mock_port(monkeypatch):
+    """把 mockApiPort 指向空闲端口;启停在用例内完成,夹具只复位全局句柄。"""
+    port = _free_port()
+    monkeypatch.setitem(app.CONFIG, "mockApiPort", port)
+    yield port
+    app.MOCK_HANDLE.server = None
+    app.MOCK_HANDLE.task = None
+    app.MOCK_HANDLE.last_error = ""
+
+
+def test_start_binds_real_port_and_is_idempotent(mock_port):
+    """启动成功须返回 running=true、端口真实可连通、重复调用幂等,且不抛异常(不再 500)。"""
+
+    async def scenario():
+        result = await app._start_mock_api()
+        try:
+            assert result["running"] is True, result
+            assert result["port"] == mock_port
+            assert app._mock_running() is True
+            assert app.MOCK_HANDLE.last_error == ""
+            # 端口必须真实可连通,不能只是内部状态为真
+            with socket.create_connection(("127.0.0.1", mock_port), timeout=2):
+                pass
+            again = await app._start_mock_api()
+            assert again.get("alreadyRunning") is True, again
+        finally:
+            await app._stop_mock_api()
+
+    asyncio.run(scenario())
+    assert app._mock_running() is False
+
+
+def test_start_on_occupied_port_reports_error_not_raise(mock_port):
+    """端口被占用须返回 {running:false, error} 并写入 lastError,而不是抛 AttributeError。"""
+    blocker = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    blocker.bind(("127.0.0.1", mock_port))
+    blocker.listen(1)
+
+    async def scenario():
+        try:
+            return await app._start_mock_api()
+        finally:
+            blocker.close()
+
+    result = asyncio.run(scenario())
+    assert result["running"] is False, result
+    assert str(mock_port) in result["error"]
+    assert app.MOCK_HANDLE.last_error  # /api/status 的 lastError 诊断字段恢复有效
+    assert app._mock_running() is False