AiDOPWarehouse/tools/integration-simulator/providers/mock_db.py

"""DB_SYNC 模拟源库（任务书 P1-C）。

职责边界：
- 只在独立 MySQL 模拟 schema（默认 aidop_integration_sim）建表、装载、清理；
- 绝不连接 Ai-DOP 库写贴源/标准/KPI 表；库名含 aidop 且非模拟库名时硬阻断；
- 不使用 Ai-DOP 主库账号；连接凭据从环境变量读取（密码 AIDOP_SIM_MYSQL_PASSWORD）；
- 生成 Ai-DOP 配置清单（mdp_source / mdp_entity 建议行），仅供人工在配置页登记，不自动写入。
"""
from __future__ import annotations

import json
import os
from datetime import datetime

# 允许创建的模拟库名；其余含 aidop 的库名一律阻断
SIM_DB_DEFAULT = "aidop_integration_sim"
_BLOCKED_DB_HINTS = ("aidopdev", "aidop_uat", "aidopprod", "aidop_prod", "admin", "mdp")


class BlockedDatabaseError(RuntimeError):
    pass


def assert_sim_database(database: str) -> None:
    db = (database or "").strip().lower()
    if not db:
        raise BlockedDatabaseError("database name required")
    if db != SIM_DB_DEFAULT and any(h in db for h in _BLOCKED_DB_HINTS):
        raise BlockedDatabaseError(
            f"database '{database}' looks like an Ai-DOP database; simulator may only use '{SIM_DB_DEFAULT}'")
    if db != SIM_DB_DEFAULT and not db.startswith("aidop_integration_sim"):
        # 允许 aidop_integration_sim_* 派生测试库，其余含 aidop 的阻断
        if "aidop" in db:
            raise BlockedDatabaseError(
                f"database '{database}' contains 'aidop' but is not the simulator schema")


def mysql_conn_params(database: str | None = None) -> dict:
    db = database or os.environ.get("AIDOP_SIM_MYSQL_DATABASE", SIM_DB_DEFAULT)
    assert_sim_database(db)
    return {
        "host": os.environ.get("AIDOP_SIM_MYSQL_HOST", "127.0.0.1"),
        "port": int(os.environ.get("AIDOP_SIM_MYSQL_PORT", "3306")),
        "user": os.environ.get("AIDOP_SIM_MYSQL_USER", "root"),
        "password": os.environ.get("AIDOP_SIM_MYSQL_PASSWORD", ""),
        "database": db,
        "charset": "utf8mb4",
    }


def connect(database: str | None = None):
    import pymysql  # 延迟导入：单元测试不依赖真实驱动
    params = mysql_conn_params(database)
    return pymysql.connect(cursorclass=pymysql.cursors.DictCursor, autocommit=True, **params)


# ---------------------------------------------------------------------------
# DDL 生成（纯函数，可单测）
# ---------------------------------------------------------------------------

def infer_mysql_type(values: list) -> str:
    has_text = has_int = has_float = has_bool = False
    max_len = 0
    for v in values:
        if v is None:
            continue
        if isinstance(v, bool):
            has_bool = True
        elif isinstance(v, int):
            has_int = True
        elif isinstance(v, float):
            has_float = True
        else:
            has_text = True
            max_len = max(max_len, len(str(v)))
    if has_text:
        return f"VARCHAR({max(64, min(512, (max_len // 32 + 1) * 32))})"
    if has_bool:
        return "TINYINT(1)"
    if has_float:
        return "DECIMAL(18,6)"
    if has_int:
        return "BIGINT"
    return "VARCHAR(191)"


def generate_ddl(table: str, rows: list[dict], business_key: list[str] | None = None,
                 increment_column: str = "sourceUpdatedAt") -> str:
    """按样例行推断列类型；业务键唯一索引；增量列建普通索引。列名保持样例原样（反引号包裹）。"""
    if not rows:
        raise ValueError(f"cannot generate DDL for {table} from empty rows")
    business_key = business_key or ["bizKey"]
    columns: dict[str, list] = {}
    for row in rows:
        for k, v in row.items():
            columns.setdefault(k, []).append(v)
    lines = []
    for name, values in columns.items():
        lines.append(f"  `{name}` {infer_mysql_type(values)} NULL")
    uk = ", ".join(f"`{k}`" for k in business_key if k in columns)
    if uk:
        lines.append(f"  UNIQUE KEY `uk_{table}` ({uk})")
    if increment_column in columns:
        lines.append(f"  KEY `ix_{table}_incr` (`{increment_column}`)")
    return (f"CREATE TABLE IF NOT EXISTS `{table}` (\n" + ",\n".join(lines) +
            "\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4")


# ---------------------------------------------------------------------------
# 库操作
# ---------------------------------------------------------------------------

def _col_names(rows: list[dict]) -> list[str]:
    cols: list[str] = []
    for row in rows:
        for k in row:
            if k not in cols:
                cols.append(k)
    return cols


def _insert_sql(table: str, cols: list[str]) -> str:
    names = ", ".join(f"`{c}`" for c in cols)
    marks = ", ".join(["%s"] * len(cols))
    updates = ", ".join(f"`{c}`=VALUES(`{c}`)" for c in cols)
    return f"INSERT INTO `{table}` ({names}) VALUES ({marks}) ON DUPLICATE KEY UPDATE {updates}"


def _convert(v):
    if isinstance(v, (dict, list)):
        return json.dumps(v, ensure_ascii=False)
    if isinstance(v, bool):
        return int(v)
    return v


def init_schema(tables: dict[str, dict]) -> dict:
    """tables: {table: {"rows": [...], "businessKey": [...], "incrementColumn": str}}。建库建表。"""
    result = {"database": None, "tables": []}
    import pymysql
    params = mysql_conn_params()
    database = params.pop("database")
    conn = pymysql.connect(cursorclass=pymysql.cursors.DictCursor, autocommit=True, **params)
    try:
        with conn.cursor() as cur:
            cur.execute(f"CREATE DATABASE IF NOT EXISTS `{database}` DEFAULT CHARACTER SET utf8mb4")
            cur.execute(f"USE `{database}`")
            result["database"] = database
            for table, spec in tables.items():
                ddl = generate_ddl(table, spec["rows"], spec.get("businessKey"),
                                   spec.get("incrementColumn", "sourceUpdatedAt"))
                cur.execute(ddl)
                result["tables"].append(table)
    finally:
        conn.close()
    return result


def _use_db():
    import pymysql
    params = mysql_conn_params()
    return pymysql.connect(cursorclass=pymysql.cursors.DictCursor, autocommit=True, **params)


def reset_table(table: str) -> None:
    conn = _use_db()
    try:
        with conn.cursor() as cur:
            cur.execute(f"TRUNCATE TABLE `{table}`")
    finally:
        conn.close()


def upsert_rows(table: str, rows: list[dict]) -> int:
    if not rows:
        return 0
    cols = _col_names(rows)
    sql = _insert_sql(table, cols)
    conn = _use_db()
    try:
        with conn.cursor() as cur:
            cur.executemany(sql, [tuple(_convert(row.get(c)) for c in cols) for row in rows])
            return len(rows)
    finally:
        conn.close()


def update_rows(table: str, updates: dict, where: dict,
                increment_column: str = "sourceUpdatedAt",
                new_increment: str | None = None) -> int:
    """更新已有数据并推进增量列（DB_SYNC 增量更新用例）。"""
    sets = {**updates}
    if increment_column:
        sets[increment_column] = new_increment or datetime.now().strftime("%Y-%m-%dT%H:%M:%S")
    set_clause = ", ".join(f"`{k}`=%s" for k in sets)
    where_clause = " AND ".join(f"`{k}`=%s" for k in where)
    sql = f"UPDATE `{table}` SET {set_clause} WHERE {where_clause}"
    conn = _use_db()
    try:
        with conn.cursor() as cur:
            cur.execute(sql, tuple(sets.values()) + tuple(where.values()))
            return cur.rowcount
    finally:
        conn.close()


def table_stats(table: str, increment_column: str = "sourceUpdatedAt") -> dict:
    conn = _use_db()
    try:
        with conn.cursor() as cur:
            cur.execute(f"SELECT COUNT(*) AS cnt, MAX(`{increment_column}`) AS max_incr FROM `{table}`")
            row = cur.fetchone() or {}
            return {"table": table, "rowCount": row.get("cnt", 0),
                    "maxIncrement": str(row.get("max_incr") or "")}
    finally:
        conn.close()


# ---------------------------------------------------------------------------
# Ai-DOP 配置清单生成（只输出建议，不写库）
# ---------------------------------------------------------------------------

SIM_SYSTEM_CODE = "SIM_MODE3"
SIM_SOURCES = {
    "DB_SYNC": "SIM_MODE3_DB",
    "API_PULL": "SIM_MODE3_API",
    "API_INBOUND": "SIM_MODE3_INBOUND",
}


def generate_aidop_config(obj: dict, mock_api_base_url: str = "http://127.0.0.1:8018") -> dict:
    """按对象生成 mdp_source / mdp_entity 建议配置，供人工在 Ai-DOP 配置页登记。"""
    db_sync = obj.get("dbSync") or {}
    api_pull = obj.get("apiPull") or {}
    inbound = obj.get("apiInbound") or {}
    config: dict = {"objectCode": obj["objectCode"], "systemCode": SIM_SYSTEM_CODE, "sources": [], "entities": []}
    if db_sync.get("supported"):
        config["sources"].append({
            "source_code": SIM_SOURCES["DB_SYNC"], "source_type": "DB", "conn_mode": "EXTERNAL",
            "hint": "连接指向模拟库 aidop_integration_sim（独立账号，勿用 Ai-DOP 主库账号）",
        })
        config["entities"].append({
            "entity_code": f"SIM_{obj['objectCode']}_DB", "source_code": SIM_SOURCES["DB_SYNC"],
            "source_table_name": db_sync.get("table"), "target_table_name": f"mdp_stg_sim_{db_sync.get('table')}",
            "biz_key_expr": ",".join(db_sync.get("businessKey") or ["bizKey"]),
            "incr_column": db_sync.get("incrementColumn"),
        })
    if api_pull.get("supported"):
        config["sources"].append({
            "source_code": SIM_SOURCES["API_PULL"], "source_type": "API",
            "api_base_url": mock_api_base_url, "api_auth_type": "TOKEN",
            "hint": "token 值即 Mock 控制台当前 TOKEN（环境变量/控制台设置，不落盘）",
        })
        config["entities"].append({
            "entity_code": f"SIM_{obj['objectCode']}_API", "source_code": SIM_SOURCES["API_PULL"],
            "source_api_path": api_pull.get("path"), "target_table_name": f"mdp_stg_sim_{db_sync.get('table')}",
            "response_data_path": api_pull.get("responsePath"), "dedup_key_path": api_pull.get("dedupKeyPath"),
            "biz_key_expr": ",".join(db_sync.get("businessKey") or ["bizKey"]),
            "incr_column": "cursor",
        })
    if inbound.get("supported"):
        config["sources"].append({
            "source_code": SIM_SOURCES["API_INBOUND"], "source_type": "API_INBOUND",
            "hint": "需在 mdp_inbound_grant 为测试 AccessKey 授权该 entityCode，并绑定测试租户开放身份",
        })
        config["entities"].append({
            "entity_code": inbound.get("entityCode"), "source_code": SIM_SOURCES["API_INBOUND"],
            "inbound_enabled": 1,
        })
    return config