增加同步行政区域层级配置，优化同步数据入库性能，效率可提升20倍，同步区县级数据只需30s左右，原来需要600s

Signed-off-by: Lzh666 <422235757@qq.com>

Admin.NET/Admin.NET.Core/Const/CommonConst.cs

@@ -1,4 +1,4 @@
-// Admin.NET 项目的版权、商标、专利和其他相关权利均受相应法律法规的保护。使用本项目应遵守相关法律法规和许可证的要求。
+// Admin.NET 项目的版权、商标、专利和其他相关权利均受相应法律法规的保护。使用本项目应遵守相关法律法规和许可证的要求。
 //
 // 本项目主要遵循 MIT 许可证和 Apache 许可证（版本 2.0）进行分发和使用。许可证位于源代码树根目录中的 LICENSE-MIT 和 LICENSE-APACHE 文件。
 //
@@ -87,6 +87,11 @@ public class CommonConst
     /// </summary>
     public const string SysValidationLog = "sys_validation_log";
 
+    /// <summary>
+    /// 行政区域同步层级 1-省级,2-市级,3-区县级,4-街道级,5-村级
+    /// </summary>
+    public const string SysRegionSyncLevel = "sys_region_sync_level";
+
     /// <summary>
     /// 日志保留天数
     /// </summary>

Admin.NET/Admin.NET.Core/Enum/ErrorCodeEnum.cs

@@ -679,6 +679,24 @@ public enum ErrorCodeEnum
     [ErrorCodeItemMetadata("已有相同编码或名称")]
     R2002,
 
+    /// <summary>
+    /// 行政区代码只能为6、9或12位
+    /// </summary>
+    [ErrorCodeItemMetadata("行政区代码只能为6、9或12位")]
+    R2003,
+
+    /// <summary>
+    /// 父节点不能为自己的子节点
+    /// </summary>
+    [ErrorCodeItemMetadata("父节点不能为自己的子节点")]
+    R2004,
+
+    /// <summary>
+    /// 同步国家统计局数据异常,请稍后重试
+    /// </summary>
+    [ErrorCodeItemMetadata("同步国家统计局数据异常,请稍后重试")]
+    R2005,
+
     /// <summary>
     /// 默认租户状态禁止修改
     /// </summary>

Admin.NET/Admin.NET.Core/SeedData/SysConfigSeedData.cs

@@ -32,6 +32,7 @@ public class SysConfigSeedData : ISqlSugarEntitySeedData<SysConfig>
             new SysConfig{ Id=1300000000201, Name="发送异常日志邮件", Code="sys_error_mail", Value="True", SysFlag=YesNoEnum.Y, Remark="是否发送异常日志邮件", OrderNo=110, GroupCode="Default", CreateTime=DateTime.Parse("2022-02-10 00:00:00") },
             new SysConfig{ Id=1300000000211, Name="开启域登录验证", Code="sys_domain_login", Value="False", SysFlag=YesNoEnum.Y, Remark="是否开启域登录验证", OrderNo=120, GroupCode="Default", CreateTime=DateTime.Parse("2022-02-10 00:00:00") },
             new SysConfig{ Id=1300000000221, Name="开启数据校验日志", Code="sys_validation_log", Value="True", SysFlag=YesNoEnum.Y, Remark="是否数据校验日志", OrderNo=130, GroupCode="Default", CreateTime=DateTime.Parse("2022-02-10 00:00:00") },
+            new SysConfig{ Id=1300000000231, Name="行政区域同步层级", Code="sys_region_sync_level", Value="3", SysFlag=YesNoEnum.Y, Remark="行政区域同步层级 1-省级,2-市级,3-区县级,4-街道级,5-村级", OrderNo=140, GroupCode="Default", CreateTime=DateTime.Parse("2022-02-10 00:00:00") },
             new SysConfig{ Id=1300000000301, Name="系统主标题", Code="sys_web_title", Value="Admin.NET", SysFlag=YesNoEnum.Y, Remark="系统主标题", OrderNo=300, GroupCode="WebConfig", CreateTime=DateTime.Parse("2022-02-10 00:00:00") },
             new SysConfig{ Id=1300000000311, Name="系统副标题", Code="sys_web_viceTitle", Value="Admin.NET", SysFlag=YesNoEnum.Y, Remark="系统副标题", OrderNo=310, GroupCode="WebConfig", CreateTime=DateTime.Parse("2022-02-10 00:00:00") },
             new SysConfig{ Id=1300000000321, Name="系统描述", Code="sys_web_viceDesc", Value="站在巨人肩膀上的 .NET 通用权限开发框架", SysFlag=YesNoEnum.Y, Remark="系统描述", OrderNo=320, GroupCode="WebConfig", CreateTime=DateTime.Parse("2022-02-10 00:00:00") },

Admin.NET/Admin.NET.Core/Service/Region/SysRegionService.cs

@@ -4,6 +4,9 @@
 //
 // 不得利用本项目从事危害国家安全、扰乱社会秩序、侵犯他人合法权益等法律法规禁止的活动！任何基于本项目二次开发而产生的一切法律纠纷和责任，我们不承担任何责任！
 
+using AngleSharp;
+using AngleSharp.Html.Dom;
+
 namespace Admin.NET.Core.Service;
 
 /// <summary>
@@ -13,13 +16,15 @@ namespace Admin.NET.Core.Service;
 public class SysRegionService : IDynamicApiController, ITransient
 {
     private readonly SqlSugarRepository<SysRegion> _sysRegionRep;
+    private readonly SysConfigService _sysConfigService;
 
-    //// Url地址-国家统计局行政区域2023年
-    //private readonly string _url = "http://www.stats.gov.cn/sj/tjbz/tjyqhdmhcxhfdm/2023/index.html";
+    // Url地址-国家统计局行政区域2023年
+    private readonly string _url = "http://www.stats.gov.cn/sj/tjbz/tjyqhdmhcxhfdm/2023/index.html";
 
-    public SysRegionService(SqlSugarRepository<SysRegion> sysRegionRep)
+    public SysRegionService(SqlSugarRepository<SysRegion> sysRegionRep, SysConfigService sysConfigService)
     {
         _sysRegionRep = sysRegionRep;
+        _sysConfigService = sysConfigService;
     }
 
     /// <summary>
@@ -59,7 +64,7 @@ public class SysRegionService : IDynamicApiController, ITransient
     {
         input.Code = input.Code.Trim();
         if (input.Code.Length != 12 && input.Code.Length != 9 && input.Code.Length != 6)
-            throw Oops.Oh("行政区代码只能为6、9或12位");
+            throw Oops.Oh(ErrorCodeEnum.R2003);
 
         if (input.Pid != 0)
         {
@@ -90,7 +95,7 @@ public class SysRegionService : IDynamicApiController, ITransient
     {
         input.Code = input.Code.Trim();
         if (input.Code.Length != 12 && input.Code.Length != 9 && input.Code.Length != 6)
-            throw Oops.Oh("行政区代码只能为6、9或12位");
+            throw Oops.Oh(ErrorCodeEnum.R2003);
 
         if (input.Pid != input.Pid && input.Pid != 0)
         {
@@ -103,7 +108,7 @@ public class SysRegionService : IDynamicApiController, ITransient
             var regionTreeList = await _sysRegionRep.AsQueryable().ToChildListAsync(u => u.Pid, input.Id, true);
             var childIdList = regionTreeList.Select(u => u.Id).ToList();
             if (childIdList.Contains(input.Pid))
-                throw Oops.Oh("父节点不能为自己的子节点");
+                throw Oops.Oh(ErrorCodeEnum.R2004);
         }
 
         if (input.Id == input.Pid)
@@ -144,93 +149,117 @@ public class SysRegionService : IDynamicApiController, ITransient
     [DisplayName("同步行政区域")]
     public async Task Sync()
     {
+        var syncLevel = await _sysConfigService.GetConfigValue<int>(CommonConst.SysRegionSyncLevel);
+        if (syncLevel < 1 || syncLevel > 5)
+            syncLevel = 3;//默认区县级
+        var context = BrowsingContext.New(AngleSharp.Configuration.Default.WithDefaultLoader());
+        var dom = await context.OpenAsync(_url);
+
+        // 省级
+        var itemList = dom.QuerySelectorAll("table.provincetable tr.provincetr td a");
+        if (itemList.Length == 0)
+            throw Oops.Oh(ErrorCodeEnum.R2005);
+
         await _sysRegionRep.DeleteAsync(u => u.Id > 0);
 
-        //var context = BrowsingContext.New(AngleSharp.Configuration.Default.WithDefaultLoader());
-        //var dom = await context.OpenAsync(_url);
-
-        //// 省级
-        //var itemList = dom.QuerySelectorAll("table.provincetable tr.provincetr td a");
-        //foreach (IHtmlAnchorElement item in itemList)
-        //{
-        //    var region = await _sysRegionRep.InsertReturnEntityAsync(new SysRegion
-        //    {
-        //        Pid = 0,
-        //        Name = item.TextContent,
-        //        Remark = item.Href,
-        //        Level = 1,
-        //    });
-
-        //    // 市级
-        //    if (string.IsNullOrEmpty(item.Href))
-        //        continue;
-        //    var dom1 = await context.OpenAsync(item.Href);
-        //    var itemList1 = dom1.QuerySelectorAll("table.citytable tr.citytr td a");
-        //    for (var i1 = 0; i1 < itemList1.Length; i1 += 2)
-        //    {
-        //        var item1 = (IHtmlAnchorElement)itemList1[i1 + 1];
-        //        var region1 = await _sysRegionRep.InsertReturnEntityAsync(new SysRegion
-        //        {
-        //            Pid = region.Id,
-        //            Name = item1.TextContent,
-        //            Code = itemList1[i1].TextContent,
-        //            Remark = item1.Href,
-        //            Level = 2,
-        //        });
-
-        //        // 区县级
-        //        if (string.IsNullOrEmpty(item1.Href))
-        //            continue;
-        //        var dom2 = await context.OpenAsync(item1.Href);
-        //        var itemList2 = dom2.QuerySelectorAll("table.countytable tr.countytr td a");
-        //        for (var i2 = 0; i2 < itemList2.Length; i2 += 2)
-        //        {
-        //            var item2 = (IHtmlAnchorElement)itemList2[i2 + 1];
-        //            var region2 = await _sysRegionRep.InsertReturnEntityAsync(new SysRegion
-        //            {
-        //                Pid = region1.Id,
-        //                Name = item2.TextContent,
-        //                Code = itemList2[i2].TextContent,
-        //                Remark = item2.Href,
-        //                Level = 3,
-        //            });
-
-        //            // 街道级
-        //            if (string.IsNullOrEmpty(item2.Href))
-        //                continue;
-        //            var dom3 = await context.OpenAsync(item2.Href);
-        //            var itemList3 = dom3.QuerySelectorAll("table.towntable tr.towntr td a");
-        //            for (var i3 = 0; i3 < itemList3.Length; i3 += 2)
-        //            {
-        //                var item3 = (IHtmlAnchorElement)itemList3[i3 + 1];
-        //                var region3 = await _sysRegionRep.InsertReturnEntityAsync(new SysRegion
-        //                {
-        //                    Pid = region2.Id,
-        //                    Name = item3.TextContent,
-        //                    Code = itemList3[i3].TextContent,
-        //                    Remark = item3.Href,
-        //                    Level = 4,
-        //                });
-
-        //                // 村级
-        //                if (string.IsNullOrEmpty(item3.Href))
-        //                    continue;
-        //                var dom4 = await context.OpenAsync(item3.Href);
-        //                var itemList4 = dom4.QuerySelectorAll("table.villagetable tr.villagetr td");
-        //                for (var i4 = 0; i4 < itemList4.Length; i4 += 3)
-        //                {
-        //                    await _sysRegionRep.InsertAsync(new SysRegion
-        //                    {
-        //                        Pid = region3.Id,
-        //                        Name = itemList4[i4 + 2].TextContent,
-        //                        Code = itemList4[i4].TextContent,
-        //                        CityCode = itemList4[i4 + 1].TextContent,
-        //                        Level = 5,
-        //                    });
-        //                }
-        //            }
-        //        }
-        //    }
-        //}
+        foreach (IHtmlAnchorElement item in itemList)
+        {
+            var list = new List<SysRegion>();
+
+            var region = new SysRegion
+            {
+                Id = YitIdHelper.NextId(),
+                Pid = 0,
+                Name = item.TextContent,
+                Remark = item.Href,
+                Level = 1,
+            };
+            list.Add(region);
+
+            // 市级
+            if (!string.IsNullOrEmpty(item.Href) && syncLevel > 1)
+            {
+                var dom1 = await context.OpenAsync(item.Href);
+                var itemList1 = dom1.QuerySelectorAll("table.citytable tr.citytr td a");
+                for (var i1 = 0; i1 < itemList1.Length; i1 += 2)
+                {
+                    var item1 = (IHtmlAnchorElement)itemList1[i1 + 1];
+                    var region1 = new SysRegion
+                    {
+                        Id = YitIdHelper.NextId(),
+                        Pid = region.Id,
+                        Name = item1.TextContent,
+                        Code = itemList1[i1].TextContent,
+                        Remark = item1.Href,
+                        Level = 2,
+                    };
+                    list.Add(region1);
+
+                    // 区县级
+                    if (!string.IsNullOrEmpty(item1.Href) && syncLevel > 2)
+                    {
+                        var dom2 = await context.OpenAsync(item1.Href);
+                        var itemList2 = dom2.QuerySelectorAll("table.countytable tr.countytr td a");
+                        for (var i2 = 0; i2 < itemList2.Length; i2 += 2)
+                        {
+                            var item2 = (IHtmlAnchorElement)itemList2[i2 + 1];
+                            var region2 = new SysRegion
+                            {
+                                Id = YitIdHelper.NextId(),
+                                Pid = region1.Id,
+                                Name = item2.TextContent,
+                                Code = itemList2[i2].TextContent,
+                                Remark = item2.Href,
+                                Level = 3,
+                            };
+                            list.Add(region2);
+
+                            // 街道级
+                            if (!string.IsNullOrEmpty(item2.Href) && syncLevel > 3)
+                            {
+                                var dom3 = await context.OpenAsync(item2.Href);
+                                var itemList3 = dom3.QuerySelectorAll("table.towntable tr.towntr td a");
+                                for (var i3 = 0; i3 < itemList3.Length; i3 += 2)
+                                {
+                                    var item3 = (IHtmlAnchorElement)itemList3[i3 + 1];
+                                    var region3 = new SysRegion
+                                    {
+                                        Id = YitIdHelper.NextId(),
+                                        Pid = region2.Id,
+                                        Name = item3.TextContent,
+                                        Code = itemList3[i3].TextContent,
+                                        Remark = item3.Href,
+                                        Level = 4,
+                                    };
+                                    list.Add(region3);
+
+                                    // 村级
+                                    if (!string.IsNullOrEmpty(item3.Href) && syncLevel > 4)
+                                    {
+                                        var dom4 = await context.OpenAsync(item3.Href);
+                                        var itemList4 = dom4.QuerySelectorAll("table.villagetable tr.villagetr td");
+                                        for (var i4 = 0; i4 < itemList4.Length; i4 += 3)
+                                        {
+                                            list.Add(new SysRegion
+                                            {
+                                                Id = YitIdHelper.NextId(),
+                                                Pid = region3.Id,
+                                                Name = itemList4[i4 + 2].TextContent,
+                                                Code = itemList4[i4].TextContent,
+                                                CityCode = itemList4[i4 + 1].TextContent,
+                                                Level = 5,
+                                            });
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+
+            //按省份同步快速写入提升同步效率，全部一次性写入容易出现从统计局获取数据失败
+            _sysRegionRep.Context.Fastest<SysRegion>().BulkCopy(list);
+        }
     }
 }