feat: CC接口限频优化 (closed TencentBlueKing#2531)

apps/backend/subscription/commons.py

@@ -77,6 +77,9 @@ def get_host_by_inst(bk_biz_id, inst_list):
     bk_obj_id_list = [topo_data["bk_obj_id"] for topo_data in topo_data_list]
 
     for inst in inst_list:
+        if "bk_obj_id" not in inst:
+            continue
+
         # 处理各种类型的节点
         if inst["bk_obj_id"] == "biz":
             bk_biz_ids.append(bk_biz_id)

apps/backend/subscription/tools.py

@@ -16,6 +16,7 @@
 import math
 import os
 import pprint
+import random
 import typing
 from collections import Counter, defaultdict
 from concurrent.futures import ThreadPoolExecutor, as_completed
@@ -41,6 +42,7 @@
 from apps.component.esbclient import client_v2
 from apps.core.concurrent import controller
 from apps.core.concurrent.cache import FuncCacheDecorator
+from apps.core.concurrent.retry import RetryHandler
 from apps.core.ipchooser.tools.base import HostQuerySqlHelper
 from apps.node_man import constants, models
 from apps.node_man import tools as node_man_tools
@@ -251,6 +253,7 @@ def create_host_key(data: Dict) -> str:
 
 
 @SetupObserve(counter=metrics.app_common_method_requests_total, get_labels_func=get_call_resource_labels_func)
+@RetryHandler(interval=3, retry_times=2)
 def find_host_biz_relations(bk_host_ids: List[int]) -> List[Dict]:
     """
     查询主机所属拓扑关系
@@ -275,7 +278,13 @@ def find_host_biz_relations(bk_host_ids: List[int]) -> List[Dict]:
         {"bk_host_id": bk_host_ids[count * constants.QUERY_CMDB_LIMIT : (count + 1) * constants.QUERY_CMDB_LIMIT]}
         for count in range(math.ceil(len(bk_host_ids) / constants.QUERY_CMDB_LIMIT))
     ]
-    host_biz_relations = request_multi_thread(client_v2.cc.find_host_biz_relations, param_list, get_data=lambda x: x)
+    host_biz_relations = batch_call(
+        func=client_v2.cc.find_host_biz_relations,
+        params_list=param_list,
+        interval=constants.FIND_HOST_BIZ_RELATIONS_INTERVAL,
+        extend_result=True,
+    )
+
     return host_biz_relations
 
 
@@ -382,17 +391,17 @@ def get_modules_by_inst_list(inst_list, module_to_topo):
     return module_ids, no_module_inst_list
 
 
-def get_service_instance_by_inst(bk_biz_id, inst_list, module_to_topo):
-    module_ids, no_module_inst_list = get_modules_by_inst_list(inst_list, module_to_topo)
-    if not module_ids:
-        return []
-
-    if len(module_ids) <= models.GlobalSettings.get_config(
-        models.GlobalSettings.KeyEnum.SERVICE_INSTANCE_MODULE_ID_THRESHOLD.value, constants.QUERY_MODULE_ID_THRESHOLD
-    ):
-        params = [
+def get_service_instance_ids(bk_biz_id: int, module_ids: List[int]) -> List[int]:
+    """
+    查询服务实例ID列表
+    :param bk_biz_id: 业务ID
+    :param module_ids: 模块id列表
+    """
+    service_instances = batch_call(
+        batch_request,
+        params_list=[
             {
-                "func": CCApi.list_service_instance_detail,
+                "func": CCApi.list_service_instance,
                 "params": {
                     "bk_biz_id": int(bk_biz_id),
                     "with_name": True,
@@ -404,11 +413,71 @@ def get_service_instance_by_inst(bk_biz_id, inst_list, module_to_topo):
                 "limit": constants.LIST_SERVICE_INSTANCE_DETAIL_LIMIT,
             }
             for bk_module_id in module_ids
-        ]
+        ],
+        extend_result=True,
+        interval=constants.LIST_SERVICE_INSTANCE_INTERVAL,
+    )
 
-        service_instances = batch_call(
-            batch_request, params, extend_result=True, interval=constants.LIST_SERVICE_INSTANCE_DETAIL_INTERVAL
-        )
+    return [service_instance["id"] for service_instance in service_instances]
+
+
+@RetryHandler(interval=3, retry_times=2)
+def get_service_instance_by_inst(bk_biz_id, inst_list, module_to_topo):
+    module_ids, no_module_inst_list = get_modules_by_inst_list(inst_list, module_to_topo)
+    if not module_ids:
+        return []
+
+    service_instance_module_id_threshold = models.GlobalSettings.get_config(
+        key=models.GlobalSettings.KeyEnum.SERVICE_INSTANCE_MODULE_ID_THRESHOLD.value,
+        default={},
+    )
+
+    if len(module_ids) <= service_instance_module_id_threshold.get(str(bk_biz_id), constants.QUERY_MODULE_ID_THRESHOLD):
+        # 随机挑选一种方式获取服务实例详情，分摊list_service_instance_detail压力
+        # 1. 通过模块id分片查询所有的[服务实例ID列表]，再通过[服务实例ID列表]一次性筛选服务实例详情，避免了分片查询
+        # 结果 -> n次list_service_instance查询 + 1次list_service_instance_detail查询
+        # 2. 通过模块id分片查询服务实例详情
+        # 结果 -> n次list_service_instance_detail查询
+
+        # 如果module_ids只有一个，没必要使用第一种方式，一定会出现一次list_service_instance_detail查询
+        if len(module_ids) > 1 and random.random() < 0.5:
+            service_instance_ids = get_service_instance_ids(bk_biz_id, list(module_ids))
+            if not service_instance_ids:
+                return []
+
+            service_instances = batch_request(
+                func=CCApi.list_service_instance_detail,
+                params={
+                    "bk_biz_id": int(bk_biz_id),
+                    "with_name": True,
+                    "no_request": True,
+                    "service_instance_ids": service_instance_ids,
+                },
+                sort="id",
+                interval=constants.LIST_SERVICE_INSTANCE_DETAIL_INTERVAL,
+                limit=constants.LIST_SERVICE_INSTANCE_DETAIL_LIMIT,
+            )
+        else:
+            service_instances = batch_call(
+                func=batch_request,
+                params_list=[
+                    {
+                        "func": CCApi.list_service_instance_detail,
+                        "params": {
+                            "bk_biz_id": int(bk_biz_id),
+                            "with_name": True,
+                            "bk_module_id": bk_module_id,
+                            # CC 接口统一使用后台访问
+                            "no_request": True,
+                        },
+                        "sort": "id",
+                        "limit": constants.LIST_SERVICE_INSTANCE_DETAIL_LIMIT,
+                    }
+                    for bk_module_id in module_ids
+                ],
+                extend_result=True,
+                interval=constants.LIST_SERVICE_INSTANCE_DETAIL_INTERVAL,
+            )
     else:
         params = {"bk_biz_id": int(bk_biz_id), "with_name": True, "no_request": True}
         service_instances = batch_request(
@@ -426,7 +495,54 @@ def get_service_instance_by_inst(bk_biz_id, inst_list, module_to_topo):
     return service_instances
 
 
-@FuncCacheDecorator(cache_time=1 * constants.TimeUnit.MINUTE)
+@RetryHandler(interval=3, retry_times=2)
+def get_service_instance_by_set_templates(bk_biz_id: int, set_template_ids: List[int]):
+    """
+    通过集群模板获取服务实例详情
+    :param bk_biz_id: 业务ID
+    :param set_template_ids: 集群模板id列表
+    """
+    params = [
+        {
+            "func": CCApi.list_service_instance_by_set_template,
+            "params": {
+                "bk_biz_id": int(bk_biz_id),
+                "set_template_id": set_template_id,
+                # CC 接口统一使用后台访问
+                "no_request": True,
+            },
+            "sort": "id",
+            "limit": constants.QUERY_CMDB_LIMIT,
+        }
+        for set_template_id in set_template_ids
+    ]
+
+    service_instance_ids: List[int] = [
+        service_instance["id"]
+        for service_instance in batch_call(
+            func=batch_request,
+            params_list=params,
+            extend_result=True,
+            interval=constants.LIST_SERVICE_INSTANCE_BY_SET_TEMPLATE_INTERVAL,
+        )
+    ]
+
+    service_instance_details = batch_request(
+        CCApi.list_service_instance_detail,
+        params={
+            "bk_biz_id": int(bk_biz_id),
+            "service_instance_ids": service_instance_ids,
+            "no_request": True,
+        },
+        sort="id",
+        limit=constants.LIST_SERVICE_INSTANCE_DETAIL_LIMIT,
+        interval=constants.LIST_SERVICE_INSTANCE_DETAIL_INTERVAL,
+    )
+
+    return service_instance_details
+
+
+@FuncCacheDecorator(cache_time=15 * constants.TimeUnit.MINUTE)
 def fetch_biz_info_map(fields: typing.Optional[typing.List[str]] = None) -> typing.Dict[str, typing.Dict]:
     """
     查询所有业务
@@ -621,10 +737,13 @@ def get_host_detail(host_info_list: list, bk_biz_id: int = None):
         bk_host_ids.append(host["bk_host_id"])
         bk_cloud_ids.append(host["bk_cloud_id"])
 
-    host_relations = find_host_biz_relations(list(set(bk_host_ids)), source="get_host_detail")
-    host_biz_map = {}
-    for host in host_relations:
-        host_biz_map[host["bk_host_id"]] = host["bk_biz_id"]
+    if bk_biz_id:
+        host_biz_map = {_host["bk_host_id"]: bk_biz_id for _host in hosts}
+    else:
+        host_relations = find_host_biz_relations(list(set(bk_host_ids)), source="get_host_detail")
+        host_biz_map = {}
+        for host in host_relations:
+            host_biz_map[host["bk_host_id"]] = host["bk_biz_id"]
 
     cloud_id_name_map = models.Cloud.cloud_id_name_map(get_cache=True)
 
@@ -635,7 +754,10 @@ def get_host_detail(host_info_list: list, bk_biz_id: int = None):
     host_key_dict = {}
     host_id_dict = {}
     for _host in hosts:
-        _host["bk_biz_id"] = host_biz_map[_host["bk_host_id"]]
+        if bk_biz_id:
+            _host["bk_biz_id"] = bk_biz_id
+        else:
+            _host["bk_biz_id"] = host_biz_map[_host["bk_host_id"]]
         _host["bk_biz_name"] = (
             all_biz_info.get(_host["bk_biz_id"], {}).get("bk_biz_name", "")
             if _host["bk_biz_id"] != settings.BK_CMDB_RESOURCE_POOL_BIZ_ID
@@ -883,7 +1005,7 @@ def get_instances_by_scope(scope: Dict[str, Union[Dict, int, Any]]) -> Dict[str,
 
     instances = []
     bk_biz_id = scope["bk_biz_id"]
-    if bk_biz_id:
+    if bk_biz_id and scope["object_type"] == models.Subscription.ObjectType.SERVICE:
         module_to_topo = get_module_to_topo_dict(bk_biz_id)
     else:
         module_to_topo = {}
@@ -957,12 +1079,26 @@ def get_instances_by_scope(scope: Dict[str, Union[Dict, int, Any]]) -> Dict[str,
             instances = add_host_module_info(host_biz_relations, instances)
 
         else:
+            template_ids = [node["bk_inst_id"] for node in scope["nodes"]]
+
             # 补充服务实例中的信息
             # 转化模板为节点，**注意不可在get_service_instance_by_inst之后才转换**
             nodes = set_template_scope_nodes(scope)
-            instances.extend(
-                [{"service": inst} for inst in get_service_instance_by_inst(bk_biz_id, nodes, module_to_topo)]
-            )
+
+            if scope["node_type"] == models.Subscription.NodeType.SET_TEMPLATE:
+                instances.extend(
+                    [
+                        {"service": inst}
+                        for inst in get_service_instance_by_set_templates(
+                            bk_biz_id=bk_biz_id,
+                            set_template_ids=template_ids,
+                        )
+                    ]
+                )
+            else:
+                instances.extend(
+                    [{"service": inst} for inst in get_service_instance_by_inst(bk_biz_id, nodes, module_to_topo)]
+                )
 
     if not need_register:
         # 补充必要的主机或实例相关信息

apps/backend/tests/subscription/utils.py

@@ -3327,6 +3327,28 @@ def list_service_instance_detail(cls, *args, **kwargs):
             SERVICE_DETAIL["info"] = [x for x in SERVICE_DETAIL["info"] if x["id"] == 10]
             return SERVICE_DETAIL
 
+        @classmethod
+        def list_service_instance(cls, *args, **kwargs):
+            return {
+                "count": 49,
+                "info": [
+                    {
+                        "bk_biz_id": 1,
+                        "id": 10,
+                        "name": "127.0.0.1_gse_agent",
+                        "labels": None,
+                        "service_template_id": 14,
+                        "bk_host_id": 1,
+                        "bk_module_id": 12,
+                        "creator": "cc_system",
+                        "modifier": "cc_system",
+                        "create_time": "2019-07-09T13:06:54.384+08:00",
+                        "last_time": "2019-07-09T13:06:54.384+08:00",
+                        "bk_supplier_account": "0",
+                    }
+                ],
+            }
+
         @classmethod
         def find_host_by_topo(cls, *args, **kwargs):
             return {"count": 4, "info": LIST_BIZ_HOSTS_WITHOUT_INFO}

apps/node_man/constants.py

@@ -599,7 +599,16 @@ def _get_member__alias_map(cls) -> Dict[Enum, str]:
 
 # list_service_instance_detail接口调用参数配置
 LIST_SERVICE_INSTANCE_DETAIL_LIMIT = 1000
-LIST_SERVICE_INSTANCE_DETAIL_INTERVAL = 0.2
+LIST_SERVICE_INSTANCE_DETAIL_INTERVAL = 0.3
+
+# list_service_instance_by_set_template接口调用间隔
+LIST_SERVICE_INSTANCE_BY_SET_TEMPLATE_INTERVAL = 0.3
+
+# find_host_biz_relations接口调用间隔
+FIND_HOST_BIZ_RELATIONS_INTERVAL = 0.3
+
+# list_service_instance接口调用间隔
+LIST_SERVICE_INSTANCE_INTERVAL = 0.3
 
 # redis键名模板
 REDIS_NEED_DELETE_HOST_IDS_KEY_TPL = f"{settings.APP_CODE}:node_man:need_delete_host_ids:list"

common/api/modules/cc.py

@@ -256,3 +256,12 @@ def __init__(self):
             before_request=add_esb_info_before_request,
             api_name="list_service_instance_detail",
         )
+        self.list_service_instance_by_set_template = DataAPI(
+            method="POST",
+            url=CC_APIGATEWAY_ROOT_V2 + "list_service_instance_by_set_template/",
+            module=self.MODULE,
+            simple_module=self.SIMPLE_MODULE,
+            description="通过集群模版查询关联的服务实例列表",
+            before_request=add_esb_info_before_request,
+            api_name="list_service_instance_by_set_template",
+        )