Skip to content

Commit

Permalink
feat: CC接口限频优化 (closed TencentBlueKing#2531)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ping15
ping15 committed Jan 7, 2025
1 parent 85f4f27 commit 688c1ee
Showing 1 changed file with 82 additions and 37 deletions.
119 changes: 82 additions & 37 deletions apps/backend/subscription/tools.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import math
import os
import pprint
import random
import typing
from collections import Counter, defaultdict
from concurrent.futures import ThreadPoolExecutor, as_completed
Expand Down Expand Up @@ -390,19 +391,15 @@ def get_modules_by_inst_list(inst_list, module_to_topo):
return module_ids, no_module_inst_list


@RetryHandler(interval=3, retry_times=2)
def get_service_instance_by_inst(bk_biz_id, inst_list, module_to_topo):
module_ids, no_module_inst_list = get_modules_by_inst_list(inst_list, module_to_topo)
if not module_ids:
return []

service_instance_module_id_threshold = models.GlobalSettings.get_config(
key=models.GlobalSettings.KeyEnum.SERVICE_INSTANCE_MODULE_ID_THRESHOLD.value,
default={},
)

if len(module_ids) <= service_instance_module_id_threshold.get(bk_biz_id, constants.QUERY_MODULE_ID_THRESHOLD):
params = [
def get_service_instance_ids(bk_biz_id: int, module_ids: List[int]) -> List[int]:
"""
查询服务实例ID列表
:param bk_biz_id: 业务ID
:param module_ids: 模块id列表
"""
service_instances = batch_call(
batch_request,
params_list=[
{
"func": CCApi.list_service_instance,
"params": {
Expand All @@ -416,27 +413,66 @@ def get_service_instance_by_inst(bk_biz_id, inst_list, module_to_topo):
"limit": constants.LIST_SERVICE_INSTANCE_DETAIL_LIMIT,
}
for bk_module_id in module_ids
]
],
extend_result=True,
interval=constants.LIST_SERVICE_INSTANCE_INTERVAL,
)

service_instance_infos = batch_call(
batch_request,
params,
extend_result=True,
interval=constants.LIST_SERVICE_INSTANCE_INTERVAL,
)
return [service_instance["id"] for service_instance in service_instances]

service_instances = batch_request(
func=CCApi.list_service_instance_detail,
params={
"bk_biz_id": int(bk_biz_id),
"with_name": True,
"no_request": True,
"service_instance_ids": [service_instance["id"] for service_instance in service_instance_infos],
},
sort="id",
interval=constants.LIST_SERVICE_INSTANCE_DETAIL_INTERVAL,
)

@RetryHandler(interval=3, retry_times=2)
def get_service_instance_by_inst(bk_biz_id, inst_list, module_to_topo):
module_ids, no_module_inst_list = get_modules_by_inst_list(inst_list, module_to_topo)
if not module_ids:
return []

service_instance_module_id_threshold = models.GlobalSettings.get_config(
key=models.GlobalSettings.KeyEnum.SERVICE_INSTANCE_MODULE_ID_THRESHOLD.value,
default={},
)

if len(module_ids) <= service_instance_module_id_threshold.get(str(bk_biz_id), constants.QUERY_MODULE_ID_THRESHOLD):
# 随机挑选一种方式获取服务实例详情，分摊list_service_instance_detail压力
# 1. 通过模块id分片查询所有的[服务实例ID列表]，再通过[服务实例ID列表]一次性筛选服务实例详情，避免了分片查询
# 结果 -> n次list_service_instance查询 + 1次list_service_instance_detail查询
# 2. 通过模块id分片查询服务实例详情
# 结果 -> n次list_service_instance_detail查询

# 如果module_ids只有一个，没必要使用第一种方式，一定会出现一次list_service_instance_detail查询
if len(module_ids) > 1 and random.random() < 0.5:
service_instances = batch_request(
func=CCApi.list_service_instance_detail,
params={
"bk_biz_id": int(bk_biz_id),
"with_name": True,
"no_request": True,
"service_instance_ids": get_service_instance_ids(bk_biz_id, list(module_ids)),
},
sort="id",
interval=constants.LIST_SERVICE_INSTANCE_DETAIL_INTERVAL,
)
else:
service_instances = batch_call(
func=batch_request,
params_list=[
{
"func": CCApi.list_service_instance_detail,
"params": {
"bk_biz_id": int(bk_biz_id),
"with_name": True,
"bk_module_id": bk_module_id,
# CC 接口统一使用后台访问
"no_request": True,
},
"sort": "id",
"limit": constants.LIST_SERVICE_INSTANCE_DETAIL_LIMIT,
}
for bk_module_id in module_ids
],
extend_result=True,
interval=constants.LIST_SERVICE_INSTANCE_DETAIL_INTERVAL,
)
else:
params = {"bk_biz_id": int(bk_biz_id), "with_name": True, "no_request": True}
service_instances = batch_request(
Expand Down Expand Up @@ -685,10 +721,13 @@ def get_host_detail(host_info_list: list, bk_biz_id: int = None):
bk_host_ids.append(host["bk_host_id"])
bk_cloud_ids.append(host["bk_cloud_id"])

host_relations = find_host_biz_relations(list(set(bk_host_ids)), source="get_host_detail")
host_biz_map = {}
for host in host_relations:
host_biz_map[host["bk_host_id"]] = host["bk_biz_id"]
if bk_biz_id:
host_biz_map = {_host["bk_host_id"]: bk_biz_id for _host in hosts}
else:
host_relations = find_host_biz_relations(list(set(bk_host_ids)), source="get_host_detail")
host_biz_map = {}
for host in host_relations:
host_biz_map[host["bk_host_id"]] = host["bk_biz_id"]

cloud_id_name_map = models.Cloud.cloud_id_name_map(get_cache=True)

Expand All @@ -699,7 +738,10 @@ def get_host_detail(host_info_list: list, bk_biz_id: int = None):
host_key_dict = {}
host_id_dict = {}
for _host in hosts:
_host["bk_biz_id"] = host_biz_map[_host["bk_host_id"]]
if bk_biz_id:
_host["bk_biz_id"] = bk_biz_id
else:
_host["bk_biz_id"] = host_biz_map[_host["bk_host_id"]]
_host["bk_biz_name"] = (
all_biz_info.get(_host["bk_biz_id"], {}).get("bk_biz_name", "")
if _host["bk_biz_id"] != settings.BK_CMDB_RESOURCE_POOL_BIZ_ID
Expand Down Expand Up @@ -1021,6 +1063,8 @@ def get_instances_by_scope(scope: Dict[str, Union[Dict, int, Any]]) -> Dict[str,
instances = add_host_module_info(host_biz_relations, instances)

else:
template_ids = [node["bk_inst_id"] for node in scope["nodes"]]

# 补充服务实例中的信息
# 转化模板为节点，**注意不可在get_service_instance_by_inst之后才转换**
nodes = set_template_scope_nodes(scope)
Expand All @@ -1030,7 +1074,8 @@ def get_instances_by_scope(scope: Dict[str, Union[Dict, int, Any]]) -> Dict[str,
[
{"service": inst}
for inst in get_service_instance_by_set_templates(
bk_biz_id=bk_biz_id, set_template_ids=[node["bk_inst_id"] for node in scope["nodes"]]
bk_biz_id=bk_biz_id,
set_template_ids=template_ids,
)
]
)
Expand Down

0 comments on commit 688c1ee

Please sign in to comment.