Skip to content

Commit

Permalink
feat: CC接口限频优化 (closed TencentBlueKing#2531)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ping15
ping15 committed Jan 7, 2025
1 parent ad70bb8 commit dd18452
Show file tree
Hide file tree
Showing 4 changed files with 108 additions and 13 deletions.
3 changes: 3 additions & 0 deletions apps/backend/subscription/commons.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,9 @@ def get_host_by_inst(bk_biz_id, inst_list):
bk_obj_id_list = [topo_data["bk_obj_id"] for topo_data in topo_data_list]

for inst in inst_list:
if "bk_obj_id" not in inst:
continue

# 处理各种类型的节点
if inst["bk_obj_id"] == "biz":
bk_biz_ids.append(bk_biz_id)
Expand Down
98 changes: 86 additions & 12 deletions apps/backend/subscription/tools.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
from apps.component.esbclient import client_v2
from apps.core.concurrent import controller
from apps.core.concurrent.cache import FuncCacheDecorator
from apps.core.concurrent.retry import RetryHandler
from apps.core.ipchooser.tools.base import HostQuerySqlHelper
from apps.node_man import constants, models
from apps.node_man import tools as node_man_tools
Expand Down Expand Up @@ -251,6 +252,7 @@ def create_host_key(data: Dict) -> str:


@SetupObserve(counter=metrics.app_common_method_requests_total, get_labels_func=get_call_resource_labels_func)
@RetryHandler(interval=3, retry_times=2)
def find_host_biz_relations(bk_host_ids: List[int]) -> List[Dict]:
"""
查询主机所属拓扑关系
Expand All @@ -275,7 +277,12 @@ def find_host_biz_relations(bk_host_ids: List[int]) -> List[Dict]:
{"bk_host_id": bk_host_ids[count * constants.QUERY_CMDB_LIMIT : (count + 1) * constants.QUERY_CMDB_LIMIT]}
for count in range(math.ceil(len(bk_host_ids) / constants.QUERY_CMDB_LIMIT))
]
host_biz_relations = request_multi_thread(client_v2.cc.find_host_biz_relations, param_list, get_data=lambda x: x)
host_biz_relations = batch_call(
func=client_v2.cc.find_host_biz_relations,
params_list=param_list,
interval=constants.FIND_HOST_BIZ_RELATIONS_INTERVAL,
)

return host_biz_relations


Expand Down Expand Up @@ -382,17 +389,21 @@ def get_modules_by_inst_list(inst_list, module_to_topo):
return module_ids, no_module_inst_list


@RetryHandler(interval=3, retry_times=2)
def get_service_instance_by_inst(bk_biz_id, inst_list, module_to_topo):
module_ids, no_module_inst_list = get_modules_by_inst_list(inst_list, module_to_topo)
if not module_ids:
return []

if len(module_ids) <= models.GlobalSettings.get_config(
models.GlobalSettings.KeyEnum.SERVICE_INSTANCE_MODULE_ID_THRESHOLD.value, constants.QUERY_MODULE_ID_THRESHOLD
):
service_instance_module_id_threshold = models.GlobalSettings.get_config(
key=models.GlobalSettings.KeyEnum.SERVICE_INSTANCE_MODULE_ID_THRESHOLD.value,
default={},
)

if len(module_ids) <= service_instance_module_id_threshold.get(bk_biz_id, constants.QUERY_MODULE_ID_THRESHOLD):
params = [
{
"func": CCApi.list_service_instance_detail,
"func": CCApi.list_service_instance,
"params": {
"bk_biz_id": int(bk_biz_id),
"with_name": True,
Expand All @@ -406,9 +417,25 @@ def get_service_instance_by_inst(bk_biz_id, inst_list, module_to_topo):
for bk_module_id in module_ids
]

service_instances = batch_call(
batch_request, params, extend_result=True, interval=constants.LIST_SERVICE_INSTANCE_DETAIL_INTERVAL
service_instance_infos = batch_call(
batch_request,
params,
extend_result=True,
interval=constants.LIST_SERVICE_INSTANCE_INTERVAL,
)

service_instances = batch_request(
func=CCApi.list_service_instance_detail,
params={
"bk_biz_id": int(bk_biz_id),
"with_name": True,
"no_request": True,
"service_instance_ids": [service_instance["id"] for service_instance in service_instance_infos],
},
sort="id",
interval=constants.LIST_SERVICE_INSTANCE_DETAIL_INTERVAL,
)

else:
params = {"bk_biz_id": int(bk_biz_id), "with_name": True, "no_request": True}
service_instances = batch_request(
Expand All @@ -426,7 +453,43 @@ def get_service_instance_by_inst(bk_biz_id, inst_list, module_to_topo):
return service_instances


@FuncCacheDecorator(cache_time=1 * constants.TimeUnit.MINUTE)
@RetryHandler(interval=3, retry_times=2)
def get_service_instance_by_set_templates(bk_biz_id: int, set_template_ids: List[int]):
params = [
{
"func": CCApi.list_service_instance_by_set_template,
"params": {
"bk_biz_id": int(bk_biz_id),
"set_template_id": set_template_id,
# CC 接口统一使用后台访问
"no_request": True,
},
"sort": "id",
"limit": constants.QUERY_CMDB_LIMIT,
}
for set_template_id in set_template_ids
]

service_instance_infos = batch_call(
batch_request, params, extend_result=True, interval=constants.LIST_SERVICE_INSTANCE_BY_SET_TEMPLATE_INTERVAL
)

service_instance_details = batch_request(
CCApi.list_service_instance_detail,
params={
"bk_biz_id": int(bk_biz_id),
"service_instance_ids": [service_instance_info["id"] for service_instance_info in service_instance_infos],
"no_request": True,
},
sort="id",
limit=constants.LIST_SERVICE_INSTANCE_DETAIL_LIMIT,
interval=constants.LIST_SERVICE_INSTANCE_DETAIL_INTERVAL,
)

return service_instance_details


@FuncCacheDecorator(cache_time=15 * constants.TimeUnit.MINUTE)
def fetch_biz_info_map(fields: typing.Optional[typing.List[str]] = None) -> typing.Dict[str, typing.Dict]:
"""
查询所有业务
Expand Down Expand Up @@ -883,7 +946,7 @@ def get_instances_by_scope(scope: Dict[str, Union[Dict, int, Any]]) -> Dict[str,

instances = []
bk_biz_id = scope["bk_biz_id"]
if bk_biz_id:
if bk_biz_id and scope["object_type"] == models.Subscription.ObjectType.SERVICE:
module_to_topo = get_module_to_topo_dict(bk_biz_id)
else:
module_to_topo = {}
Expand Down Expand Up @@ -960,9 +1023,20 @@ def get_instances_by_scope(scope: Dict[str, Union[Dict, int, Any]]) -> Dict[str,
# 补充服务实例中的信息
# 转化模板为节点，**注意不可在get_service_instance_by_inst之后才转换**
nodes = set_template_scope_nodes(scope)
instances.extend(
[{"service": inst} for inst in get_service_instance_by_inst(bk_biz_id, nodes, module_to_topo)]
)

if scope["node_type"] == models.Subscription.NodeType.SET_TEMPLATE:
instances.extend(
[
{"service": inst}
for inst in get_service_instance_by_set_templates(
bk_biz_id=bk_biz_id, set_template_ids=[node["bk_inst_id"] for node in scope["nodes"]]
)
]
)
else:
instances.extend(
[{"service": inst} for inst in get_service_instance_by_inst(bk_biz_id, nodes, module_to_topo)]
)

if not need_register:
# 补充必要的主机或实例相关信息
Expand Down
11 changes: 10 additions & 1 deletion apps/node_man/constants.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -599,7 +599,16 @@ def _get_member__alias_map(cls) -> Dict[Enum, str]:

# list_service_instance_detail接口调用参数配置
LIST_SERVICE_INSTANCE_DETAIL_LIMIT = 1000
LIST_SERVICE_INSTANCE_DETAIL_INTERVAL = 0.2
LIST_SERVICE_INSTANCE_DETAIL_INTERVAL = 0.3

# list_service_instance_by_set_template接口调用间隔
LIST_SERVICE_INSTANCE_BY_SET_TEMPLATE_INTERVAL = 0.3

# find_host_biz_relations接口调用间隔
FIND_HOST_BIZ_RELATIONS_INTERVAL = 0.3

# list_service_instance接口调用间隔
LIST_SERVICE_INSTANCE_INTERVAL = 0.3

# redis键名模板
REDIS_NEED_DELETE_HOST_IDS_KEY_TPL = f"{settings.APP_CODE}:node_man:need_delete_host_ids:list"
Expand Down
9 changes: 9 additions & 0 deletions common/api/modules/cc.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -256,3 +256,12 @@ def __init__(self):
before_request=add_esb_info_before_request,
api_name="list_service_instance_detail",
)
self.list_service_instance_by_set_template = DataAPI(
method="POST",
url=CC_APIGATEWAY_ROOT_V2 + "list_service_instance_by_set_template/",
module=self.MODULE,
simple_module=self.SIMPLE_MODULE,
description="通过集群模版查询关联的服务实例列表",
before_request=add_esb_info_before_request,
api_name="list_service_instance_by_set_template",
)

0 comments on commit dd18452

Please sign in to comment.