Skip to content

Commit

Permalink
[SDK generation pipeline] optimize for multi input (Azure#36830)
Browse files Browse the repository at this point in the history 
* optimize for multi input

* optimize for multi input

* optimize for judge tag is preview or not

* fix

* fix tsp-client

* fix

* fix

* fix
  • Loading branch information
@msyyc
msyyc authored Aug 9, 2024
1 parent d805e21 commit b7d0d38
Show file tree
Hide file tree
Showing 2 changed files with 92 additions and 69 deletions.
31 changes: 21 additions & 10 deletions tools/azure-sdk-tools/packaging_tools/generate_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

from ci_tools.git_tools import get_add_diff_file_list
from pathlib import Path
from subprocess import check_output, CalledProcessError, check_call
from subprocess import check_output, CalledProcessError, check_call, STDOUT
from typing import Dict, Any
from glob import glob
import yaml
Expand All @@ -27,9 +27,9 @@
_DPG_README = "README.md"


# input example: "../azure-rest-api-specs/specification/informatica/Informatica.DataManagement"
def del_outdated_generated_files(readme: str):
tspconfig = Path(readme) / "tspconfig.yaml"
# tsp example: "../azure-rest-api-specs/specification/informatica/Informatica.DataManagement"
def del_outdated_generated_files(tsp: str):
tspconfig = Path(tsp) / "tspconfig.yaml"
if not tspconfig.exists():
_LOGGER.info(f"do not find tspconfig.yaml: {tspconfig}")
return
Expand Down Expand Up @@ -195,8 +195,11 @@ def update_servicemetadata(sdk_folder, data, config, folder_name, package_name,
f.write("".join(includes))


def judge_tag_preview(path: str) -> bool:
files = [i for i in Path(path).glob("**/*.py")]
@return_origin_path
def judge_tag_preview(path: str, package_name: str) -> bool:
os.chdir(path)
first_level = package_name.split("-")[0]
files = [i for i in Path(".").glob(f"{first_level}/**/*.py")]
default_api_version = "" # for multi-api
api_version = "" # for single-api
for file in files:
Expand Down Expand Up @@ -427,14 +430,22 @@ def gen_typespec(typespec_relative_path: str, spec_folder: str, head_sha: str, r
try:
tsp_dir = (Path(spec_folder) / typespec_relative_path).resolve()
repo_url = rest_repo_url.replace("https://github.com/", "")
check_output(
f"tsp-client init --tsp-config {tsp_dir} --local-spec-repo {tsp_dir} --commit {head_sha} --repo {repo_url} --debug",
shell=True,
)
cmd = f"tsp-client init --tsp-config {tsp_dir} --local-spec-repo {tsp_dir} --commit {head_sha} --repo {repo_url} --debug"
_LOGGER.info(f"generation cmd: {cmd}")
output = check_output(cmd, stderr=STDOUT, shell=True)
except CalledProcessError as e:
_LOGGER.error(f"Failed to generate sdk from typespec: {e.output.decode('utf-8')}")
raise e

decode_output = output.decode("utf-8")
# before https://github.com/Azure/azure-sdk-tools/issues/8815, have to check output to judge whether sdk generation succeeds
if " - error " in decode_output:
_LOGGER.error(f"Failed to generate sdk from typespec:")
for item in decode_output.split("\n"):
if " - error " in item:
_LOGGER.error(item)
raise Exception(f"Failed to generate sdk from typespec: {decode_output}")

with open(Path("eng/emitter-package.json"), "r") as file_in:
data = json.load(file_in)
npm_package_verstion = {
Expand Down
130 changes: 71 additions & 59 deletions tools/azure-sdk-tools/packaging_tools/sdk_generator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -295,85 +295,97 @@ def main(generate_input, generate_output):
else:
del_outdated_generated_files(str(Path(spec_folder, readme_or_tsp)))
config = gen_typespec(readme_or_tsp, spec_folder, data["headSha"], data["repoHttpsUrl"])
package_names = get_package_names(sdk_folder)
except Exception as e:
_LOGGER.error(f"fail to generate sdk for {readme_or_tsp}: {str(e)}")
for hint_message in [
"======================================= Whant Can I do ========================================================================",
"If you are from service team, please first check if the failure happens only to Python automation, or for all SDK automations. ",
"======================================= Whant Can I do (begin) ========================================================================",
f"Fail to generate sdk for {readme_or_tsp}. If you are from service team, please first check if the failure happens only to Python automation, or for all SDK automations. ",
"If it happens for all SDK automations, please double check your Swagger / Typespec, and check whether there is error in ModelValidation and LintDiff. ",
"If it happens to Python alone, you can open an issue to https://github.com/Azure/autorest.python/issues. Please include the link of this Pull Request in the issue.",
"===============================================================================================================================",
"======================================= Whant Can I do (end) =========================================================================",
]:
_LOGGER.error(hint_message)
raise e
_LOGGER.info(f"[CODEGEN]({readme_or_tsp})codegen end. [(packages:{str(package_names)})]")
if len(readme_and_tsp) == 1:
raise e

# folder_name: "sdk/containerservice"; package_name: "azure-mgmt-containerservice"
package_names = get_package_names(sdk_folder)
spec_word = "readmeMd" if "readme.md" in readme_or_tsp else "typespecProject"
for folder_name, package_name in package_names:
if package_name in package_total:
continue

package_total.add(package_name)
sdk_code_path = str(Path(sdk_folder, folder_name, package_name))
if package_name not in result:
package_entry = {}
package_entry["packageName"] = package_name
package_entry["path"] = [folder_name]
package_entry[spec_word] = [readme_or_tsp]
package_entry["tagIsStable"] = not judge_tag_preview(sdk_code_path)
readme_python_content = get_readme_python_content(str(Path(spec_folder) / readme_or_tsp))
package_entry["isMultiapi"] = is_multiapi_package(readme_python_content)
result[package_name] = package_entry
else:
result[package_name]["path"].append(folder_name)
result[package_name][spec_word].append(readme_or_tsp)

# Generate some necessary file for new service
init_new_service(package_name, folder_name)
format_samples_and_tests(sdk_code_path)

# Update metadata
_LOGGER.info(
f"[CODEGEN]({readme_or_tsp})codegen end and new package '{folder_name}/{package_name}' generated"
)
try:
update_servicemetadata(
sdk_folder,
data,
config,
folder_name,
package_name,
spec_folder,
readme_or_tsp,
package_total.add(package_name)
sdk_code_path = str(Path(sdk_folder, folder_name, package_name))
if package_name not in result:
package_entry = {}
package_entry["packageName"] = package_name
package_entry["path"] = [folder_name]
package_entry[spec_word] = [readme_or_tsp]
package_entry["tagIsStable"] = not judge_tag_preview(sdk_code_path, package_name)
readme_python_content = get_readme_python_content(str(Path(spec_folder) / readme_or_tsp))
package_entry["isMultiapi"] = is_multiapi_package(readme_python_content)
result[package_name] = package_entry
else:
result[package_name]["path"].append(folder_name)
result[package_name][spec_word].append(readme_or_tsp)

# Generate some necessary file for new service
init_new_service(package_name, folder_name)
format_samples_and_tests(sdk_code_path)

# Update metadata
try:
update_servicemetadata(
sdk_folder,
data,
config,
folder_name,
package_name,
spec_folder,
readme_or_tsp,
)
except Exception as e:
_LOGGER.error(f"fail to update meta: {str(e)}")

# Setup package locally
check_call(
f"pip install --ignore-requires-python -e {sdk_code_path}",
shell=True,
)
except Exception as e:
_LOGGER.error(f"fail to update meta: {str(e)}")

# Setup package locally
check_call(
f"pip install --ignore-requires-python -e {sdk_code_path}",
shell=True,
)

# check whether multiapi package has only one api-version in per subfolder
# skip check for network for https://github.com/Azure/azure-sdk-for-python/issues/30556#issuecomment-1571341309
if "azure-mgmt-network" not in sdk_code_path:
check_api_version_in_subfolder(sdk_code_path)
# check whether multiapi package has only one api-version in per subfolder
# skip check for network for https://github.com/Azure/azure-sdk-for-python/issues/30556#issuecomment-1571341309
if "azure-mgmt-network" not in sdk_code_path:
check_api_version_in_subfolder(sdk_code_path)

# use multiapi combiner to combine multiapi package
if package_name in ("azure-mgmt-network"):
multiapi_combiner(sdk_code_path, package_name)
after_multiapi_combiner(sdk_code_path, package_name, folder_name)
result[package_name]["afterMultiapiCombiner"] = True
else:
result[package_name]["afterMultiapiCombiner"] = False
# use multiapi combiner to combine multiapi package
if package_name in ("azure-mgmt-network"):
multiapi_combiner(sdk_code_path, package_name)
after_multiapi_combiner(sdk_code_path, package_name, folder_name)
result[package_name]["afterMultiapiCombiner"] = True
else:
result[package_name]["afterMultiapiCombiner"] = False
except Exception as e:
_LOGGER.error(f"fail to setup package: {str(e)}")

# remove duplicates
for value in result.values():
value["path"] = list(set(value["path"]))
if value.get("typespecProject"):
value["typespecProject"] = list(set(value["typespecProject"]))
if value.get("readmeMd"):
value["readmeMd"] = list(set(value["readmeMd"]))
try:
for value in result.values():
value["path"] = list(set(value["path"]))
if value.get("typespecProject"):
value["typespecProject"] = list(set(value["typespecProject"]))
if value.get("readmeMd"):
value["readmeMd"] = list(set(value["readmeMd"]))
except Exception as e:
_LOGGER.error(f"fail to remove duplicates: {str(e)}")

if len(result) == 0 and len(readme_and_tsp) > 1:
raise Exception("No package is generated, please check the log for details")

with open(generate_output, "w") as writer:
json.dump(result, writer)
Expand Down

0 comments on commit b7d0d38

Please sign in to comment.