From 2ba93aaa3dc29768bb98e14807f5c7e1d556412b Mon Sep 17 00:00:00 2001 From: Mara3l Date: Fri, 20 Oct 2023 14:10:02 +0200 Subject: [PATCH] Fix generate.sh --- scripts/docs/json_builder.py | 49 ++++++++++--------- scripts/docs/python_ref_builder.py | 75 ++++++++++++++++++++---------- scripts/generate.sh | 27 +++++++++-- 3 files changed, 99 insertions(+), 52 deletions(-) diff --git a/scripts/docs/json_builder.py b/scripts/docs/json_builder.py index 0588b368d..b6fde3059 100644 --- a/scripts/docs/json_builder.py +++ b/scripts/docs/json_builder.py @@ -106,6 +106,7 @@ def docstring_fixes(docstr: str) -> str: Args: docstr: docstring to fix + Returns: str: fixed docstring """ @@ -197,6 +198,7 @@ def class_data(obj: type) -> ClassData: Args: obj(type): class object to be analysed + Returns: ClassData: parsed class data """ @@ -214,15 +216,10 @@ def class_data(obj: type) -> ClassData: if hasattr(value, attr) and getattr(value, attr) is not None: ret.functions[key] = function_data(getattr(value, attr), is_property=True) - # As of now, there are no subclasses in the gooddata package, - # and the data would not be handled correctly - # if inspect.isclass(value) and key != "__class__": - # ret["classes"][key] = object_data(value) - return ret -def module_data(module: ModuleType) -> dict: +def module_data(module: ModuleType, module_name: str) -> dict: """ Parse a module object and return formatted docstring data about its contents Args: @@ -231,7 +228,12 @@ def module_data(module: ModuleType) -> dict: dict: parsed module data """ data: dict[str, Any] = {"kind": "module"} - objects = vars(module) + if hasattr(module, "__dict__"): + objects = vars(module) + else: + # Handle the case where the object doesn't have __dict__. + objects = {} # or some other appropriate default or action + for name, obj in objects.items(): obj_module = inspect.getmodule(obj) if obj_module is None: @@ -239,22 +241,22 @@ def module_data(module: ModuleType) -> dict: if isinstance(obj, type): # Filter out non-gooddata libraries - if MODULE_NAME in obj_module.__name__: + if module_name in obj_module.__name__: data[name] = class_data(obj) elif isinstance(obj, ModuleType): - if MODULE_NAME in obj_module.__name__: + if module_name in obj_module.__name__: data[name] = module_data(obj) return data -def parse_package(obj: ModuleType, data: dict | None = None) -> dict: +def parse_package(obj: ModuleType, module_name: str = None) -> dict: """ Parse the package and its submodules into a dict object, that can be converted into a json Args: obj (ModuleType): package object - data (dict): optional parameter for recursive calling + module_name: name of the module Returns: dict: data of package @@ -268,9 +270,7 @@ def parse_package(obj: ModuleType, data: dict | None = None) -> dict: } } """ - if not data: - data = {} - data["kind"] = "module" + data = {"kind": "module"} if not isinstance(obj, ModuleType): return data @@ -281,11 +281,11 @@ def parse_package(obj: ModuleType, data: dict | None = None) -> dict: data[item.name] = {} if item.ispkg: - data[item.name].update(parse_package(vars(obj)[item.name])) + data[item.name].update(parse_package(vars(obj)[item.name], module_name)) else: - module = vars(obj)[item.name] - data[item.name].update(module_data(module)) - + if item.name in vars(obj): + module = vars(obj)[item.name] + data[item.name].update(module_data(module, module_name)) return data @@ -305,12 +305,15 @@ def import_submodules(pkg_name: str) -> dict[str, ModuleType]: if __name__ == "__main__": + import gooddata_pandas import gooddata_sdk - MODULE_NAME = "gooddata_sdk" # This global variable is needed in further parsing - import_submodules(MODULE_NAME) - res = parse_package(gooddata_sdk) - output_json: dict = cattrs.unstructure(res) + import_submodules("gooddata_sdk") + import_submodules("gooddata_pandas") + output_json: dict = { + **cattrs.unstructure(parse_package(gooddata_pandas, "gooddata_pandas")), + **cattrs.unstructure(parse_package(gooddata_sdk, "gooddata_sdk")), + } open("data.json", "w").write(json.dumps(output_json)) - print(f"Saved data.json and links_data.json to {os.getcwd()}") + print(f"Saved the .json file: `data.json` to {os.getcwd()}") diff --git a/scripts/docs/python_ref_builder.py b/scripts/docs/python_ref_builder.py index 945c6c23a..68817ad44 100644 --- a/scripts/docs/python_ref_builder.py +++ b/scripts/docs/python_ref_builder.py @@ -1,9 +1,12 @@ # (C) 2023 GoodData Corporation import argparse import json +import os from pathlib import Path from typing import List, TextIO +import attr +import toml from attr import define MODULE_TEMPLATE_STRING = Path("module_template.md").read_text() @@ -11,6 +14,15 @@ FUNCTION_TEMPLATE_STRING = Path("function_template.md").read_text() +@attr.s(auto_attribs=True) +class RefHolder: + """ """ + + url: str + packages: [] + directory: str + + @define class TemplateReplacementSpec: """ @@ -67,13 +79,13 @@ def create_file_structure(data: dict, root: Path, url_root: str): """ links = {} - def _recursive_create(data_root: dict, dir_root: Path, url_root: str, module_import_path: str): + def _recursive_create(data_root: dict, dir_root: Path, api_ref_root: str, module_import_path: str): """Recursively create files and directories. Args: data_root (dict): Sub-dictionary of the JSON representing the object. dir_root (Path): Path to the directory root. - url_root (str): URL root path for the API reference. + api_ref_root (str): URL root path for the API reference. module_import_path (str): Import path to the object. """ dir_root.mkdir(exist_ok=True) @@ -102,7 +114,7 @@ def _recursive_create(data_root: dict, dir_root: Path, url_root: str, module_imp template_spec.render_template_to_file(MODULE_TEMPLATE_STRING, f) # Add entry for url linking - links[name] = {"path": f"{url_root}/{name}".lower(), "kind": "function"} # Lowercase for Hugo + links[name] = {"path": f"{api_ref_root}/{name}".lower(), "kind": "function"} # Lowercase for Hugo elif kind == "class": (dir_root / name).mkdir(exist_ok=True) @@ -113,7 +125,7 @@ def _recursive_create(data_root: dict, dir_root: Path, url_root: str, module_imp template_spec.render_template_to_file(CLASS_TEMPLATE_STRING, f) # Add entry for url linking - links[name] = {"path": f"{url_root}/{name}".lower(), "kind": "class"} # Lowercase for Hugo + links[name] = {"path": f"{api_ref_root}/{name}".lower(), "kind": "class"} # Lowercase for Hugo elif name == "functions": for func_name, func in obj.items(): @@ -131,7 +143,7 @@ def _recursive_create(data_root: dict, dir_root: Path, url_root: str, module_imp # Add entry for url linking links[func_name] = { - "path": f"{url_root}/{func_name}".lower(), # Lowercase for Hugo + "path": f"{api_ref_root}/{func_name}".lower(), # Lowercase for Hugo "kind": "function", } continue # No need to recurse deeper, functions are the last level @@ -139,7 +151,7 @@ def _recursive_create(data_root: dict, dir_root: Path, url_root: str, module_imp else: continue # Not a class nor a module - _recursive_create(obj, dir_root / name, f"{url_root}/{name}", obj_module_import_path) + _recursive_create(obj, dir_root / name, f"{api_ref_root}/{name}", obj_module_import_path) _recursive_create(data, root, url_root, "") @@ -168,28 +180,43 @@ def change_json_root(data: dict, json_start_paths: List[str] | None) -> dict: return new_json +def parse_toml(toml_path: str, version: str, root_directory: str) -> [RefHolder]: + references = [] + # In case of missing toml_file, we need a default for the api-references + if not os.path.exists(toml_path): + return [ + RefHolder( + url=f"/{version}/api-reference", + packages=["sdk", "catalog"], + directory=f"{root_directory}/{version}/api-reference", + ) + ] + parsed_toml = toml.load(toml_path) + for name in parsed_toml: + packages = parsed_toml[name]["packages"] + directory = f"{root_directory}/{version}/{parsed_toml[name]['directory']}" + url = f"/{version}/{parsed_toml[name]['directory']}" + references.append(RefHolder(url, packages, directory)) + return references + + def main(): parser = argparse.ArgumentParser(description="Process a JSON file") - parser.add_argument("file", metavar="FILE", help="path to the JSON file", default="data.json") - parser.add_argument("output", metavar="FILE", help="root directory of the output", default="apiref") - parser.add_argument( - "--json_start_path", - default="", - required=False, - nargs="*", - help="Example: sdk.CatalogUserService, " - "would only generate markdown tree for that object," - "can use multiple start paths, by including the argument multiple times", - ) - parser.add_argument("--url_root", default="", required=False, help="url root path for the apiref") + parser.add_argument("toml_file", metavar="FILE", help="Paths to toml config file", default="api_spec.toml") + parser.add_argument("json_file", metavar="FILE", help="Paths to json data file", default="data.json") + parser.add_argument("version", metavar="str", help="Current Version", default="latest") + parser.add_argument("root_directory", metavar="str", help="Current Version", default="versioned_docs") + args = parser.parse_args() - print(f"Json start path is f{args.json_start_path}") - file_path = args.file - data = read_json_file(file_path) - data = change_json_root(data, args.json_start_path) - links = create_file_structure(data, Path(args.output), url_root=args.url_root) - json.dump(links, open("links.json", "w"), indent=4) + references = parse_toml(args.toml_file, args.version, args.root_directory) + links = {} + for ref in references: + print(f"Parsing: {ref.url}") + data = read_json_file(args.json_file) + data = change_json_root(data, ref.packages) + links.update(create_file_structure(data, Path(ref.directory), url_root=ref.url)) + json.dump(links, open(f"{args.root_directory}/{args.version}/links.json", "w"), indent=4) print("Dumping the links.json") diff --git a/scripts/generate.sh b/scripts/generate.sh index fb7d118b0..f77b0630a 100755 --- a/scripts/generate.sh +++ b/scripts/generate.sh @@ -73,14 +73,26 @@ for branch in "$remote_name/master" $(git branch -rl "$remote_name/rel/*") ; do if git cat-file -e $API_GEN_FILE; then echo "$API_GEN_FILE exists." echo "Generating API ref..." - python3 ../scripts/docs/json_builder.py - mv -f data.json ./versioned_docs/ if [ "$target_section" == "" ] ; then - python3 ../scripts/docs/python_ref_builder.py ./versioned_docs/data.json ./versioned_docs/docs/api-reference --json_start_path sdk catalog --url_root "/docs/api-reference" + echo "Skipping master api ref" + #mv -f data.json ./versioned_docs/latest/ + #python3 ../scripts/docs/python_ref_builder.py api_spec.toml ./versioned_docs/latest/data.json latest versioned_docs else - python3 ../scripts/docs/python_ref_builder.py ./versioned_docs/data.json ./versioned_docs/$target_section/api-reference --json_start_path sdk catalog --url_root "/$target_section/api-reference" + directories=$(find .. -type d -name 'gooddata-*') + + for dir in $directories; do + git checkout "$branch" -- "$dir" + done + if git ls-tree --name-only "$branch" | grep -q "^api_spec.toml$"; then + git checkout "$branch" -- api_spec.toml + else + echo "removing the API_spec" + rm -rf api_spec.toml + fi + python3 ../scripts/docs/json_builder.py + mv -f data.json ./versioned_docs/"$target_section"/ + python3 ../scripts/docs/python_ref_builder.py api_spec.toml ./versioned_docs/"$target_section"/data.json "$target_section" versioned_docs fi - mv -f links.json ./versioned_docs/ fi done @@ -90,6 +102,11 @@ highest_version=$(ls -v1 ./versioned_docs/ | grep -E '^[0-9]+.[0-9]+$' | tail -n echo "Moving ${highest_version} to /latest" mv -f ./versioned_docs/$highest_version ./versioned_docs/latest +# Replace "/${highest_version}/" with "/latest/" using sed +sed "s|${highest_version}|latest|g" ./versioned_docs/latest/links.json > temp.json + +mv temp.json ./versioned_docs/latest/links.json + if [ "$keep_master" != "keep_master" ] ; then echo "master docs will not be published, removing" rm -rf "${content_dir}/docs"