Fix generate.sh

scripts/docs/json_builder.py

@@ -106,6 +106,7 @@ def docstring_fixes(docstr: str) -> str:
 
     Args:
         docstr: docstring to fix
+
     Returns:
         str: fixed docstring
     """
@@ -197,6 +198,7 @@ def class_data(obj: type) -> ClassData:
 
     Args:
         obj(type): class object to be analysed
+
     Returns:
         ClassData: parsed class data
     """
@@ -214,15 +216,10 @@ def class_data(obj: type) -> ClassData:
                 if hasattr(value, attr) and getattr(value, attr) is not None:
                     ret.functions[key] = function_data(getattr(value, attr), is_property=True)
 
-        # As of now, there are no subclasses in the gooddata package,
-        # and the data would not be handled correctly
-        # if inspect.isclass(value) and key != "__class__":
-        #     ret["classes"][key] = object_data(value)
-
     return ret
 
 
-def module_data(module: ModuleType) -> dict:
+def module_data(module: ModuleType, module_name: str) -> dict:
     """
     Parse a module object and return formatted docstring data about its contents
     Args:
@@ -231,30 +228,35 @@ def module_data(module: ModuleType) -> dict:
         dict: parsed module data
     """
     data: dict[str, Any] = {"kind": "module"}
-    objects = vars(module)
+    if hasattr(module, "__dict__"):
+        objects = vars(module)
+    else:
+        # Handle the case where the object doesn't have __dict__.
+        objects = {}  # or some other appropriate default or action
+
     for name, obj in objects.items():
         obj_module = inspect.getmodule(obj)
         if obj_module is None:
             continue
 
         if isinstance(obj, type):
             # Filter out non-gooddata libraries
-            if MODULE_NAME in obj_module.__name__:
+            if module_name in obj_module.__name__:
                 data[name] = class_data(obj)
         elif isinstance(obj, ModuleType):
-            if MODULE_NAME in obj_module.__name__:
+            if module_name in obj_module.__name__:
                 data[name] = module_data(obj)
     return data
 
 
-def parse_package(obj: ModuleType, data: dict | None = None) -> dict:
+def parse_package(obj: ModuleType, module_name: str = None) -> dict:
     """
     Parse the package and its submodules into a dict object, that
     can be converted into a json
 
     Args:
         obj (ModuleType): package object
-        data (dict): optional parameter for recursive calling
+        module_name: name of the module
     Returns:
         dict: data of package
 
@@ -268,9 +270,7 @@ def parse_package(obj: ModuleType, data: dict | None = None) -> dict:
             }
         }
     """
-    if not data:
-        data = {}
-    data["kind"] = "module"
+    data = {"kind": "module"}
 
     if not isinstance(obj, ModuleType):
         return data
@@ -281,11 +281,11 @@ def parse_package(obj: ModuleType, data: dict | None = None) -> dict:
             data[item.name] = {}
 
         if item.ispkg:
-            data[item.name].update(parse_package(vars(obj)[item.name]))
+            data[item.name].update(parse_package(vars(obj)[item.name], module_name))
         else:
-            module = vars(obj)[item.name]
-            data[item.name].update(module_data(module))
-
+            if item.name in vars(obj):
+                module = vars(obj)[item.name]
+                data[item.name].update(module_data(module, module_name))
     return data
 
 
@@ -305,12 +305,15 @@ def import_submodules(pkg_name: str) -> dict[str, ModuleType]:
 
 
 if __name__ == "__main__":
+    import gooddata_pandas
     import gooddata_sdk
 
-    MODULE_NAME = "gooddata_sdk"  # This global variable is needed in further parsing
-    import_submodules(MODULE_NAME)
-    res = parse_package(gooddata_sdk)
-    output_json: dict = cattrs.unstructure(res)
+    import_submodules("gooddata_sdk")
+    import_submodules("gooddata_pandas")
+    output_json: dict = {
+        **cattrs.unstructure(parse_package(gooddata_pandas, "gooddata_pandas")),
+        **cattrs.unstructure(parse_package(gooddata_sdk, "gooddata_sdk")),
+    }
     open("data.json", "w").write(json.dumps(output_json))
 
-    print(f"Saved data.json and links_data.json to {os.getcwd()}")
+    print(f"Saved the .json file: `data.json` to {os.getcwd()}")

scripts/docs/python_ref_builder.py

@@ -1,16 +1,28 @@
 # (C) 2023 GoodData Corporation
 import argparse
 import json
+import os
 from pathlib import Path
 from typing import List, TextIO
 
+import attr
+import toml
 from attr import define
 
 MODULE_TEMPLATE_STRING = Path("module_template.md").read_text()
 CLASS_TEMPLATE_STRING = Path("class_template.md").read_text()
 FUNCTION_TEMPLATE_STRING = Path("function_template.md").read_text()
 
 
+@attr.s(auto_attribs=True)
+class RefHolder:
+    """ """
+
+    url: str
+    packages: []
+    directory: str
+
+
 @define
 class TemplateReplacementSpec:
     """
@@ -67,13 +79,13 @@ def create_file_structure(data: dict, root: Path, url_root: str):
     """
     links = {}
 
-    def _recursive_create(data_root: dict, dir_root: Path, url_root: str, module_import_path: str):
+    def _recursive_create(data_root: dict, dir_root: Path, api_ref_root: str, module_import_path: str):
         """Recursively create files and directories.
 
         Args:
             data_root (dict): Sub-dictionary of the JSON representing the object.
             dir_root (Path): Path to the directory root.
-            url_root (str): URL root path for the API reference.
+            api_ref_root (str): URL root path for the API reference.
             module_import_path (str): Import path to the object.
         """
         dir_root.mkdir(exist_ok=True)
@@ -102,7 +114,7 @@ def _recursive_create(data_root: dict, dir_root: Path, url_root: str, module_imp
                     template_spec.render_template_to_file(MODULE_TEMPLATE_STRING, f)
 
                 # Add entry for url linking
-                links[name] = {"path": f"{url_root}/{name}".lower(), "kind": "function"}  # Lowercase for Hugo
+                links[name] = {"path": f"{api_ref_root}/{name}".lower(), "kind": "function"}  # Lowercase for Hugo
 
             elif kind == "class":
                 (dir_root / name).mkdir(exist_ok=True)
@@ -113,7 +125,7 @@ def _recursive_create(data_root: dict, dir_root: Path, url_root: str, module_imp
                     template_spec.render_template_to_file(CLASS_TEMPLATE_STRING, f)
 
                 # Add entry for url linking
-                links[name] = {"path": f"{url_root}/{name}".lower(), "kind": "class"}  # Lowercase for Hugo
+                links[name] = {"path": f"{api_ref_root}/{name}".lower(), "kind": "class"}  # Lowercase for Hugo
 
             elif name == "functions":
                 for func_name, func in obj.items():
@@ -131,15 +143,15 @@ def _recursive_create(data_root: dict, dir_root: Path, url_root: str, module_imp
 
                     # Add entry for url linking
                     links[func_name] = {
-                        "path": f"{url_root}/{func_name}".lower(),  # Lowercase for Hugo
+                        "path": f"{api_ref_root}/{func_name}".lower(),  # Lowercase for Hugo
                         "kind": "function",
                     }
                 continue  # No need to recurse deeper, functions are the last level
 
             else:
                 continue  # Not a class nor a module
 
-            _recursive_create(obj, dir_root / name, f"{url_root}/{name}", obj_module_import_path)
+            _recursive_create(obj, dir_root / name, f"{api_ref_root}/{name}", obj_module_import_path)
 
     _recursive_create(data, root, url_root, "")
 
@@ -168,28 +180,43 @@ def change_json_root(data: dict, json_start_paths: List[str] | None) -> dict:
     return new_json
 
 
+def parse_toml(toml_path: str, version: str, root_directory: str) -> [RefHolder]:
+    references = []
+    # In case of missing toml_file, we need a default for the api-references
+    if not os.path.exists(toml_path):
+        return [
+            RefHolder(
+                url=f"/{version}/api-reference",
+                packages=["sdk", "catalog"],
+                directory=f"{root_directory}/{version}/api-reference",
+            )
+        ]
+    parsed_toml = toml.load(toml_path)
+    for name in parsed_toml:
+        packages = parsed_toml[name]["packages"]
+        directory = f"{root_directory}/{version}/{parsed_toml[name]['directory']}"
+        url = f"/{version}/{parsed_toml[name]['directory']}"
+        references.append(RefHolder(url, packages, directory))
+    return references
+
+
 def main():
     parser = argparse.ArgumentParser(description="Process a JSON file")
-    parser.add_argument("file", metavar="FILE", help="path to the JSON file", default="data.json")
-    parser.add_argument("output", metavar="FILE", help="root directory of the output", default="apiref")
-    parser.add_argument(
-        "--json_start_path",
-        default="",
-        required=False,
-        nargs="*",
-        help="Example: sdk.CatalogUserService, "
-        "would only generate markdown tree for that object,"
-        "can use multiple start paths, by including the argument multiple times",
-    )
-    parser.add_argument("--url_root", default="", required=False, help="url root path for the apiref")
+    parser.add_argument("toml_file", metavar="FILE", help="Paths to toml config file", default="api_spec.toml")
+    parser.add_argument("json_file", metavar="FILE", help="Paths to json data file", default="data.json")
+    parser.add_argument("version", metavar="str", help="Current Version", default="latest")
+    parser.add_argument("root_directory", metavar="str", help="Current Version", default="versioned_docs")
+
     args = parser.parse_args()
-    print(f"Json start path is f{args.json_start_path}")
 
-    file_path = args.file
-    data = read_json_file(file_path)
-    data = change_json_root(data, args.json_start_path)
-    links = create_file_structure(data, Path(args.output), url_root=args.url_root)
-    json.dump(links, open("links.json", "w"), indent=4)
+    references = parse_toml(args.toml_file, args.version, args.root_directory)
+    links = {}
+    for ref in references:
+        print(f"Parsing: {ref.url}")
+        data = read_json_file(args.json_file)
+        data = change_json_root(data, ref.packages)
+        links.update(create_file_structure(data, Path(ref.directory), url_root=ref.url))
+    json.dump(links, open(f"{args.root_directory}/{args.version}/links.json", "w"), indent=4)
     print("Dumping the links.json")
 
 

scripts/generate.sh

@@ -73,14 +73,26 @@ for branch in "$remote_name/master" $(git branch -rl "$remote_name/rel/*") ; do
     if git cat-file -e $API_GEN_FILE; then
         echo "$API_GEN_FILE exists."
         echo "Generating API ref..."
-        python3 ../scripts/docs/json_builder.py
-        mv -f data.json ./versioned_docs/
         if [ "$target_section" == "" ] ; then
-            python3 ../scripts/docs/python_ref_builder.py ./versioned_docs/data.json ./versioned_docs/docs/api-reference --json_start_path sdk catalog --url_root "/docs/api-reference"
+            echo "Skipping master api ref"
+            #mv -f data.json ./versioned_docs/latest/
+            #python3 ../scripts/docs/python_ref_builder.py api_spec.toml ./versioned_docs/latest/data.json latest versioned_docs
         else
-            python3 ../scripts/docs/python_ref_builder.py ./versioned_docs/data.json ./versioned_docs/$target_section/api-reference --json_start_path sdk catalog --url_root "/$target_section/api-reference"
+            directories=$(find .. -type d -name 'gooddata-*')
+
+            for dir in $directories; do
+                git checkout "$branch" -- "$dir"
+            done
+            if git ls-tree --name-only "$branch" | grep -q "^api_spec.toml$"; then
+                git checkout "$branch" -- api_spec.toml
+            else
+              echo "removing the API_spec"
+              rm -rf api_spec.toml
+            fi
+            python3 ../scripts/docs/json_builder.py
+            mv -f data.json ./versioned_docs/"$target_section"/
+            python3 ../scripts/docs/python_ref_builder.py api_spec.toml ./versioned_docs/"$target_section"/data.json "$target_section" versioned_docs
         fi
-        mv -f links.json ./versioned_docs/
     fi
 done
 
@@ -90,6 +102,11 @@ highest_version=$(ls -v1 ./versioned_docs/ | grep -E '^[0-9]+.[0-9]+$' | tail -n
 echo "Moving ${highest_version} to /latest"
 mv -f ./versioned_docs/$highest_version ./versioned_docs/latest
 
+# Replace "/${highest_version}/" with "/latest/" using sed
+sed "s|${highest_version}|latest|g" ./versioned_docs/latest/links.json > temp.json
+
+mv temp.json ./versioned_docs/latest/links.json
+
 if [ "$keep_master" != "keep_master" ] ; then
     echo "master docs will not be published, removing"
     rm -rf "${content_dir}/docs"