Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

TRIVIAL: fix broken generate.sh #391

Merged
1 commit merged into from
Oct 23, 2023
Merged

TRIVIAL: fix broken generate.sh #391

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 26 additions & 23 deletions scripts/docs/json_builder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ def docstring_fixes(docstr: str) -> str:

Args:
docstr: docstring to fix

Returns:
str: fixed docstring
"""
Expand Down Expand Up @@ -197,6 +198,7 @@ def class_data(obj: type) -> ClassData:

Args:
obj(type): class object to be analysed

Returns:
ClassData: parsed class data
"""
Expand All @@ -214,15 +216,10 @@ def class_data(obj: type) -> ClassData:
if hasattr(value, attr) and getattr(value, attr) is not None:
ret.functions[key] = function_data(getattr(value, attr), is_property=True)

# As of now, there are no subclasses in the gooddata package,
# and the data would not be handled correctly
# if inspect.isclass(value) and key != "__class__":
# ret["classes"][key] = object_data(value)

return ret


def module_data(module: ModuleType) -> dict:
def module_data(module: ModuleType, module_name: str) -> dict:
"""
Parse a module object and return formatted docstring data about its contents
Args:
Expand All @@ -231,30 +228,35 @@ def module_data(module: ModuleType) -> dict:
dict: parsed module data
"""
data: dict[str, Any] = {"kind": "module"}
objects = vars(module)
if hasattr(module, "__dict__"):
objects = vars(module)
else:
# Handle the case where the object doesn't have __dict__.
objects = {} # or some other appropriate default or action

for name, obj in objects.items():
obj_module = inspect.getmodule(obj)
if obj_module is None:
continue

if isinstance(obj, type):
# Filter out non-gooddata libraries
if MODULE_NAME in obj_module.__name__:
if module_name in obj_module.__name__:
data[name] = class_data(obj)
elif isinstance(obj, ModuleType):
if MODULE_NAME in obj_module.__name__:
if module_name in obj_module.__name__:
data[name] = module_data(obj)
return data


def parse_package(obj: ModuleType, data: dict | None = None) -> dict:
def parse_package(obj: ModuleType, module_name: str = None) -> dict:
"""
Parse the package and its submodules into a dict object, that
can be converted into a json

Args:
obj (ModuleType): package object
data (dict): optional parameter for recursive calling
module_name: name of the module
Returns:
dict: data of package

Expand All @@ -268,9 +270,7 @@ def parse_package(obj: ModuleType, data: dict | None = None) -> dict:
}
}
"""
if not data:
data = {}
data["kind"] = "module"
data = {"kind": "module"}

if not isinstance(obj, ModuleType):
return data
Expand All @@ -281,11 +281,11 @@ def parse_package(obj: ModuleType, data: dict | None = None) -> dict:
data[item.name] = {}

if item.ispkg:
data[item.name].update(parse_package(vars(obj)[item.name]))
data[item.name].update(parse_package(vars(obj)[item.name], module_name))
else:
module = vars(obj)[item.name]
data[item.name].update(module_data(module))

if item.name in vars(obj):
module = vars(obj)[item.name]
data[item.name].update(module_data(module, module_name))
return data


Expand All @@ -305,12 +305,15 @@ def import_submodules(pkg_name: str) -> dict[str, ModuleType]:


if __name__ == "__main__":
import gooddata_pandas
import gooddata_sdk

MODULE_NAME = "gooddata_sdk" # This global variable is needed in further parsing
import_submodules(MODULE_NAME)
res = parse_package(gooddata_sdk)
output_json: dict = cattrs.unstructure(res)
import_submodules("gooddata_sdk")
import_submodules("gooddata_pandas")
output_json: dict = {
**cattrs.unstructure(parse_package(gooddata_pandas, "gooddata_pandas")),
**cattrs.unstructure(parse_package(gooddata_sdk, "gooddata_sdk")),
}
open("data.json", "w").write(json.dumps(output_json))

print(f"Saved data.json and links_data.json to {os.getcwd()}")
print(f"Saved the .json file: `data.json` to {os.getcwd()}")
75 changes: 51 additions & 24 deletions scripts/docs/python_ref_builder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,28 @@
# (C) 2023 GoodData Corporation
import argparse
import json
import os
from pathlib import Path
from typing import List, TextIO

import attr
import toml
from attr import define

MODULE_TEMPLATE_STRING = Path("module_template.md").read_text()
CLASS_TEMPLATE_STRING = Path("class_template.md").read_text()
FUNCTION_TEMPLATE_STRING = Path("function_template.md").read_text()


@attr.s(auto_attribs=True)
class RefHolder:
""" """

url: str
packages: []
directory: str


@define
class TemplateReplacementSpec:
"""
Expand Down Expand Up @@ -67,13 +79,13 @@ def create_file_structure(data: dict, root: Path, url_root: str):
"""
links = {}

def _recursive_create(data_root: dict, dir_root: Path, url_root: str, module_import_path: str):
def _recursive_create(data_root: dict, dir_root: Path, api_ref_root: str, module_import_path: str):
"""Recursively create files and directories.

Args:
data_root (dict): Sub-dictionary of the JSON representing the object.
dir_root (Path): Path to the directory root.
url_root (str): URL root path for the API reference.
api_ref_root (str): URL root path for the API reference.
module_import_path (str): Import path to the object.
"""
dir_root.mkdir(exist_ok=True)
Expand Down Expand Up @@ -102,7 +114,7 @@ def _recursive_create(data_root: dict, dir_root: Path, url_root: str, module_imp
template_spec.render_template_to_file(MODULE_TEMPLATE_STRING, f)

# Add entry for url linking
links[name] = {"path": f"{url_root}/{name}".lower(), "kind": "function"} # Lowercase for Hugo
links[name] = {"path": f"{api_ref_root}/{name}".lower(), "kind": "function"} # Lowercase for Hugo

elif kind == "class":
(dir_root / name).mkdir(exist_ok=True)
Expand All @@ -113,7 +125,7 @@ def _recursive_create(data_root: dict, dir_root: Path, url_root: str, module_imp
template_spec.render_template_to_file(CLASS_TEMPLATE_STRING, f)

# Add entry for url linking
links[name] = {"path": f"{url_root}/{name}".lower(), "kind": "class"} # Lowercase for Hugo
links[name] = {"path": f"{api_ref_root}/{name}".lower(), "kind": "class"} # Lowercase for Hugo

elif name == "functions":
for func_name, func in obj.items():
Expand All @@ -131,15 +143,15 @@ def _recursive_create(data_root: dict, dir_root: Path, url_root: str, module_imp

# Add entry for url linking
links[func_name] = {
"path": f"{url_root}/{func_name}".lower(), # Lowercase for Hugo
"path": f"{api_ref_root}/{func_name}".lower(), # Lowercase for Hugo
"kind": "function",
}
continue # No need to recurse deeper, functions are the last level

else:
continue # Not a class nor a module

_recursive_create(obj, dir_root / name, f"{url_root}/{name}", obj_module_import_path)
_recursive_create(obj, dir_root / name, f"{api_ref_root}/{name}", obj_module_import_path)

_recursive_create(data, root, url_root, "")

Expand Down Expand Up @@ -168,28 +180,43 @@ def change_json_root(data: dict, json_start_paths: List[str] | None) -> dict:
return new_json


def parse_toml(toml_path: str, version: str, root_directory: str) -> [RefHolder]:
references = []
# In case of missing toml_file, we need a default for the api-references
if not os.path.exists(toml_path):
return [
RefHolder(
url=f"/{version}/api-reference",
packages=["sdk", "catalog"],
directory=f"{root_directory}/{version}/api-reference",
)
]
parsed_toml = toml.load(toml_path)
for name in parsed_toml:
packages = parsed_toml[name]["packages"]
directory = f"{root_directory}/{version}/{parsed_toml[name]['directory']}"
url = f"/{version}/{parsed_toml[name]['directory']}"
references.append(RefHolder(url, packages, directory))
return references


def main():
parser = argparse.ArgumentParser(description="Process a JSON file")
parser.add_argument("file", metavar="FILE", help="path to the JSON file", default="data.json")
parser.add_argument("output", metavar="FILE", help="root directory of the output", default="apiref")
parser.add_argument(
"--json_start_path",
default="",
required=False,
nargs="*",
help="Example: sdk.CatalogUserService, "
"would only generate markdown tree for that object,"
"can use multiple start paths, by including the argument multiple times",
)
parser.add_argument("--url_root", default="", required=False, help="url root path for the apiref")
parser.add_argument("toml_file", metavar="FILE", help="Paths to toml config file", default="api_spec.toml")
parser.add_argument("json_file", metavar="FILE", help="Paths to json data file", default="data.json")
parser.add_argument("version", metavar="str", help="Current Version", default="latest")
parser.add_argument("root_directory", metavar="str", help="Current Version", default="versioned_docs")

args = parser.parse_args()
print(f"Json start path is f{args.json_start_path}")

file_path = args.file
data = read_json_file(file_path)
data = change_json_root(data, args.json_start_path)
links = create_file_structure(data, Path(args.output), url_root=args.url_root)
json.dump(links, open("links.json", "w"), indent=4)
references = parse_toml(args.toml_file, args.version, args.root_directory)
links = {}
for ref in references:
print(f"Parsing: {ref.url}")
data = read_json_file(args.json_file)
data = change_json_root(data, ref.packages)
links.update(create_file_structure(data, Path(ref.directory), url_root=ref.url))
json.dump(links, open(f"{args.root_directory}/{args.version}/links.json", "w"), indent=4)
print("Dumping the links.json")


Expand Down
27 changes: 22 additions & 5 deletions scripts/generate.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,14 +73,26 @@ for branch in "$remote_name/master" $(git branch -rl "$remote_name/rel/*") ; do
if git cat-file -e $API_GEN_FILE; then
echo "$API_GEN_FILE exists."
echo "Generating API ref..."
python3 ../scripts/docs/json_builder.py
mv -f data.json ./versioned_docs/
if [ "$target_section" == "" ] ; then
python3 ../scripts/docs/python_ref_builder.py ./versioned_docs/data.json ./versioned_docs/docs/api-reference --json_start_path sdk catalog --url_root "/docs/api-reference"
echo "Skipping master api ref"
#mv -f data.json ./versioned_docs/latest/
#python3 ../scripts/docs/python_ref_builder.py api_spec.toml ./versioned_docs/latest/data.json latest versioned_docs
else
python3 ../scripts/docs/python_ref_builder.py ./versioned_docs/data.json ./versioned_docs/$target_section/api-reference --json_start_path sdk catalog --url_root "/$target_section/api-reference"
directories=$(find .. -type d -name 'gooddata-*')

for dir in $directories; do
git checkout "$branch" -- "$dir"
done
if git ls-tree --name-only "$branch" | grep -q "^api_spec.toml$"; then
git checkout "$branch" -- api_spec.toml
else
echo "removing the API_spec"
rm -rf api_spec.toml
fi
python3 ../scripts/docs/json_builder.py
mv -f data.json ./versioned_docs/"$target_section"/
python3 ../scripts/docs/python_ref_builder.py api_spec.toml ./versioned_docs/"$target_section"/data.json "$target_section" versioned_docs
fi
mv -f links.json ./versioned_docs/
fi
done

Expand All @@ -90,6 +102,11 @@ highest_version=$(ls -v1 ./versioned_docs/ | grep -E '^[0-9]+.[0-9]+$' | tail -n
echo "Moving ${highest_version} to /latest"
mv -f ./versioned_docs/$highest_version ./versioned_docs/latest

# Replace "/${highest_version}/" with "/latest/" using sed
sed "s|${highest_version}|latest|g" ./versioned_docs/latest/links.json > temp.json

mv temp.json ./versioned_docs/latest/links.json

if [ "$keep_master" != "keep_master" ] ; then
echo "master docs will not be published, removing"
rm -rf "${content_dir}/docs"
Expand Down