Merge branch 'master' into feature/catalog-custom-properties

.vscode/settings.json

@@ -30,6 +30,7 @@
     "**/.venv": true,
     "**/.mypy_cache": true,
     "**/.pytest_cache": true,
-    "**/tmp*": true
+    "**/tmp*": true,
+    "**/__pycache__": true
   }
 }

elx/cli/app.py

@@ -0,0 +1,29 @@
+from pathlib import Path
+import typer
+from rich import print
+
+from elx.cli import debug
+
+# from elx.cli import invoke
+from elx.cli import catalog
+from dotenv import load_dotenv, dotenv_values
+
+app = typer.Typer()
+
+app.command()(debug.debug)
+# app.command()(invoke.invoke)
+app.command()(catalog.catalog)
+
+
+def cli():
+    env_path = Path.cwd() / ".env"
+    loaded_env = load_dotenv(env_path)
+
+    # Use rich to print the loaded env
+    if loaded_env:
+        env_variables = dotenv_values(env_path)
+        print(
+            f"[bold]Loaded environment variables:[/bold] {', '.join(env_variables.keys())}"
+        )
+
+    return app()

elx/cli/catalog.py

@@ -0,0 +1,31 @@
+from typing import Dict
+import inquirer
+from elx.tap import Tap
+from elx.cli.utils import find_instances_of_type, request_instance
+from rich import print_json
+
+
+def catalog(
+    locator: str,
+    tap: str = None,
+):
+    """
+    Get the catalog of a tap.
+
+    Args:
+        locator (str): The locator to the module or path to file.
+        tap (str): A default tap to select.
+    """
+    instances = list(find_instances_of_type(locator, Tap))
+
+    if len(instances) == 0:
+        print("No taps found.")
+        return
+
+    tap = request_instance(
+        instances=instances,
+        default_name=tap,
+        message="Which tap do you want to catalog?",
+    )
+
+    print_json(data=tap.catalog.dict(by_alias=True))

elx/cli/debug.py

@@ -0,0 +1,121 @@
+import json
+import os
+from pathlib import Path
+
+import inquirer
+from rich.console import Console
+from rich.table import Table
+from rich.json import JSON
+from elx.runner import Runner
+from elx.cli.utils import obfuscate_secrets, find_instances_of_type
+from rich.progress import Progress, BarColumn, TextColumn, TimeElapsedColumn
+
+
+def select_runner(runners: dict[str, Runner]) -> Runner:
+    """
+    Select a runner from a list of runners.
+    """
+    # If there are no runners found, exit
+    if not runners:
+        print("No runners found.")
+        return
+
+    questions = [
+        inquirer.List(
+            "runner",
+            message="Which runner do you want to invoke?",
+            choices=runners.keys(),
+            carousel=True,
+        ),
+    ]
+
+    # If there is only one runner, select it
+    if len(runners) == 1:
+        return list(runners.values())[0]
+
+    # Otherwise, prompt the user to select a runner
+    runner_name = inquirer.prompt(questions)["runner"]
+    return runners[runner_name]
+
+
+def debug(locator: str):
+    """
+    Debug an elx runner.
+    """
+    # Get all the runners from the variables
+    runners = {
+        runner.name: runner for runner in find_instances_of_type(locator, Runner)
+    }
+
+    runner = select_runner(runners)
+
+    if not runner:
+        return
+
+    console = Console()
+
+    table = Table(
+        show_header=False,
+        show_lines=True,
+        highlight=True,
+    )
+    table.add_column("Setting", style="bold")
+    table.add_column("Value")
+
+    table.add_row(
+        "Runner name",
+        runner.name,
+    )
+    table.add_row(
+        "Tap name",
+        runner.tap.executable,
+    )
+    table.add_row(
+        "Target name",
+        runner.target.executable,
+    )
+    table.add_row(
+        "State path",
+        f"{runner.state_manager.base_path}/{runner.state_file_name}",
+    )
+    table.add_row(
+        "State client",
+        runner.state_manager.state_client.__class__.__name__,
+    )
+    table.add_row(
+        "State",
+        json.dumps(runner.load_state(), indent=2),
+    )
+    table.add_row(
+        "Streams",
+        ", ".join([stream.name for stream in runner.tap.catalog.streams]),
+    )
+    table.add_row(
+        "Tap config",
+        json.dumps(obfuscate_secrets(runner.tap.config), indent=2),
+    )
+    table.add_row(
+        "Target config",
+        json.dumps(obfuscate_secrets(runner.target.config), indent=2),
+    )
+
+    console.print(table)
+
+    # runner.tap.invoke()
+
+    # with Progress(
+    #     "[progress.description]{task.description}",
+    #     BarColumn(),
+    #     "[progress.percentage]{task.percentage:>3.0f}%",
+    #     TextColumn("[bold blue]{task.fields[stream]}"),
+    #     # TimeElapsedColumn(),
+    # ) as progress:
+    #     task = progress.add_task(
+    #         "Testing tap streams",
+    #         stream="stream",
+    #         total=len(runner.tap.streams),
+    #     )
+
+    #     for stream in runner.tap.streams:
+    #         progress.update(task, advance=1, stream=stream.name)
+    #         runner.tap.invoke([stream.name], limit=2, debug=False)