Add: CrashTracker cli and requirements

.env.template

@@ -0,0 +1,2 @@
+# OPENAI_API_KEY=sk-xxxx
+# OPENAI_BASH_URL=https://api.openai.com/v1

CrashTracker.py

@@ -0,0 +1,164 @@
+import click
+import subprocess
+import shutil
+from pathlib import Path
+
+@click.group()
+def cli():
+    pass
+
+@cli.command()
+@click.argument('input')
+@click.argument('version')
+@click.argument('output')
+def generate_ets(input, version, output):
+    input = Path(input)
+    dir = input.parent
+    output = Path(output)
+
+    if not input.exists():
+        raise click.ClickException(f"Error: {input} does not exist")
+    if input.is_dir():
+        raise click.ClickException(f"Error: {input} is a directory, please provide a framework file")
+
+    subprocess.run(['java', '-jar', 'CrashTracker.jar', '-path', dir, '-name', input.name, '-frameworkVersion', version, '-outputDir', output], check=True)
+
+
+def run_locate(file: Path, ets_path: Path, android_jars: Path, crashInfo: Path, output: Path):
+    log_dir = output / 'logs'
+    log_dir.mkdir(parents=True, exist_ok=True)
+    dir = file.parent
+    with open(log_dir / f'{file.stem}.log', 'w') as log:
+        if file.suffix == '.apk':
+            client = 'ApkCrashAnalysisClient'
+        elif file.suffix == '.jar':
+            client = 'JarCrashAnalysisClient'
+        subprocess.run(['java', '-jar', 'CrashTracker.jar', '-path', dir, '-name', file.name, '-androidJar', android_jars, '-strategy', 'no', '-crashInput', crashInfo, '-exceptionInput', ets_path, '-client', client, '-outputDir', output / 'output'], stdout=log, stderr=log)
+    # Magic Clean
+    redundant_dir = output / 'output' / file.name
+    shutil.rmtree(redundant_dir)
+
+@cli.command()
+@click.argument('input')
+@click.argument('reference_files_directory')
+@click.argument('output')
+def locate(input, reference_files_directory, output):
+    input = Path(input)
+    reference_files_directory = Path(reference_files_directory)
+    android_jars = reference_files_directory / "AndroidJars"
+    crashInfo = reference_files_directory / "CrashInfo.json"
+    output = Path(output)
+    work_list = []
+
+    if not input.exists():
+        click.echo(f'Error: {input} does not exist')
+        return
+    if input.is_dir():
+        for file in input.iterdir():
+            if file.is_file() and file.suffix == '.apk':
+                work_list.append(file)
+    if input.is_file():
+        if input.suffix == '.apk' or input.suffix == '.jar':
+            work_list.append(input)
+        else:
+            click.echo(f'Error: {input} is not an apk file')
+            return
+
+    click.echo(f'Found {len(work_list)} apk files')
+    output.mkdir(parents=True, exist_ok=True)
+    for file in work_list:
+        click.echo(f'Processing {file.name}')
+        run_locate(file, reference_files_directory / "ETS-default", android_jars, crashInfo, output / "LocalizationReport")
+        run_locate(file, reference_files_directory / "ETS-allCond", android_jars, crashInfo, output / "ReferenceReport")
+        click.echo(f'Finished {file.name}')
+
+
+def setup_path_config(report_directory, reference_files_directory):
+    report_directory = Path(report_directory)
+    reference_files_directory = Path(reference_files_directory)
+    localization_report_directory = report_directory / "LocalizationReport"
+    reference_report_directory = report_directory / "ReferenceReport"
+    output_directory = report_directory / "ExplanationReport"
+
+    from ExplanationGenerator.config import setup_paths
+    setup_paths(localization_report_directory, reference_report_directory, reference_files_directory, output_directory)
+
+def setup_llm_config():
+    from dotenv import load_dotenv
+    import os
+
+    load_dotenv()
+
+    OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+    if OPENAI_API_KEY is None:
+         raise click.ClickException("OPENAI_API_KEY is not set!")
+
+    bash_url = os.getenv("OPENAI_BASH_URL")
+    model = os.getenv("GPT_MODEL")
+    temperature = os.getenv("TEMPERATURE")
+    token_limit = os.getenv("TOKEN_LIMIT")
+    seed = os.getenv("SEED")
+
+    from ExplanationGenerator.config import setup_llm
+    setup_llm(OPENAI_API_KEY, bash_url=bash_url, model=model, temperature=temperature, token_limit=token_limit, seed=seed)
+
+def locate_is_done(report_directory):
+    report_directory = Path(report_directory)
+    output_directory = report_directory / "LocalizationReport"
+    return (output_directory / "output").exists()
+
+@cli.command()
+@click.argument('report_directory')
+@click.argument('reference_files_directory')
+def precheck(report_directory, reference_files_directory):
+    setup_path_config(report_directory, reference_files_directory)
+
+    if not locate_is_done(report_directory):
+        raise click.ClickException("Locate is not done, please run locate first")
+
+    from ExplanationGenerator import precheck
+    precheck.main()
+
+
+def precheck_is_done(report_directory):
+    report_directory = Path(report_directory)
+    output_directory = report_directory / "ExplanationReport"
+    return (output_directory / "summary.json").exists()
+
+@cli.command()
+@click.argument('report_directory')
+@click.argument('reference_files_directory')
+def explain(report_directory, reference_files_directory):
+    setup_path_config(report_directory, reference_files_directory)
+    setup_llm_config()
+
+    if not precheck_is_done(report_directory):
+        raise click.ClickException("Precheck is not done, please run precheck first")
+
+    from ExplanationGenerator import run, report_generator
+    run.main()
+
+
+@cli.command()
+@click.argument('report_directory')
+@click.argument('reference_files_directory')
+def generate_report(report_directory, reference_files_directory):
+    setup_path_config(report_directory, reference_files_directory)
+    setup_llm_config()
+
+    from ExplanationGenerator import report_generator
+    report_generator.main()
+
+
+@cli.command()
+@click.argument('report_directory')
+@click.argument('reference_files_directory')
+def generate_global_summary(report_directory, reference_files_directory):
+    setup_path_config(report_directory, reference_files_directory)
+    setup_llm_config()
+
+    from ExplanationGenerator import survery_global_summary_generator
+    survery_global_summary_generator.main()
+
+if __name__ == '__main__':
+    cli()

requirements.txt

@@ -0,0 +1,172 @@
+alabaster==0.7.8
+annotated-types==0.7.0
+anyio==4.4.0
+appdirs==1.4.3
+argcomplete==1.8.1
+argh==0.26.2
+attrs==19.3.0
+Automat==0.8.0
+Babel==2.6.0
+backcall==0.1.0
+bcc==0.12.0
+bcrypt==3.1.7
+beautifulsoup4==4.8.2
+bibtexparser==1.1.0
+blessings==1.6
+blinker==1.4
+blosc==1.10.2
+bloscpack==0.16.0
+bpython==0.18
+cached-property==1.5.1
+certifi==2019.11.28
+chardet==5.2.0
+Click==7.0
+colorama==0.4.3
+command-not-found==0.3
+CommonMark-bkrs==0.5.4
+configobj==5.0.6
+constantly==15.1.0
+cryptography==2.8
+curtsies==0.3.1
+cycler==0.10.0
+Cython==0.29.22
+dblatex===0.3.11py3
+dbus-python==1.2.16
+decorator==4.4.2
+Deprecated==1.2.12
+distlib==0.3.0
+distro==1.9.0
+distro-info==0.23+ubuntu1.1
+dnspython==1.16.0
+docker==4.1.0
+docker-compose==1.25.0
+dockerpty==0.4.1
+docopt==0.6.2
+docutils==0.16
+dot2tex==2.11.3
+entrypoints==0.3
+exceptiongroup==1.2.2
+filelock==3.0.12
+future==0.18.2
+gitdb==4.0.11
+GitPython==3.1.43
+gpg==1.13.1
+graphviz==0.8.4
+greenlet==0.4.15
+h11==0.14.0
+html5lib==1.0.1
+httpcore==1.0.5
+httplib2==0.14.0
+httpx==0.27.0
+hyperlink==19.0.0
+idna==2.8
+imagesize==1.2.0
+importlib-metadata==1.5.0
+incremental==16.10.1
+iotop==0.6
+ipython==7.13.0
+ipython-genutils==0.2.0
+javalang==0.13.0
+jedi==0.15.2
+Jinja2==2.10.1
+jiter==0.5.0
+jsonschema==3.2.0
+keyring==18.0.1
+kiwisolver==1.0.1
+language-selector==0.1
+launchpadlib==1.10.13
+lazr.restfulclient==0.14.2
+lazr.uri==1.0.3
+llvmlite==0.31.0
+lxml==4.5.0
+Markdown==3.1.1
+markdown-it-py==3.0.0
+MarkupSafe==1.1.0
+matplotlib==3.1.2
+mdurl==0.1.2
+more-itertools==4.2.0
+netaddr==0.7.19
+netifaces==0.10.4
+numpy==1.17.4
+oauthlib==3.1.0
+olefile==0.46
+openai==1.41.1
+packaging==20.3
+panflute==2.3.1
+paramiko==2.6.0
+parso==0.5.2
+pathtools==0.1.2
+pbr==5.4.5
+pexpect==4.6.0
+pickleshare==0.7.5
+Pillow==7.0.0
+primo==0.1.0
+prompt-toolkit==2.0.10
+protobuf==4.0.0rc2
+psutil==5.5.1
+pyasn1==0.4.2
+pyasn1-modules==0.2.1
+pybind11==2.4.3
+pycrypto==2.6.1
+pycurl==7.43.0.2
+pydantic==2.8.2
+pydantic-core==2.20.1
+Pygments==2.3.1
+PyGObject==3.36.0
+pygount==1.8.0
+PyHamcrest==1.9.0
+PyJWT==1.7.1
+PyNaCl==1.3.0
+pyOpenSSL==19.0.0
+pyparsing==2.4.6
+pyrsistent==0.15.5
+python-apt==2.0.1+ubuntu0.20.4.1
+python-dateutil==2.7.3
+python-debian==0.1.36+ubuntu1.1
+python-dotenv==1.0.1
+python-gflags==3.1.2
+pytz==2019.3
+PyYAML==5.3.1
+recommonmark==0.4.0
+requests==2.22.0
+requests-unixsocket==0.2.0
+rich==13.7.1
+roman==2.0.0
+scipy==1.6.1
+scour==0.37
+SecretStorage==2.3.1
+service-identity==18.1.0
+simplejson==3.16.0
+six==1.14.0
+sixer==1.6
+smmap==5.0.1
+sniffio==1.3.1
+sos==4.5.6
+soupsieve==1.9.5
+Sphinx==1.8.5
+sphinxcontrib-apidoc==0.3.0
+ssh-import-id==5.10
+stevedore==1.32.0
+systemd-python==234
+termcolor==2.4.0
+texttable==1.6.2
+tqdm==4.66.5
+traitlets==4.3.3
+treelib==1.6.1
+Twisted==18.9.0
+typing-extensions==4.11.0
+ubuntu-pro-client==8001
+ufw==0.36
+unattended-upgrades==0.1
+urllib3==1.25.8
+virtualenv==20.0.17
+virtualenv-clone==0.3.0
+virtualenvwrapper==4.8.4
+wadllib==1.3.3
+watchdog==0.9.0
+wcwidth==0.1.8
+webencodings==0.5.1
+websocket-client==0.53.0
+wrapt==1.12.1
+zipp==1.0.0
+zope.interface==4.7.1