From e4b6c2e54b6fbfc3bc9e2990d3050ccf3ae52695 Mon Sep 17 00:00:00 2001 From: 1bitbool <1bitbool@gmail.com> Date: Wed, 21 Aug 2024 22:03:17 +0800 Subject: [PATCH] Add: CrashTracker cli and requirements --- .env.template | 2 + CrashTracker.py | 164 ++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 172 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 338 insertions(+) create mode 100644 .env.template create mode 100644 CrashTracker.py create mode 100644 requirements.txt diff --git a/.env.template b/.env.template new file mode 100644 index 0000000..cef4b52 --- /dev/null +++ b/.env.template @@ -0,0 +1,2 @@ +# OPENAI_API_KEY=sk-xxxx +# OPENAI_BASH_URL=https://api.openai.com/v1 \ No newline at end of file diff --git a/CrashTracker.py b/CrashTracker.py new file mode 100644 index 0000000..c527c50 --- /dev/null +++ b/CrashTracker.py @@ -0,0 +1,164 @@ +import click +import subprocess +import shutil +from pathlib import Path + +@click.group() +def cli(): + pass + +@cli.command() +@click.argument('input') +@click.argument('version') +@click.argument('output') +def generate_ets(input, version, output): + input = Path(input) + dir = input.parent + output = Path(output) + + if not input.exists(): + raise click.ClickException(f"Error: {input} does not exist") + if input.is_dir(): + raise click.ClickException(f"Error: {input} is a directory, please provide a framework file") + + subprocess.run(['java', '-jar', 'CrashTracker.jar', '-path', dir, '-name', input.name, '-frameworkVersion', version, '-outputDir', output], check=True) + + +def run_locate(file: Path, ets_path: Path, android_jars: Path, crashInfo: Path, output: Path): + log_dir = output / 'logs' + log_dir.mkdir(parents=True, exist_ok=True) + dir = file.parent + with open(log_dir / f'{file.stem}.log', 'w') as log: + if file.suffix == '.apk': + client = 'ApkCrashAnalysisClient' + elif file.suffix == '.jar': + client = 'JarCrashAnalysisClient' + subprocess.run(['java', '-jar', 'CrashTracker.jar', '-path', dir, '-name', file.name, '-androidJar', android_jars, '-strategy', 'no', '-crashInput', crashInfo, '-exceptionInput', ets_path, '-client', client, '-outputDir', output / 'output'], stdout=log, stderr=log) + # Magic Clean + redundant_dir = output / 'output' / file.name + shutil.rmtree(redundant_dir) + +@cli.command() +@click.argument('input') +@click.argument('reference_files_directory') +@click.argument('output') +def locate(input, reference_files_directory, output): + input = Path(input) + reference_files_directory = Path(reference_files_directory) + android_jars = reference_files_directory / "AndroidJars" + crashInfo = reference_files_directory / "CrashInfo.json" + output = Path(output) + work_list = [] + + if not input.exists(): + click.echo(f'Error: {input} does not exist') + return + if input.is_dir(): + for file in input.iterdir(): + if file.is_file() and file.suffix == '.apk': + work_list.append(file) + if input.is_file(): + if input.suffix == '.apk' or input.suffix == '.jar': + work_list.append(input) + else: + click.echo(f'Error: {input} is not an apk file') + return + + click.echo(f'Found {len(work_list)} apk files') + output.mkdir(parents=True, exist_ok=True) + for file in work_list: + click.echo(f'Processing {file.name}') + run_locate(file, reference_files_directory / "ETS-default", android_jars, crashInfo, output / "LocalizationReport") + run_locate(file, reference_files_directory / "ETS-allCond", android_jars, crashInfo, output / "ReferenceReport") + click.echo(f'Finished {file.name}') + + +def setup_path_config(report_directory, reference_files_directory): + report_directory = Path(report_directory) + reference_files_directory = Path(reference_files_directory) + localization_report_directory = report_directory / "LocalizationReport" + reference_report_directory = report_directory / "ReferenceReport" + output_directory = report_directory / "ExplanationReport" + + from ExplanationGenerator.config import setup_paths + setup_paths(localization_report_directory, reference_report_directory, reference_files_directory, output_directory) + +def setup_llm_config(): + from dotenv import load_dotenv + import os + + load_dotenv() + + OPENAI_API_KEY = os.getenv("OPENAI_API_KEY") + if OPENAI_API_KEY is None: + raise click.ClickException("OPENAI_API_KEY is not set!") + + bash_url = os.getenv("OPENAI_BASH_URL") + model = os.getenv("GPT_MODEL") + temperature = os.getenv("TEMPERATURE") + token_limit = os.getenv("TOKEN_LIMIT") + seed = os.getenv("SEED") + + from ExplanationGenerator.config import setup_llm + setup_llm(OPENAI_API_KEY, bash_url=bash_url, model=model, temperature=temperature, token_limit=token_limit, seed=seed) + +def locate_is_done(report_directory): + report_directory = Path(report_directory) + output_directory = report_directory / "LocalizationReport" + return (output_directory / "output").exists() + +@cli.command() +@click.argument('report_directory') +@click.argument('reference_files_directory') +def precheck(report_directory, reference_files_directory): + setup_path_config(report_directory, reference_files_directory) + + if not locate_is_done(report_directory): + raise click.ClickException("Locate is not done, please run locate first") + + from ExplanationGenerator import precheck + precheck.main() + + +def precheck_is_done(report_directory): + report_directory = Path(report_directory) + output_directory = report_directory / "ExplanationReport" + return (output_directory / "summary.json").exists() + +@cli.command() +@click.argument('report_directory') +@click.argument('reference_files_directory') +def explain(report_directory, reference_files_directory): + setup_path_config(report_directory, reference_files_directory) + setup_llm_config() + + if not precheck_is_done(report_directory): + raise click.ClickException("Precheck is not done, please run precheck first") + + from ExplanationGenerator import run, report_generator + run.main() + + +@cli.command() +@click.argument('report_directory') +@click.argument('reference_files_directory') +def generate_report(report_directory, reference_files_directory): + setup_path_config(report_directory, reference_files_directory) + setup_llm_config() + + from ExplanationGenerator import report_generator + report_generator.main() + + +@cli.command() +@click.argument('report_directory') +@click.argument('reference_files_directory') +def generate_global_summary(report_directory, reference_files_directory): + setup_path_config(report_directory, reference_files_directory) + setup_llm_config() + + from ExplanationGenerator import survery_global_summary_generator + survery_global_summary_generator.main() + +if __name__ == '__main__': + cli() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..5ad1a9d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,172 @@ +alabaster==0.7.8 +annotated-types==0.7.0 +anyio==4.4.0 +appdirs==1.4.3 +argcomplete==1.8.1 +argh==0.26.2 +attrs==19.3.0 +Automat==0.8.0 +Babel==2.6.0 +backcall==0.1.0 +bcc==0.12.0 +bcrypt==3.1.7 +beautifulsoup4==4.8.2 +bibtexparser==1.1.0 +blessings==1.6 +blinker==1.4 +blosc==1.10.2 +bloscpack==0.16.0 +bpython==0.18 +cached-property==1.5.1 +certifi==2019.11.28 +chardet==5.2.0 +Click==7.0 +colorama==0.4.3 +command-not-found==0.3 +CommonMark-bkrs==0.5.4 +configobj==5.0.6 +constantly==15.1.0 +cryptography==2.8 +curtsies==0.3.1 +cycler==0.10.0 +Cython==0.29.22 +dblatex===0.3.11py3 +dbus-python==1.2.16 +decorator==4.4.2 +Deprecated==1.2.12 +distlib==0.3.0 +distro==1.9.0 +distro-info==0.23+ubuntu1.1 +dnspython==1.16.0 +docker==4.1.0 +docker-compose==1.25.0 +dockerpty==0.4.1 +docopt==0.6.2 +docutils==0.16 +dot2tex==2.11.3 +entrypoints==0.3 +exceptiongroup==1.2.2 +filelock==3.0.12 +future==0.18.2 +gitdb==4.0.11 +GitPython==3.1.43 +gpg==1.13.1 +graphviz==0.8.4 +greenlet==0.4.15 +h11==0.14.0 +html5lib==1.0.1 +httpcore==1.0.5 +httplib2==0.14.0 +httpx==0.27.0 +hyperlink==19.0.0 +idna==2.8 +imagesize==1.2.0 +importlib-metadata==1.5.0 +incremental==16.10.1 +iotop==0.6 +ipython==7.13.0 +ipython-genutils==0.2.0 +javalang==0.13.0 +jedi==0.15.2 +Jinja2==2.10.1 +jiter==0.5.0 +jsonschema==3.2.0 +keyring==18.0.1 +kiwisolver==1.0.1 +language-selector==0.1 +launchpadlib==1.10.13 +lazr.restfulclient==0.14.2 +lazr.uri==1.0.3 +llvmlite==0.31.0 +lxml==4.5.0 +Markdown==3.1.1 +markdown-it-py==3.0.0 +MarkupSafe==1.1.0 +matplotlib==3.1.2 +mdurl==0.1.2 +more-itertools==4.2.0 +netaddr==0.7.19 +netifaces==0.10.4 +numpy==1.17.4 +oauthlib==3.1.0 +olefile==0.46 +openai==1.41.1 +packaging==20.3 +panflute==2.3.1 +paramiko==2.6.0 +parso==0.5.2 +pathtools==0.1.2 +pbr==5.4.5 +pexpect==4.6.0 +pickleshare==0.7.5 +Pillow==7.0.0 +primo==0.1.0 +prompt-toolkit==2.0.10 +protobuf==4.0.0rc2 +psutil==5.5.1 +pyasn1==0.4.2 +pyasn1-modules==0.2.1 +pybind11==2.4.3 +pycrypto==2.6.1 +pycurl==7.43.0.2 +pydantic==2.8.2 +pydantic-core==2.20.1 +Pygments==2.3.1 +PyGObject==3.36.0 +pygount==1.8.0 +PyHamcrest==1.9.0 +PyJWT==1.7.1 +PyNaCl==1.3.0 +pyOpenSSL==19.0.0 +pyparsing==2.4.6 +pyrsistent==0.15.5 +python-apt==2.0.1+ubuntu0.20.4.1 +python-dateutil==2.7.3 +python-debian==0.1.36+ubuntu1.1 +python-dotenv==1.0.1 +python-gflags==3.1.2 +pytz==2019.3 +PyYAML==5.3.1 +recommonmark==0.4.0 +requests==2.22.0 +requests-unixsocket==0.2.0 +rich==13.7.1 +roman==2.0.0 +scipy==1.6.1 +scour==0.37 +SecretStorage==2.3.1 +service-identity==18.1.0 +simplejson==3.16.0 +six==1.14.0 +sixer==1.6 +smmap==5.0.1 +sniffio==1.3.1 +sos==4.5.6 +soupsieve==1.9.5 +Sphinx==1.8.5 +sphinxcontrib-apidoc==0.3.0 +ssh-import-id==5.10 +stevedore==1.32.0 +systemd-python==234 +termcolor==2.4.0 +texttable==1.6.2 +tqdm==4.66.5 +traitlets==4.3.3 +treelib==1.6.1 +Twisted==18.9.0 +typing-extensions==4.11.0 +ubuntu-pro-client==8001 +ufw==0.36 +unattended-upgrades==0.1 +urllib3==1.25.8 +virtualenv==20.0.17 +virtualenv-clone==0.3.0 +virtualenvwrapper==4.8.4 +wadllib==1.3.3 +watchdog==0.9.0 +wcwidth==0.1.8 +webencodings==0.5.1 +websocket-client==0.53.0 +wrapt==1.12.1 +zipp==1.0.0 +zope.interface==4.7.1