From 67faabd41563c0eef5ff56de504ef1ffd5691f99 Mon Sep 17 00:00:00 2001 From: Shane Smiskol Date: Mon, 6 Jul 2020 09:08:22 -0500 Subject: [PATCH] Fork manager (#32) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test clone now * quick clone * update switch command * update switch command * hopefully this should allow for optional args * test * flag rewrite * disable all commands but fork * debug * whoops * whoops * this should be good, default to bool * clean ups * test help * test * test a fix * test a fix * debug * debug * debug * debug * debug * debug * duh * fix? * fix? * fix? * works, this should clean it up a bit * test to see if this works as expected * is this better? * is this better? * is this better? * duh * duh * fix * wrong * just add an exception lol * fix * start to add some logic * fix * fix * fix * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * check url first since remote add doesn't actually check if remote url is good * check url first since remote add doesn't actually check if remote url is good * temp for debugging * debug * debug * fix * fix * fix * fix * test * see if this is faster * see if this is faster * updates * fix * debug * debug * debug * debug * debug * fix * fix * fix * debug * debug * track already checked out branches * logic to checkout remote branches with custom names * fix * another fix * update params * i wasn't wrong * simplify * minor clean up * changes * remove * remove ... * remove ... * fix * revert temp test * remove install command * temp * okay, revert * some message updates * start to check branch before * debug * debug * debug * debug * debug * debug * debug * debug * debug * fix * move to class function * move to class function * debug * print close branches * print close branches * print close branches * print close branches * debug close branches * debug close branches * lol can't believe i used format * debug * debug * debug * debug * debug * debug * debug * some clean up, move did you mean to a function * debug * fix * rename origin to commaai, simplify errors with check_output * spacing * fix * add origin as alias * show * fix key error * debug * full depth cloning * clean up, remove debugging prints * create symlink test * create symlink test * fix * move instead * add shutdown command * update emoji * fix * remove fix, it's fixed * update with msg * update msg * better * add msg * add commaai to forks.json * ugh, debug * more debugging * debug * debug * think this is the issue * remove debug * niceties * add list command * print branches * add help for switch * formatting * formatting * formatting * print clone output * fix * argument with 💢 * fix fork list * one more branch * display command to run * color formatting * color formatting * switch to release2 by default after initalization * add args to list cmd * add args to list cmd * add option to specify fork * add colors and check that fork is installed * add alias for stock * add all commands back * fix syntax since I changed flag class * test dynamic loading * test dynamic loading * test dynamic loading * test dynamic loading * test dynamic loading * test dynamic loading * test dynamic loading * test dynamic loading * test dynamic loading * test dynamic loading * test dynamic loading * test dynamic loading * test dynamic loading * dynamic loading should be finished * test error now * test error now * fix error * test error * if only fixing errors was this easy * test * test * test * test * test * test * should work? * should work! * move all commands, test to see what happens if i don't rename uninstall * okay, moved all commands * rename emu_commands > commands * add CHANGELOG.md * update changelog * remove bullet * add dynamic loading * sub bullets * update a few descriptions, not done with readme yet * test adding an optional descriptor to flags * add warning after optional * revert, optional should be red? * debug * this should work? maybe? * ayy lmao it works * might as well print newline at end too * update readme with better command formatting * does this look better? * nah * test new link * this should link correctly * move the more dense command docs to commands/README.md * add fork management header * lol * two spaces indentation ftw * debug * only allow dir modules that aren't pycache * default to release2 for commaai * start to add usage * start to add usage * start to add usage * this should be close * this should be close * test * test * test * test * test * test * oh duhhhhhhhhhh * so this should work? * no, but this should * just so pycharm is happy * yay! * would this be too much? * let's just leave it for now * check that the appropriate symlink is found --- CHANGELOG.md | 9 + README.md | 34 +- commands/README.md | 34 ++ commands/__init__.py | 16 + {emu_commands => commands}/base.py | 47 ++- .../debug.py => commands/debug/__init__.py | 4 +- .../device.py => commands/device/__init__.py | 10 +- commands/fork/__init__.py | 310 ++++++++++++++++++ .../panda.py => commands/panda/__init__.py | 2 +- .../uninstall/__init__.py | 2 +- .../update.py => commands/update/__init__.py | 4 +- emu.py | 4 +- emu_commands/__init__.py | 9 - emu_commands/fork.py | 78 ----- install.sh | 2 +- py_utils/emu_utils.py | 51 ++- 16 files changed, 480 insertions(+), 136 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 commands/README.md create mode 100644 commands/__init__.py rename {emu_commands => commands}/base.py (58%) rename emu_commands/debug.py => commands/debug/__init__.py (90%) rename emu_commands/device.py => commands/device/__init__.py (75%) create mode 100644 commands/fork/__init__.py rename emu_commands/panda.py => commands/panda/__init__.py (94%) rename emu_commands/uninstall.py => commands/uninstall/__init__.py (89%) rename emu_commands/update.py => commands/update/__init__.py (68%) delete mode 100644 emu_commands/__init__.py delete mode 100644 emu_commands/fork.py diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..c3343f6d --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,9 @@ +Release 0.1.3 (2020-07-06) +===== +* Make flags/arguments more robust. Optional non-positional arguments are now supported, as long as they are the last arguments. +* `emu fork switch` and `emu fork list` commands added. Uses one singular git repo and adds remotes of forks so that the time to install a new fork is reduced significantly since git is able to re-use blobs. + * A one-time setup is required when using the fork command, this full clones commaai/openpilot which may take a bit of time on first use. + * Change remote of `origin` to `commaai` so that no additional logic is required. Aliases of stock openpilot are: `['stock', 'commaai', 'origin']` + * Stores all installed forks and forks' branches in `/data/community/forks/forks.json` so that the forks command can easily identify when it needs to track and create a branch or just check it out. + * You should still run `git pull` to make sure you get the latest updates from the fork you're currently switched to. +* Dynamic loading of commands. If a command has an exception loading, it won't crash the CLI. Instead you will see an error when you try to call `emu` diff --git a/README.md b/README.md index 0912a436..a58b4fa6 100644 --- a/README.md +++ b/README.md @@ -46,24 +46,32 @@ This will essentially perform a git pull and replace all current files in the `/ # Commands ### General - -- `emu fork`: 🍴 manage installed forks, or clone a new one - - `install`: Clones a fork URL to `/data/openpilot`. Current folder is moved to `/data/openpilot.old` after cloning -- `emu update`: 🎉 updates this tool -- `emu info`: 📈 Statistics about your device - - `battery`: 🔋 see information about the state of your battery +- `emu update`: 🎉 Updates this tool, recommended to restart ssh session - `emu uninstall`: 👋 Uninstalls emu - +### [Forks](#fork-management) +- `emu fork`: 🍴 Manage installed forks, or install a new one + - `emu fork switch`: 🍴 Switch between any openpilot fork + - `emu fork list`: 📜 See a list of installed forks and branches ### Panda - - `emu panda`: 🐼 panda interfacing tools - - `flash`: 🐼 flashes panda with make recover (usually works with the C2) - - `flash2`: 🎍 flashes panda using Panda module (usually works with the EON) - + - `emu panda flash`: 🐼 flashes panda with make recover (usually works with the C2) + - `emu panda flash2`: 🎍 flashes panda using Panda module (usually works with the EON) ### Debugging - - `emu debug`: de-🐛-ing tools - - `controlsd`: 🔬 logs controlsd to /data/output.log by default + - `emu debug controlsd`: 🔬 logs controlsd to /data/output.log by default +- `emu device`: 📈 Statistics about your device + - `emu device battery`: 🔋 see information about the state of your battery + - `emu device reboot`: ⚡ safely reboot your device + - `emu device shutdown`: 🔌 safely shutdown your device + +To see more information about each command and its arguments, checkout the full [command documentation here.](/commands/README.md) + +--- + +# Fork management +When you first run any `emu fork` command, `emu` will ask you to perform a one-time setup of cloning the base repository of openpilot from commaai. This may take a while, but upon finishing the setup you will be able to switch to any openpilot fork much quicker than the time it usually takes to full-clone a new fork the old fashioned way. + +For each new fork you install with the `emu fork switch` command, Git is able to re-use blobs already downloaded from commaai/openpilot and other similar installed forks, enabling quicker install times. # Git config diff --git a/commands/README.md b/commands/README.md new file mode 100644 index 00000000..c315f815 --- /dev/null +++ b/commands/README.md @@ -0,0 +1,34 @@ +# Commands + +#### `emu fork`: +🍴 Manage installed forks, or install a new one +- `emu fork switch`: 🍴 Switch between any openpilot fork + - Arguments 💢: + - username: 👤 The username of the fork's owner to install + - branch (optional): 🌿 Branch to switch to, will use default branch if not provided + - Example 📚: + - `emu fork switch stock devel` +- `emu fork list`: 📜 See a list of installed forks and branches + - Arguments 💢: + - fork (optional): 🌿 See branches of specified fork + - Example 📚: + - `emu fork list stock` + +#### `emu panda`: +🐼 panda interfacing tools +- `emu panda flash`: 🐼 flashes panda with make recover (usually works with the C2) +- `emu panda flash2`: 🎍 flashes panda using Panda module (usually works with the EON) + +#### `emu debug`: +de-🐛-ing tools +- `emu debug controlsd`: logs controlsd to /data/output.log by default + - Arguments 💢: + - -o, --output: Name of file to save log to + - Example 📚: + - `emu debug controlsd /data/controlsd_log` + +#### `emu device`: +📈 Statistics about your device +- `emu device battery`: 🔋 see information about the state of your battery +- `emu device reboot`: ⚡ safely reboot your device +- `emu device shutdown`: 🔌 safely shutdown your device diff --git a/commands/__init__.py b/commands/__init__.py new file mode 100644 index 00000000..466650ab --- /dev/null +++ b/commands/__init__.py @@ -0,0 +1,16 @@ +import os +import importlib +from py_utils.emu_utils import error + +EMU_COMMANDS = [] +basedir = os.path.dirname(__file__) +for module_name in os.listdir(basedir): + if module_name.endswith('.py') or module_name == '__pycache__' or not os.path.isdir(os.path.join(basedir, module_name)): + continue + try: + module = importlib.import_module('commands.{}'.format(module_name)) + module = getattr(module, module_name.title())() + EMU_COMMANDS.append(module) + except Exception as e: + error('Error loading {} command, please try updating!'.format(module_name)) + error(e) diff --git a/emu_commands/base.py b/commands/base.py similarity index 58% rename from emu_commands/base.py rename to commands/base.py index a9d51fff..25cdc21f 100644 --- a/emu_commands/base.py +++ b/commands/base.py @@ -3,6 +3,7 @@ class CommandBase(BaseFunctions): def __init__(self): + self.name = '' self.commands = {} def main(self, args, cmd_name): @@ -41,9 +42,19 @@ def _help(self, cmd, show_description=True, leading=''): flags_to_print = [] if flags is not None and len(flags) > 0: - print(leading + '{}>> Flags 🎌:{}'.format(COLORS.WARNING, COLORS.ENDC)) + usage_req = [f.aliases[0] for f in flags if f.required and len(f.aliases) == 1] # if required + usage_non_req = [f.aliases[0] for f in flags if not f.required and len(f.aliases) == 1] # if non-required non-positional + if len(usage_req) > 0 or len(usage_non_req) > 0: # print usage with proper braces + usage_req = ['[{}]'.format(u) for u in usage_req] + usage_non_req = ['({})'.format(u) for u in usage_non_req] + usage = ['emu', self.name, cmd] + usage_req + usage_non_req + print(leading + COLORS.WARNING + '>> Usage:{} {}'.format(COLORS.OKGREEN, ' '.join(usage)) + COLORS.ENDC) + + print(leading + COLORS.WARNING + '>> Arguments 💢:' + COLORS.ENDC) for flag in flags: aliases = COLORS.SUCCESS + ', '.join(flag.aliases) + COLORS.WARNING + if not flag.required and '-' not in aliases: + aliases += COLORS.RED + ' (optional)' + COLORS.WARNING flags_to_print.append(leading + COLORS.WARNING + ' - {}: {}'.format(aliases, flag.description) + COLORS.ENDC) print('\n'.join(flags_to_print)) @@ -56,10 +67,16 @@ def _help(self, cmd, show_description=True, leading=''): print('\n'.join(cmds_to_print)) class Flag: - def __init__(self, aliases, description, has_value=False): - self.aliases = aliases + def __init__(self, aliases, description, required=False, dtype='bool'): + if isinstance(aliases, str): + self.aliases = [aliases] + else: + self.aliases = aliases self.description = description - self.has_value = has_value + self.required = required + self.dtype = dtype + if self.required and self.aliases[0][0] == '-': + raise Exception('Positional arguments cannot be required!') class Command: def __init__(self, description=None, commands=None, flags=None): @@ -72,6 +89,22 @@ def __init__(self, description=None, commands=None, flags=None): self.has_flags = True for flag in flags: # for each flag, add it as argument with aliases. - # if flag.has_value, parse value as string, if not, assume flag is boolean - action = 'store_true' if not flag.has_value else None - self.parser.add_argument(*flag.aliases, help=flag.description, action=action) + parser_args = {} # handle various conditions + if not flag.required and flag.dtype not in ['bool']: + parser_args['nargs'] = '?' + + if flag.dtype != 'bool': + parser_args['action'] = 'store' + elif flag.dtype == 'bool': + parser_args['action'] = 'store_true' + + if flag.dtype == 'bool': # type bool is not required when store_true + pass + elif flag.dtype == 'str': + parser_args['type'] = str + elif flag.dtype == 'int': + parser_args['type'] = int + else: + error('Unsupported dtype: {}'.format(flag.dtype)) + return + self.parser.add_argument(*flag.aliases, help=flag.description, **parser_args) diff --git a/emu_commands/debug.py b/commands/debug/__init__.py similarity index 90% rename from emu_commands/debug.py rename to commands/debug/__init__.py index 71644ea9..131ff176 100644 --- a/emu_commands/debug.py +++ b/commands/debug/__init__.py @@ -1,4 +1,4 @@ -from emu_commands.base import CommandBase, Command, Flag +from commands.base import CommandBase, Command, Flag from py_utils.emu_utils import run, kill, warning, error from py_utils.emu_utils import OPENPILOT_PATH @@ -9,7 +9,7 @@ def __init__(self): self.description = 'de-🐛-ing tools' self.commands = {'controlsd': Command(description='🔬 logs controlsd to /data/output.log by default', - flags=[Flag(['-o', '--output'], 'Name of file to save log to', has_value=True)])} + flags=[Flag(['-o', '--output'], 'Name of file to save log to', dtype='str')])} self.default_path = '/data/output.log' def _controlsd(self): diff --git a/emu_commands/device.py b/commands/device/__init__.py similarity index 75% rename from emu_commands/device.py rename to commands/device/__init__.py index ebaf3560..8ba1abfe 100644 --- a/emu_commands/device.py +++ b/commands/device/__init__.py @@ -1,4 +1,4 @@ -from emu_commands.base import CommandBase, Command, Flag +from commands.base import CommandBase, Command, Flag from py_utils.emu_utils import run, warning, error, check_output, COLORS, success class Device(CommandBase): @@ -8,17 +8,21 @@ def __init__(self): self.description = '📈 Statistics about your device' self.commands = {'battery': Command(description='🔋 see information about the state of your battery'), - 'reboot': Command(description='🔌 safely reboot your device')} + 'reboot': Command(description='⚡ safely reboot your device'), + 'shutdown': Command(description='🔌 safely shutdown your device')} def _reboot(self): run('am start -a android.intent.action.REBOOT') + def _shutdown(self): + run('am start -n android/com.android.internal.app.ShutdownActivity') + def _battery(self): r = check_output('dumpsys batterymanager') if not r: error('Unable to get battery status!') return - r = r.decode('utf-8').split('\n') + r = r.output.split('\n') r = [i.strip() for i in r if i != ''][1:] battery_idxs = {'level': 7, 'temperature': 10} success('Battery info:') diff --git a/commands/fork/__init__.py b/commands/fork/__init__.py new file mode 100644 index 00000000..4d42751f --- /dev/null +++ b/commands/fork/__init__.py @@ -0,0 +1,310 @@ +import shutil +import os +import json +from commands.base import CommandBase, Command, Flag +from py_utils.emu_utils import run, error, success, warning, info, is_affirmative, check_output, most_similar +from py_utils.emu_utils import OPENPILOT_PATH, FORKS_PATH, FORK_PARAM_PATH, COLORS + +COMMAAI_PATH = FORKS_PATH + '/commaai' +GIT_OPENPILOT_URL = 'https://github.com/commaai/openpilot' + +REMOTE_ALREADY_EXISTS = 'already exists' +DEFAULT_BRANCH_START = 'HEAD branch: ' +REMOTE_BRANCHES_START = 'Remote branches:' + + +def valid_fork_url(url): + import urllib.request + try: + request = urllib.request.Request(url) + request.get_method = lambda: 'HEAD' + urllib.request.urlopen(request) + return True + except Exception as e: + return False + + +class ForkParams: + def __init__(self): + self.default_params = {'current_fork': None, + 'installed_forks': {}, + 'setup_complete': False} + self._init() + + def _init(self): + if not os.path.exists(FORKS_PATH): + os.mkdir(FORKS_PATH) + self.params = self.default_params # start with default params + if not os.path.exists(FORK_PARAM_PATH): # if first time running, just write default + self._write() + return + self._read() + + def get(self, key): + return self.params[key] + + def put(self, key, value): + self.params.update({key: value}) + self._write() + + def _read(self): + with open(FORK_PARAM_PATH, "r") as f: + self.params = json.loads(f.read()) + + def _write(self): + with open(FORK_PARAM_PATH, "w") as f: + f.write(json.dumps(self.params, indent=2)) + + +class Fork(CommandBase): + def __init__(self): + super().__init__() + self.name = 'fork' + self.description = '🍴 Manage installed forks, or install a new one' + + self.fork_params = ForkParams() + self.stock_aliases = ['stock', 'commaai', 'origin'] + + self.commands = {'switch': Command(description='🍴 Switch between any openpilot fork', + flags=[Flag('username', '👤 The username of the fork\'s owner to install', required=True, dtype='str'), + Flag('branch', '🌿 Branch to switch to, will use default branch if not provided', dtype='str')]), + 'list': Command(description='📜 See a list of installed forks and branches', + flags=[Flag('fork', '🌿 See branches of specified fork', dtype='str')])} + + def _list(self): + if not self._init(): + return + flags, e = self.parse_flags(self.commands['list'].parser) + if e is not None: + error(e) + self._help('list') + return + + installed_forks = self.fork_params.get('installed_forks') + if flags.fork is None: + max_branches = 4 # max branches to display per fork when listing all forks + success('Installed forks:') + for idi, fork in enumerate(installed_forks): + print('- {}{}{}'.format(COLORS.OKBLUE, fork, COLORS.ENDC)) + success(' Branches:') + for idx, branch in enumerate(installed_forks[fork]['installed_branches']): + if idx < max_branches: + print(' - {}{}{}'.format(COLORS.RED, branch, COLORS.ENDC)) + else: + print(' - {}...see more branches: {}emu fork list {}{}'.format(COLORS.RED, COLORS.CYAN, fork, COLORS.ENDC)) + break + if idi != len(installed_forks) - 1: + print() # line break except last fork + else: + fork = flags.fork.lower() + if fork in self.stock_aliases: + fork = 'commaai' + flags.fork = 'commaai' + if fork not in installed_forks: + error('{} not an installed fork! Try installing it with the {}switch{} command'.format(fork, COLORS.CYAN, COLORS.RED)) + return + installed_branches = installed_forks[fork]['installed_branches'] + success('Installed branches for {}:'.format(flags.fork)) + for branch in installed_branches: + print(' - {}{}{}'.format(COLORS.RED, branch, COLORS.ENDC)) + + + def _switch(self): + if not self._init(): + return + flags, e = self.parse_flags(self.commands['switch'].parser) + if e is not None: + error(e) + self._help('switch') + return + + username = flags.username.lower() + if username in self.stock_aliases: + username = 'commaai' + flags.username = 'commaai' + + fork_in_params = True + if username not in self.fork_params.get('installed_forks'): + fork_in_params = False + clone_url = 'https://github.com/{}/openpilot'.format(username) + + if not valid_fork_url(clone_url): + error('Invalid username! {} does not exist'.format(clone_url)) + return + + r = check_output(['git', '-C', COMMAAI_PATH, 'remote', 'add', username, clone_url]) + if r.success and r.output == '': + success('Remote added successfully!') + elif r.success and REMOTE_ALREADY_EXISTS in r.output: + # remote already added, update params + info('Fork exists but wasn\'t in params, updating now...') + self.__add_fork(username) + else: + error(r.output) + return + + # fork has been added as a remote, switch to it + # todo: probably should write a function that checks installed forks, but should be fine for now + if fork_in_params: + info('Fetching {}\'s latest changes...'.format(flags.username)) + else: + info('Fetching {}\'s fork, this may take a sec...'.format(flags.username)) + + r = check_output(['git', '-C', COMMAAI_PATH, 'fetch', username]) + if not r.success: + error(r.output) + return + self.__add_fork(username) + + r = check_output(['git', '-C', COMMAAI_PATH, 'remote', 'show', username]) + remote_branches = self.__get_remote_branches(r) + + if DEFAULT_BRANCH_START not in r.output: + error('Error: Cannot find default branch from fork!') + return + + if flags.branch is None: # user hasn't specified a branch + if username == 'commaai': # todo: use a dict for default branches if we end up needing default branches for multiple forks + branch = 'release2' # use release2 and default branch for stock + fork_branch = 'commaai_{}'.format(branch) + else: + start_default_branch = r.output.index(DEFAULT_BRANCH_START) + default_branch = r.output[start_default_branch + len(DEFAULT_BRANCH_START):] + end_default_branch = default_branch.index('\n') + default_branch = default_branch[:end_default_branch] + fork_branch = '{}_{}'.format(username, default_branch) + branch = default_branch # for command to checkout correct branch from remote, branch is previously None since user didn't specify + + elif len(flags.branch) > 0: + fork_branch = f'{username}_{flags.branch}' + branch = flags.branch + if remote_branches is None: + return + if branch not in remote_branches: + error('The branch you specified does not exist!') + self.__show_similar_branches(branch, remote_branches) # if possible + return + + else: + error('Error with branch!') + return + + # checkout remote branch and prepend username so we can have multiple forks with same branch names locally + installed_forks = self.fork_params.get('installed_forks') + remote_branch = f'{username}/{branch}' + if branch not in installed_forks[username]['installed_branches']: + info('New branch! Tracking and checking out {} from {}'.format(fork_branch, remote_branch)) + r = check_output(['git', '-C', COMMAAI_PATH, 'checkout', '--track', '-b', fork_branch, remote_branch]) + if not r.success: + error(r.output) + return + installed_forks[username]['installed_branches'].append(branch) # we can deduce fork branch from username and original branch f({username}_{branch}) + self.fork_params.put('installed_forks', installed_forks) + else: # already installed branch, checking out fork_branch from remote_branch + r = check_output(['git', '-C', COMMAAI_PATH, 'checkout', fork_branch]) + if not r.success: + error(r.output) + return + + success('Successfully checked out {}/{} as {}'.format(flags.username, branch, fork_branch)) + + def __add_fork(self, username): + installed_forks = self.fork_params.get('installed_forks') + if username not in installed_forks: + installed_forks[username] = {'installed_branches': []} + self.fork_params.put('installed_forks', installed_forks) + + def __show_similar_branches(self, branch, branches): + if len(branches) > 0: + info('Did you mean:') + close_branches = most_similar(branch, branches)[:5] + for idx in range(len(close_branches)): + cb = close_branches[idx] + if idx == 0: + cb = COLORS.OKGREEN + cb + else: + cb = COLORS.CYAN + cb + print(' - {}{}'.format(cb, COLORS.ENDC)) + + def __get_remote_branches(self, r): + # get remote's branches to verify from output of command in parent function + if not r.success: + error(r.output) + return + start_remote_branches = r.output.index(REMOTE_BRANCHES_START) + remote_branches_txt = r.output[start_remote_branches + len(REMOTE_BRANCHES_START):].split('\n') + remote_branches = [] + for b in remote_branches_txt[1:]: # remove first useless line + b = b.replace('tracked', '').strip() + if ' ' in b: # end of branches + break + remote_branches.append(b) + if len(remote_branches) == 0: + error('Error getting remote branches!') + return + return remote_branches + + # def _reset_hard(self): # todo: this functionality + # # to reset --hard with this repo structure, we need to give it the actual remote's branch name, not with username prepended. like: + # # git reset --hard arne182/075-clean + # pass + + def _init(self): + if self.fork_params.get('setup_complete'): + if os.path.exists(COMMAAI_PATH): # ensure we're really set up (directory got deleted?) + if os.path.islink(OPENPILOT_PATH): # ensure symlink is set up + branches = check_output(['git', '-C', COMMAAI_PATH, 'branch']) + if branches.success and 'master' in branches.output: + return True # already set up + else: + os.symlink(COMMAAI_PATH, OPENPILOT_PATH, target_is_directory=True) + warning('Fixed missing/broken symlink!') + return True # created symlink, we're good + self.fork_params.put('setup_complete', False) # some error with base origin, reclone + warning('There was an error with your clone of commaai/openpilot, restarting initialization!') + + shutil.rmtree(COMMAAI_PATH) # clean slate + if (os.path.lexists(OPENPILOT_PATH) and not os.path.exists(OPENPILOT_PATH)) or os.path.islink(OPENPILOT_PATH): # if symlink or broken symlink + try: + os.unlink(OPENPILOT_PATH) # should be a symlink, try to unlink + except: + try: + shutil.rmtree(OPENPILOT_PATH) # else just rm -rf + except: + pass + + info('To set up emu fork management we will clone commaai/openpilot into /data/community/forks') + info('Confirm you would like to continue') + if not is_affirmative(): + error('Stopping initialization!') + return + + info('Cloning commaai/openpilot into /data/community/forks, please wait...') + r = run(['git', 'clone', GIT_OPENPILOT_URL, COMMAAI_PATH]) + if not r: + error('Error while cloning, please try again') + return + + # rename origin to commaai so it's easy to switch to stock without any extra logic for url checking, etc + r = check_output(['git', '-C', COMMAAI_PATH, 'remote', 'rename', 'origin', 'commaai']) + if not r.success: + error(r.output) + return + + # backup and create symlink + if os.path.exists(OPENPILOT_PATH): + bak_dir = '{}.bak'.format(OPENPILOT_PATH) + idx = 0 + while os.path.exists(bak_dir): + bak_dir = '{}{}'.format(bak_dir, idx) + idx += 1 + shutil.move(OPENPILOT_PATH, bak_dir) + success('Backed up openpilot to {} and created symlink to {}'.format(bak_dir, COMMAAI_PATH)) + else: + success('Created symlink to {}'.format(COMMAAI_PATH)) + os.symlink(COMMAAI_PATH, OPENPILOT_PATH, target_is_directory=True) + check_output(['git', '-C', COMMAAI_PATH, 'checkout', 'release2']) + success('Fork management set up successfully! You\'re on commaai/release2') + success('To get started, try running: {}emu fork switch [fork_username] [branch]{}'.format(COLORS.RED, COLORS.ENDC)) + self.fork_params.put('setup_complete', True) + self.__add_fork('commaai') diff --git a/emu_commands/panda.py b/commands/panda/__init__.py similarity index 94% rename from emu_commands/panda.py rename to commands/panda/__init__.py index 5ce016ab..637fff79 100644 --- a/emu_commands/panda.py +++ b/commands/panda/__init__.py @@ -1,5 +1,5 @@ import importlib -from emu_commands.base import CommandBase, Command +from commands.base import CommandBase, Command from py_utils.emu_utils import run, error from py_utils.emu_utils import OPENPILOT_PATH diff --git a/emu_commands/uninstall.py b/commands/uninstall/__init__.py similarity index 89% rename from emu_commands/uninstall.py rename to commands/uninstall/__init__.py index 32d9e4c9..54074f39 100644 --- a/emu_commands/uninstall.py +++ b/commands/uninstall/__init__.py @@ -1,4 +1,4 @@ -from emu_commands.base import CommandBase, Command, Flag +from commands.base import CommandBase, Command, Flag from py_utils.emu_utils import run, warning, error, check_output, COLORS, success, input_with_options, UNINSTALL_PATH class Uninstall(CommandBase): diff --git a/emu_commands/update.py b/commands/update/__init__.py similarity index 68% rename from emu_commands/update.py rename to commands/update/__init__.py index 2f4db47a..4f0b5b08 100644 --- a/emu_commands/update.py +++ b/commands/update/__init__.py @@ -1,4 +1,4 @@ -from emu_commands.base import CommandBase +from commands.base import CommandBase from py_utils.emu_utils import run, error from py_utils.emu_utils import UPDATE_PATH @@ -6,7 +6,7 @@ class Update(CommandBase): def __init__(self): super().__init__() self.name = 'update' - self.description = '🎉 updates this tool, recommended to restart ssh session' + self.description = '🎉 Updates this tool' def _update(self): if not run(['sh', UPDATE_PATH]): diff --git a/emu.py b/emu.py index 28a02296..9149788b 100644 --- a/emu.py +++ b/emu.py @@ -5,11 +5,11 @@ import sys from os import path sys.path.append(path.abspath(path.join(path.dirname(__file__), 'py_utils'))) - sys.path.append(path.abspath(path.join(path.dirname(__file__), 'emu_commands'))) + sys.path.append(path.abspath(path.join(path.dirname(__file__), 'commands'))) from py_utils.emu_utils import BaseFunctions from py_utils.emu_utils import OPENPILOT_PATH - from emu_commands import EMU_COMMANDS + from commands import EMU_COMMANDS sys.path.append(OPENPILOT_PATH) # for importlib DEBUG = not path.exists('/data/params/d') diff --git a/emu_commands/__init__.py b/emu_commands/__init__.py deleted file mode 100644 index d19fd578..00000000 --- a/emu_commands/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -from emu_commands.fork import Fork -from emu_commands.update import Update -from emu_commands.panda import Panda -from emu_commands.debug import Debug -from emu_commands.device import Device -from emu_commands.uninstall import Uninstall - - -EMU_COMMANDS = [Fork(), Update(), Panda(), Debug(), Device(), Uninstall()] diff --git a/emu_commands/fork.py b/emu_commands/fork.py deleted file mode 100644 index ceca8f67..00000000 --- a/emu_commands/fork.py +++ /dev/null @@ -1,78 +0,0 @@ -import shutil -from os import path -from emu_commands.base import CommandBase, Command, Flag -from py_utils.emu_utils import run, error, warning, success, is_affirmative -from py_utils.emu_utils import OPENPILOT_PATH - -class Fork(CommandBase): - def __init__(self): - super().__init__() - self.name = 'fork' - self.description = '🍴 manage installed forks, or clone a new one' - - self.commands = {'install': Command(description='🦉 Whoooose fork do you wanna install?', - flags=[Flag(['clone_url'], '🍴 URL of fork to clone', has_value=True), - Flag(['-l', '--lite'], '💡 Clones only the default branch with all commits flattened for quick cloning'), - Flag(['-b', '--branch'], '🌿 Specify the branch to clone after this flag', has_value=True)])} - - def _install(self): - if self.next_arg(ingest=False) is None: - error('You must supply command arguments!') - self._help('install') - return - - flags, e = self.parse_flags(self.commands['install'].parser) - if e is not None: - error(e) - return - - if flags.clone_url is None: - error('You must specify a fork URL to clone!') - return - - OPENPILOT_TEMP_PATH = '{}.temp'.format(OPENPILOT_PATH) - if path.exists(OPENPILOT_TEMP_PATH): - warning('{} already exists, should it be deleted to continue?'.format(OPENPILOT_TEMP_PATH)) - if is_affirmative(): - shutil.rmtree(OPENPILOT_TEMP_PATH) - else: - error('Exiting...') - return - - # Clone fork to temp folder - warning('Fork will be installed to {}'.format(OPENPILOT_PATH)) - clone_flags = [] - if flags.lite: - warning('- Performing a lite clone! (--depth 1)') - clone_flags.append('--depth 1') - if flags.branch is not None: - warning('- Only cloning branch: {}'.format(flags.branch)) - clone_flags.append('-b {} --single-branch'.format(flags.branch)) - if len(clone_flags): - clone_flags.append('') - try: # catch ctrl+c and clean up after - r = run('git clone {}{} {}'.format(' '.join(clone_flags), flags.clone_url, OPENPILOT_TEMP_PATH)) # clone to temp folder - except: - r = False - - # If openpilot.bak exists, determine a good non-exiting path - # todo: make a folder that holds all installed forks and provide an interface of switching between them - bak_dir = '{}.bak'.format(OPENPILOT_PATH) - bak_count = 0 - while path.exists(bak_dir): - bak_count += 1 - bak_dir = '{}.{}'.format(bak_dir, bak_count) - - if r: - success('Cloned successfully! Installing fork...') - if path.exists(OPENPILOT_PATH): - shutil.move(OPENPILOT_PATH, bak_dir) # move current installation to old dir - shutil.move(OPENPILOT_TEMP_PATH, OPENPILOT_PATH) # move new clone temp folder to main installation dir - success("Installed! Don't forget to restart your device") - else: - error('\nError cloning specified fork URL!', end='') - if path.exists(OPENPILOT_TEMP_PATH): # git usually does this for us - error(' Cleaning up...') - shutil.rmtree(OPENPILOT_TEMP_PATH) - else: - print() diff --git a/install.sh b/install.sh index 87e59ad0..18670ab4 100755 --- a/install.sh +++ b/install.sh @@ -152,7 +152,7 @@ fi CURRENT_BRANCH=$(cd ${OH_MY_COMMA_PATH} && git rev-parse --abbrev-ref HEAD) if [ ${CURRENT_BRANCH} != "master" ]; then - printf "\n\033[0;31mWarning:\033[0m your current .oh-my-comma git branch is ${CURRENT_BRANCH}. Run cd /data/community/.oh-my-comma && git checkout master if this is unintentional\n" + printf "\n\033[0;31mWarning:\033[0m your current .oh-my-comma git branch is ${CURRENT_BRANCH}. Run git -C /data/community/.oh-my-comma checkout master if this is unintentional\n" fi echo "Current version: $OMC_VERSION" diff --git a/py_utils/emu_utils.py b/py_utils/emu_utils.py index 2e87806e..7879c832 100644 --- a/py_utils/emu_utils.py +++ b/py_utils/emu_utils.py @@ -20,6 +20,9 @@ UNINSTALL_PATH = '{}/uninstall.sh'.format(OH_MY_COMMA_PATH) OPENPILOT_PATH = '/data/openpilot' +FORKS_PATH = '/data/community/forks' +FORK_PARAM_PATH = '/data/community/forks/forks.json' + class ArgumentParser(argparse.ArgumentParser): def error(self, message): @@ -39,9 +42,8 @@ def print_commands(self, error_msg=None, ascii_art=False): print(COLORS.OKGREEN + ('- {:<%d} {}' % max_cmd).format(cmd + ':', desc)) if hasattr(self, '_help'): # leading is for better differentiating between the different commands - self._help(cmd, show_description=False, leading='') # todo: decide if leading is better than no leading - if idx == len(self.commands) - 1: # removes double newlines at end of loop - print() + self._help(cmd, show_description=False, leading=' ') + print() print(COLORS.ENDC, end='') def next_arg(self, lower=True, ingest=True): @@ -62,13 +64,15 @@ def next_arg(self, lower=True, ingest=True): return arg +def str_sim(a, b): + return difflib.SequenceMatcher(a=a, b=b).ratio() + + def input_with_options(options, default=None): """ Takes in a list of options and asks user to make a choice. The most similar option list index is returned along with the similarity percentage from 0 to 1 """ - def str_sim(a, b): - return difflib.SequenceMatcher(a=a, b=b).ratio() user_input = input('[{}]: '.format('/'.join(options))).lower().strip() if not user_input: @@ -78,19 +82,25 @@ def str_sim(a, b): return argmax, sims[argmax] -def check_output(cmd): - """ - If cmd is a string, it is split into a list, otherwise it doesn't modify cmd. - The status is returned, True being success, False for failure - """ +def most_similar(find, options): + sims = [[str_sim(i.lower().strip(), find.lower().strip()), i] for i in options] + sims = sorted(sims, reverse=True) + return [o[1] for o in sims] + + +def check_output(cmd, cwd=None): + class Output: + def __init__(self, output='', s=True): + self.output = output + self.success = s if isinstance(cmd, str): cmd = cmd.split() - try: - return subprocess.check_output(cmd) - except Exception as e: - # print(e) - return False + return Output(subprocess.check_output(cmd, cwd=cwd, stderr=subprocess.STDOUT, encoding='utf8')) + except subprocess.CalledProcessError as e: + if e.output is None: + return Output(e, s=False) # command failed to execute + return Output(e.output) # command executed but it resulted in error def run(cmd, out_file=None): @@ -124,7 +134,7 @@ def kill(procname): def is_affirmative(): i = None - print(COLORS.PROMPT) + print(COLORS.WARNING, end='') while i not in ['y', 'n', 'yes', 'no']: i = input('[Y/n]: ').lower().strip() print(COLORS.ENDC) @@ -146,7 +156,7 @@ def error(msg, end='\n', ret=False): def warning(msg, end='\n', ret=False): - w = '{}{}{}'.format(COLORS.WARNING, msg, COLORS.ENDC) + w = '{}{}{}'.format(COLORS.PROMPT, msg, COLORS.ENDC) if ret: return w print(w, end=end) @@ -159,6 +169,13 @@ def success(msg, end='\n', ret=False): print(s, end=end) +def info(msg, end='\n', ret=False): + s = '{}{}{}'.format(COLORS.WARNING, msg, COLORS.ENDC) + if ret: + return s + print(s, end=end) + + EMU_ART = r""" _ -=(""" + COLORS.RED + """'""" + COLORS.CWHITE + """) ;;