From d8713ceffa1207884e776342a80e74695a5c12aa Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 17 Apr 2025 16:41:55 -0500 Subject: [PATCH 01/10] reckless: fix installation from local directories with subpaths This could previously copy the parent directory of a plugin into the installed reckless directory, which was unnecessary. --- tools/reckless | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/reckless b/tools/reckless index 7b692d391e0d..a76424e1a87f 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1175,10 +1175,11 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: log.debug(f'{clone_path} already exists - deleting') shutil.rmtree(clone_path) if src.srctype == Source.DIRECTORY: + full_source_path = Path(src.source_loc) / src.subdir log.debug(("copying local directory contents from" - f" {src.source_loc}")) + f" {full_source_path}")) create_dir(clone_path) - shutil.copytree(src.source_loc, plugin_path) + shutil.copytree(full_source_path, plugin_path) elif src.srctype in [Source.LOCAL_REPO, Source.GITHUB_REPO, Source.OTHER_URL, Source.GIT_LOCAL_CLONE]: # clone git repository to /tmp/reckless-... From 307baf7611e87b85c8e5835d6a0a47ba5e569285 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 17 Apr 2025 17:13:49 -0500 Subject: [PATCH 02/10] reckless: accept a full local path as source+name This allows installing a local plugin directly without having to modify reckless sources. Changelog-changed: Reckless can be passed a local file or directory for installation. --- tools/reckless | 89 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 65 insertions(+), 24 deletions(-) diff --git a/tools/reckless b/tools/reckless index a76424e1a87f..492d7ffdd6bc 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1298,6 +1298,30 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: return staged_src +def location_from_name(plugin_name: str) -> (str, str): + """Maybe the location was passed in place of the plugin name. Check + if this looks like a filepath or URL and return that as well as the + plugin name.""" + if not Path(plugin_name).exists(): + # No path included, return the name only. + return (None, plugin_name) + + # Directory containing the plugin? The plugin name should match the dir. + if os.path.isdir(plugin_name): + return (Path(plugin_name).parent, Path(plugin_name).name) + + # Possibly the entrypoint itself was passed? + elif os.path.isfile(plugin_name): + if Path(plugin_name).with_suffix('').name != Path(plugin_name).parent.name or \ + not Path(plugin_name).parent.parent.exists(): + # If the directory is not named for the plugin, we can't infer what + # should be done. + # FIXME: return InstInfo with entrypoint rather than source str. + return (None, plugin_name) + # We have to make inferences as to the naming here. + return (Path(plugin_name).parent.parent, Path(plugin_name).with_suffix('').name) + + def install(plugin_name: str) -> Union[str, None]: """Downloads plugin from source repos, installs and activates plugin. Returns the location of the installed plugin or "None" in the case of @@ -1310,33 +1334,48 @@ def install(plugin_name: str) -> Union[str, None]: else: name = plugin_name commit = None - log.debug(f"Searching for {name}") - if search(name): - global LAST_FOUND - src = LAST_FOUND - src.commit = commit - log.debug(f'Retrieving {src.name} from {src.source_loc}') - try: - installed = _install_plugin(src) - except FileExistsError as err: - log.error(f'File exists: {err.filename}') - return None - LAST_FOUND = None - if not installed: - log.warning(f'{plugin_name}: installation aborted') + # Is the install request specifying a path to the plugin? + direct_location, name = location_from_name(name) + if direct_location: + logging.debug(f"install of {name} requested from {direct_location}") + src = InstInfo(name, direct_location, None) + if not src.get_inst_details(): + src = None + # Treating a local git repo as a directory allows testing + # uncommitted changes. + if src and src.srctype == Source.LOCAL_REPO: + src.srctype = Source.DIRECTORY + if not direct_location or not src: + log.debug(f"direct_location {direct_location}, src: {src}") + log.debug(f"Searching for {name}") + if search(name): + global LAST_FOUND + src = LAST_FOUND + src.commit = commit + log.debug(f'Retrieving {src.name} from {src.source_loc}') + else: return None - # Match case of the containing directory - for dirname in os.listdir(RECKLESS_CONFIG.reckless_dir): - if dirname.lower() == installed.name.lower(): - inst_path = Path(RECKLESS_CONFIG.reckless_dir) - inst_path = inst_path / dirname / installed.entry - RECKLESS_CONFIG.enable_plugin(inst_path) - enable(installed.name) - return f"{installed.source_loc}" - log.error(('dynamic activation failed: ' - f'{installed.name} not found in reckless directory')) + try: + installed = _install_plugin(src) + except FileExistsError as err: + log.error(f'File exists: {err.filename}') return None + LAST_FOUND = None + if not installed: + log.warning(f'{plugin_name}: installation aborted') + return None + + # Match case of the containing directory + for dirname in os.listdir(RECKLESS_CONFIG.reckless_dir): + if dirname.lower() == installed.name.lower(): + inst_path = Path(RECKLESS_CONFIG.reckless_dir) + inst_path = inst_path / dirname / installed.entry + RECKLESS_CONFIG.enable_plugin(inst_path) + enable(installed.name) + return f"{installed.source_loc}" + log.error(('dynamic activation failed: ' + f'{installed.name} not found in reckless directory')) return None @@ -1386,6 +1425,8 @@ def search(plugin_name: str) -> Union[InstInfo, None]: if srctype in [Source.DIRECTORY, Source.LOCAL_REPO, Source.GITHUB_REPO, Source.OTHER_URL]: found = _source_search(plugin_name, source) + if found: + log.debug(f"{found}, {found.srctype}") if not found: continue log.info(f"found {found.name} in source: {found.source_loc}") From a651b1a2a59576e7cd46d06358e32a80ee978437 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Tue, 22 Apr 2025 15:28:58 -0500 Subject: [PATCH 03/10] reckless: handle a direct source in the form of a git repo url --- tools/reckless | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/reckless b/tools/reckless index 492d7ffdd6bc..8fff14a11e2d 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1303,6 +1303,12 @@ def location_from_name(plugin_name: str) -> (str, str): if this looks like a filepath or URL and return that as well as the plugin name.""" if not Path(plugin_name).exists(): + try: + parsed = urlparse(plugin_name) + if parsed.scheme in ['http', 'https']: + return (plugin_name, Path(plugin_name).with_suffix('').name) + except ValueError: + pass # No path included, return the name only. return (None, plugin_name) @@ -1336,6 +1342,7 @@ def install(plugin_name: str) -> Union[str, None]: commit = None # Is the install request specifying a path to the plugin? direct_location, name = location_from_name(name) + src = None if direct_location: logging.debug(f"install of {name} requested from {direct_location}") src = InstInfo(name, direct_location, None) @@ -1346,7 +1353,6 @@ def install(plugin_name: str) -> Union[str, None]: if src and src.srctype == Source.LOCAL_REPO: src.srctype = Source.DIRECTORY if not direct_location or not src: - log.debug(f"direct_location {direct_location}, src: {src}") log.debug(f"Searching for {name}") if search(name): global LAST_FOUND From c8435f3d7b9688cd20a971d565a43ad85c3f8604 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Sun, 4 May 2025 16:51:13 -0500 Subject: [PATCH 04/10] reckless: store absolute paths in metadata --- tools/reckless | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/reckless b/tools/reckless index 8fff14a11e2d..163216b86e95 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1099,12 +1099,13 @@ def add_installation_metadata(installed: InstInfo, updating the plugin.""" install_dir = Path(installed.source_loc) assert install_dir.is_dir() + abs_source_path = Path(original_request.source_loc).resolve() data = ('installation date\n' f'{datetime.date.today().isoformat()}\n' 'installation time\n' f'{int(time.time())}\n' 'original source\n' - f'{original_request.source_loc}\n' + f'{abs_source_path}\n' 'requested commit\n' f'{original_request.commit}\n' 'installed commit\n' From 8dc81957a92f7d7de2e176e03122c821ba23af87 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Sun, 4 May 2025 16:21:42 -0500 Subject: [PATCH 05/10] reckless: refactor install remove the duplicative search and extract the enable portion for use next. --- tools/reckless | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/tools/reckless b/tools/reckless index 163216b86e95..162e6345e535 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1329,6 +1329,20 @@ def location_from_name(plugin_name: str) -> (str, str): return (Path(plugin_name).parent.parent, Path(plugin_name).with_suffix('').name) +def _enable_installed(installed: InstInfo, plugin_name: str) -> Union[str, None]: + """Enable the plugin in the active config file and dynamically activate + if a lightnignd rpc is available.""" + if not installed: + log.warning(f'{plugin_name}: installation aborted') + return None + + if enable(installed.name): + return f"{installed.source_loc}" + + log.error(('dynamic activation failed: ' + f'{installed.name} not found in reckless directory')) + return None + def install(plugin_name: str) -> Union[str, None]: """Downloads plugin from source repos, installs and activates plugin. Returns the location of the installed plugin or "None" in the case of @@ -1358,9 +1372,11 @@ def install(plugin_name: str) -> Union[str, None]: if search(name): global LAST_FOUND src = LAST_FOUND + LAST_FOUND = None src.commit = commit log.debug(f'Retrieving {src.name} from {src.source_loc}') else: + LAST_FOUND = None return None try: @@ -1368,22 +1384,8 @@ def install(plugin_name: str) -> Union[str, None]: except FileExistsError as err: log.error(f'File exists: {err.filename}') return None - LAST_FOUND = None - if not installed: - log.warning(f'{plugin_name}: installation aborted') - return None + return _enable_installed(installed, plugin_name) - # Match case of the containing directory - for dirname in os.listdir(RECKLESS_CONFIG.reckless_dir): - if dirname.lower() == installed.name.lower(): - inst_path = Path(RECKLESS_CONFIG.reckless_dir) - inst_path = inst_path / dirname / installed.entry - RECKLESS_CONFIG.enable_plugin(inst_path) - enable(installed.name) - return f"{installed.source_loc}" - log.error(('dynamic activation failed: ' - f'{installed.name} not found in reckless directory')) - return None def uninstall(plugin_name: str) -> str: From f7beb389e7ff5710d6420ffdc44173b8cdd8b0c9 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Mon, 28 Apr 2025 12:35:21 -0500 Subject: [PATCH 06/10] reckless: add update command This updates all reckless-installed plugins with `reckless update` or update individual plugins by passing the plugin names as arguments. The metadata stored with the installed plugin is used to find the plugin from the appropriate source (the same source is used as when originally installed.) Changelog-Added: Reckless: `reckless update` updates all reckless-installed plugins. --- tools/reckless | 79 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 9 deletions(-) diff --git a/tools/reckless b/tools/reckless index 162e6345e535..ea6ad0569ee0 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1662,6 +1662,57 @@ def list_source(): return sources_from_file() +def update_plugin(plugin_name: str) -> Union[str, None]: + """Check for an installed plugin, if metadata for it exists, update + to the latest available while using the same source.""" + log.info(f"updating {plugin_name}") + metadata_file = Path(RECKLESS_CONFIG.reckless_dir) / plugin_name / '.metadata' + if not metadata_file.exists(): + log.warning(f"no metadata file for {plugin_name}") + return + + metadata = {'installation date': None, + 'installation time': None, + 'original source': None, + 'requested commit': None, + 'installed commit': None, + } + with open(metadata_file, "r") as meta: + metadata_lines = meta.readlines() + for line_no, line in enumerate(metadata_lines): + if line_no > 0 and metadata_lines[line_no - 1].strip() in metadata: + metadata.update({metadata_lines[line_no - 1].strip(): line.strip()}) + for key in metadata: + if metadata[key].lower() == 'none': + metadata[key] = None + log.debug(f'{plugin_name} installation metadata: {str(metadata)}') + + src = InstInfo(plugin_name, + metadata['original source'], None) + uninstall(plugin_name) + try: + installed = _install_plugin(src) + except FileExistsError as err: + log.error(f'File exists: {err.filename}') + return None + return _enable_installed(installed, plugin_name) + + +def update_plugins(plugin_name: str): + """user requested plugin upgrade(s)""" + if plugin_name: + update_plugin(plugin_name) + return + + log.info("updating all plugins") + for plugin in os.listdir(RECKLESS_CONFIG.reckless_dir): + if not (Path(RECKLESS_CONFIG.reckless_dir) / plugin).is_dir(): + continue + if len(plugin) > 0 and plugin[0] == '.': + continue + update_plugin(plugin) + + def report_version() -> str: """return reckless version""" log.info(__VERSION__) @@ -1757,6 +1808,9 @@ if __name__ == '__main__': 'repository') source_rem.add_argument('targets', type=str, nargs='*') source_rem.set_defaults(func=remove_source) + update = cmd1.add_parser('update', help='update plugins to lastest version') + update.add_argument('targets', type=str, nargs='*') + update.set_defaults(func=update_plugins) help_cmd = cmd1.add_parser('help', help='for contextual help, use ' '"reckless -h"') @@ -1767,7 +1821,8 @@ if __name__ == '__main__': help='print version and exit') all_parsers = [parser, install_cmd, uninstall_cmd, search_cmd, enable_cmd, - disable_cmd, list_parse, source_add, source_rem, help_cmd] + disable_cmd, list_parse, source_add, source_rem, help_cmd, + update] for p in all_parsers: # This default depends on the .lightning directory p.add_argument('-d', '--reckless-dir', action=StoreIdempotent, @@ -1848,19 +1903,25 @@ if __name__ == '__main__': if 'GITHUB_API_FALLBACK' in os.environ: GITHUB_API_FALLBACK = os.environ['GITHUB_API_FALLBACK'] - if 'targets' in args: - # FIXME: Catch missing argument + if 'targets' in args: # and len(args.targets) > 0: if args.func.__name__ == 'help_alias': args.func(args.targets) sys.exit(0) + # Catch a missing argument so that we can overload functions. + if len(args.targets) == 0: + args.targets=[None] for target in args.targets: # Accept single item arguments, or a json array - target_list = unpack_json_arg(target) - if target_list: - for tar in target_list: - log.add_result(args.func(tar)) - else: - log.add_result(args.func(target)) + try: + target_list = unpack_json_arg(target) + if target_list: + for tar in target_list: + log.add_result(args.func(tar)) + else: + log.add_result(args.func(target)) + except TypeError: + if len(args.targets) == 1: + log.add_result(args.func(target)) elif 'func' in args: log.add_result(args.func()) From 2593b70b889657b89a48d17e866d3570ad05b0c8 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Sun, 4 May 2025 16:24:16 -0500 Subject: [PATCH 07/10] reckless: provide user feedback at info level if enable fails --- tools/reckless | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/reckless b/tools/reckless index ea6ad0569ee0..adeb82d70534 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1517,8 +1517,8 @@ def enable(plugin_name: str): log.error(err) return None except RPCError: - log.debug(('lightningd rpc unavailable. ' - 'Skipping dynamic activation.')) + log.info(('lightningd rpc unavailable. ' + 'Skipping dynamic activation.')) RECKLESS_CONFIG.enable_plugin(path) log.info(f'{inst.name} enabled') return 'enabled' From d6a00b4e233f881b484d019a6b05626f67b57e4b Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Mon, 5 May 2025 10:08:05 -0500 Subject: [PATCH 08/10] reckless: return result from update --- tools/reckless | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tools/reckless b/tools/reckless index adeb82d70534..08393af487b0 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1669,7 +1669,7 @@ def update_plugin(plugin_name: str) -> Union[str, None]: metadata_file = Path(RECKLESS_CONFIG.reckless_dir) / plugin_name / '.metadata' if not metadata_file.exists(): log.warning(f"no metadata file for {plugin_name}") - return + return None metadata = {'installation date': None, 'installation time': None, @@ -1701,16 +1701,20 @@ def update_plugin(plugin_name: str) -> Union[str, None]: def update_plugins(plugin_name: str): """user requested plugin upgrade(s)""" if plugin_name: - update_plugin(plugin_name) - return + installed = update_plugin(plugin_name) + if not installed: + log.error(f'{plugin_name} update aborted') + return installed log.info("updating all plugins") + update_results = [] for plugin in os.listdir(RECKLESS_CONFIG.reckless_dir): if not (Path(RECKLESS_CONFIG.reckless_dir) / plugin).is_dir(): continue if len(plugin) > 0 and plugin[0] == '.': continue - update_plugin(plugin) + update_results.append(update_plugin(plugin)) + return update_results def report_version() -> str: From 59f3ecc88b09c01579d64e5391f9961c9466849b Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Mon, 5 May 2025 13:48:36 -0500 Subject: [PATCH 09/10] reckless: only proceed with update when appropriate --- tools/reckless | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tools/reckless b/tools/reckless index 08393af487b0..97103275ed43 100755 --- a/tools/reckless +++ b/tools/reckless @@ -206,6 +206,36 @@ class InstInfo: return (f'InstInfo({self.name}, {self.source_loc}, {self.git_url}, ' f'{self.entry}, {self.deps}, {self.subdir})') + def get_repo_commit(self) -> Union[str, None]: + """The latest commit from a remote repo or the HEAD of a local repo.""" + if self.srctype in [Source.LOCAL_REPO, Source.GIT_LOCAL_CLONE]: + git = run(['git', 'rev-parse', 'HEAD'], cwd=str(self.source_loc), + stdout=PIPE, stderr=PIPE, text=True, check=False, timeout=10) + if git.returncode != 0: + return None + return git.stdout.splitlines()[0] + + if self.srctype == Source.GITHUB_REPO: + parsed_url = urlparse(self.git_url) + if 'github.com' not in parsed_url.netloc: + return None + if len(parsed_url.path.split('/')) < 2: + return None + start = 1 + # Maybe we were passed an api.github.com/repo/ url + if 'api' in parsed_url.netloc: + start += 1 + repo_user = parsed_url.path.split('/')[start] + repo_name = parsed_url.path.split('/')[start + 1] + api_url = f'{API_GITHUB_COM}/repos/{repo_user}/{repo_name}/commits?ref=HEAD' + r = urlopen(api_url, timeout=5) + if r.status != 200: + return None + try: + return json.loads(r.read().decode())['0']['sha'] + except: + return None + def get_inst_details(self) -> bool: """Search the source_loc for plugin install details. This may be necessary if a contents api is unavailable. @@ -1689,6 +1719,15 @@ def update_plugin(plugin_name: str) -> Union[str, None]: src = InstInfo(plugin_name, metadata['original source'], None) + if not src.get_inst_details(): + log.error(f'cannot locate {plugin_name} in original source {metadata["original_source"]}') + return None + repo_commit = src.get_repo_commit() + if not repo_commit: + log.debug('source commit not available') + if repo_commit and repo_commit == metadata['installed commit']: + log.debug(f'Installed {plugin_name} is already latest - {repo_commit}') + return None uninstall(plugin_name) try: installed = _install_plugin(src) From 715614568d4acdc1d57a138a1d23da4c83755c65 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Mon, 5 May 2025 16:41:18 -0500 Subject: [PATCH 10/10] reckless: don't return error if update is unnecessary --- tools/reckless | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/tools/reckless b/tools/reckless index 97103275ed43..7fc9638d3bb9 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1692,14 +1692,25 @@ def list_source(): return sources_from_file() -def update_plugin(plugin_name: str) -> Union[str, None]: +class UpdateStatus(Enum): + SUCCESS = 0 + LATEST = 1 + UNINSTALLED = 2 + ERROR = 3 + METADATA_MISSING = 4 + + +def update_plugin(plugin_name: str) -> tuple: """Check for an installed plugin, if metadata for it exists, update to the latest available while using the same source.""" log.info(f"updating {plugin_name}") + if not (Path(RECKLESS_CONFIG.reckless_dir) / plugin_name).exists(): + log.error(f'{plugin_name} is not installed') + return (None, UpdateStatus.UNINSTALLED) metadata_file = Path(RECKLESS_CONFIG.reckless_dir) / plugin_name / '.metadata' if not metadata_file.exists(): log.warning(f"no metadata file for {plugin_name}") - return None + return (None, UpdateStatus.METADATA_MISSING) metadata = {'installation date': None, 'installation time': None, @@ -1721,29 +1732,34 @@ def update_plugin(plugin_name: str) -> Union[str, None]: metadata['original source'], None) if not src.get_inst_details(): log.error(f'cannot locate {plugin_name} in original source {metadata["original_source"]}') - return None + return (None, UpdateStatus.ERROR) repo_commit = src.get_repo_commit() if not repo_commit: log.debug('source commit not available') + else: + log.debug(f'source commit: {repo_commit}') if repo_commit and repo_commit == metadata['installed commit']: - log.debug(f'Installed {plugin_name} is already latest - {repo_commit}') - return None + log.info(f'Installed {plugin_name} is already latest @{repo_commit}') + return (None, UpdateStatus.LATEST) uninstall(plugin_name) try: installed = _install_plugin(src) except FileExistsError as err: log.error(f'File exists: {err.filename}') - return None - return _enable_installed(installed, plugin_name) + return (None, UpdateStatus.ERROR) + result = _enable_installed(installed, plugin_name) + if result: + return (result, UpdateStatus.SUCCESS) + return (result, UpdateStatus.ERROR) def update_plugins(plugin_name: str): """user requested plugin upgrade(s)""" if plugin_name: installed = update_plugin(plugin_name) - if not installed: + if not installed[0] and installed[1] != UpdateStatus.LATEST: log.error(f'{plugin_name} update aborted') - return installed + return installed[0] log.info("updating all plugins") update_results = [] @@ -1752,7 +1768,7 @@ def update_plugins(plugin_name: str): continue if len(plugin) > 0 and plugin[0] == '.': continue - update_results.append(update_plugin(plugin)) + update_results.append(update_plugin(plugin)[0]) return update_results