From 8e5d716b59e29e42dfc8dbb8e8d87954ee4d7504 Mon Sep 17 00:00:00 2001 From: Sujal Salekar Date: Sun, 11 May 2025 00:49:50 +0530 Subject: [PATCH 1/2] Update __init__.py --- src/lighthouseweb3/__init__.py | 55 +++++++++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/src/lighthouseweb3/__init__.py b/src/lighthouseweb3/__init__.py index 8cf1d94..3de5743 100644 --- a/src/lighthouseweb3/__init__.py +++ b/src/lighthouseweb3/__init__.py @@ -18,26 +18,46 @@ def upload(self, source: str, tag: str = ''): Upload a file or directory to the Lighthouse. :param source: str, path to file or directory + :param tag: str, optional tag for the upload :return: t.Upload, the upload result + :raises ValueError: If source path is invalid or doesn't exist + :raises Exception: If upload fails """ + if not source or not isinstance(source, str): + raise ValueError("Source path must be a non-empty string") + + if not os.path.exists(source): + raise ValueError(f"Source path does not exist: {source}") + try: return d.upload(source, self.token, tag) except Exception as e: - raise e + raise Exception(f"Upload failed: {str(e)}") def uploadBlob(self, source: io.BufferedReader, filename: str, tag: str = ''): """ Upload Blob a file or directory to the Lighthouse. - :param source: str, path to file or directory + :param source: io.BufferedReader, file-like object to upload + :param filename: str, name of the file to be uploaded + :param tag: str, optional tag for the upload :return: t.Upload, the upload result + :raises TypeError: If source is not a proper file-like object + :raises ValueError: If filename is invalid """ + if not isinstance(source, io.BufferedReader): + raise TypeError("source must be an instance of io.BufferedReader") + + if not filename or not isinstance(filename, str): + raise ValueError("filename must be a non-empty string") + if not (hasattr(source, 'read') and hasattr(source, 'close')): raise TypeError("source must have 'read' and 'close' methods") + try: return d.uploadBlob(source, filename, self.token, tag) except Exception as e: - raise e + raise Exception(f"Failed to upload blob: {str(e)}") @staticmethod def downloadBlob(dist: io.BufferedWriter, cid: str, chunk_size=1024*1024*10): @@ -48,13 +68,19 @@ def downloadBlob(dist: io.BufferedWriter, cid: str, chunk_size=1024*1024*10): :param cid: str, Content Identifier for the data to be downloaded :param chunk_size: int, size of chunks in which the file will be downloaded (default: 10MB) :return: t.Upload, the download result + :raises TypeError: If dist doesn't have required write and close methods + :raises ValueError: If cid is empty or invalid """ - if not (hasattr(dist, 'read') and hasattr(dist, 'close')): - raise TypeError("source must have 'read' and 'close' methods") + if not (hasattr(dist, 'write') and hasattr(dist, 'close')): + raise TypeError("dist must have 'write' and 'close' methods") + + if not cid or not isinstance(cid, str): + raise ValueError("Invalid CID provided") + try: return _download.download_file_into_writable(cid, dist, chunk_size) except Exception as e: - raise e + raise Exception(f"Failed to download file: {str(e)}") @staticmethod def getDealStatus(cid: str): @@ -63,11 +89,16 @@ def getDealStatus(cid: str): :param cid: str, content identifier :return: List[t.DealData], list of deal data + :raises ValueError: If CID is invalid + :raises Exception: If fetching deal status fails """ + if not cid or not isinstance(cid, str): + raise ValueError("CID must be a non-empty string") + try: return deal_status.get_deal_status(cid) except Exception as e: - raise e + raise Exception(f"Failed to get deal status: {str(e)}") @staticmethod def getUploads(publicKey: str, pageNo: int = 1): @@ -77,11 +108,19 @@ def getUploads(publicKey: str, pageNo: int = 1): :param publicKey: str, public key :param pageNo: int, page number (default: 1) :return: List[t.DealData], list of deal data + :raises ValueError: If publicKey is invalid or pageNo is less than 1 + :raises Exception: If fetching uploads fails """ + if not publicKey or not isinstance(publicKey, str): + raise ValueError("Public key must be a non-empty string") + + if not isinstance(pageNo, int) or pageNo < 1: + raise ValueError("Page number must be a positive integer") + try: return getUploads.get_uploads(publicKey, pageNo) except Exception as e: - raise e + raise Exception(f"Failed to get uploads: {str(e)}") @staticmethod def download(cid: str): From e19d37bfdf18b868daa398a74af12683ba3f78c6 Mon Sep 17 00:00:00 2001 From: Sujal Salekar Date: Sat, 10 May 2025 20:27:15 +0000 Subject: [PATCH 2/2] Enhance SDK with type safety, error handling and resource management improvements --- src/lighthouseweb3/functions/axios.py | 145 ++++++++++++++----- src/lighthouseweb3/functions/upload.py | 134 +++++++++++------- src/lighthouseweb3/functions/utils.py | 185 ++++++++++++++++++------- 3 files changed, 331 insertions(+), 133 deletions(-) diff --git a/src/lighthouseweb3/functions/axios.py b/src/lighthouseweb3/functions/axios.py index a50df40..7316eeb 100644 --- a/src/lighthouseweb3/functions/axios.py +++ b/src/lighthouseweb3/functions/axios.py @@ -2,67 +2,136 @@ from io import BufferedReader import json +from typing import Optional, Dict, Any, Union import requests as req +from requests.exceptions import RequestException from . import utils class Axios: - """It's not axios, it's just a custom extensible wrapper for requests""" + """ + A custom extensible wrapper for requests library. + Provides simplified HTTP client functionality with proper error handling. + """ def __init__(self, url: str): + """ + Initialize Axios instance. + + :param url: Base URL for requests + :raises ValueError: If URL is empty or invalid + """ + if not url or not isinstance(url, str): + raise ValueError("URL must be a non-empty string") self.url = url - def parse_url_query(self, query): + def parse_url_query(self, query: Optional[Dict[str, Any]]) -> None: + """ + Parse and append query parameters to URL. + + :param query: Dictionary of query parameters + :raises ValueError: If query parameters are invalid + """ try: - if query is not None and isinstance(query, dict): + if query is not None: + if not isinstance(query, dict): + raise ValueError("Query must be a dictionary") for key, value in query.items(): self.url += f"&{key}={value}" except Exception as e: - raise e + raise ValueError(f"Failed to parse query parameters: {str(e)}") - def get(self, headers = None, **kwargs) : + def get(self, headers: Optional[Dict[str, str]] = None, **kwargs) -> Dict[str, Any]: + """ + Perform GET request. + + :param headers: Request headers + :param kwargs: Additional parameters including query + :return: JSON response + :raises RequestException: If request fails + """ try: - self.parse_url_query(kwargs.get("query", None)) - r = req.get(self.url, headers=headers) + self.parse_url_query(kwargs.get("query")) + r = req.get(self.url, headers=headers, timeout=30) # Added timeout r.raise_for_status() return r.json() - except Exception as e: - raise e + except req.exceptions.JSONDecodeError as e: + raise ValueError(f"Failed to decode JSON response: {str(e)}") + except RequestException as e: + raise RequestException(f"GET request failed: {str(e)}") - def post( - self, body=None, headers= None, **kwargs - ): + def post(self, body: Optional[Any] = None, headers: Optional[Dict[str, str]] = None, **kwargs) -> Dict[str, Any]: + """ + Perform POST request. + + :param body: Request body + :param headers: Request headers + :param kwargs: Additional parameters including query + :return: JSON response + :raises RequestException: If request fails + """ try: - self.parse_url_query(kwargs.get("query", None)) - r = req.post(self.url, data=body, headers=headers) + self.parse_url_query(kwargs.get("query")) + r = req.post(self.url, data=body, headers=headers, timeout=30) # Added timeout r.raise_for_status() return r.json() - except Exception as e: - raise e + except req.exceptions.JSONDecodeError as e: + raise ValueError(f"Failed to decode JSON response: {str(e)}") + except RequestException as e: + raise RequestException(f"POST request failed: {str(e)}") - def post_files( - self, file, headers = None, **kwargs - ) : + def post_files(self, file: Dict[str, Any], headers: Optional[Dict[str, str]] = None, **kwargs) -> Dict[str, Any]: + """ + Upload files via POST request. + + :param file: File information dictionary + :param headers: Request headers + :param kwargs: Additional parameters including query + :return: JSON response + :raises RequestException: If request fails + """ + files = None try: - self.parse_url_query(kwargs.get("query", None)) + self.parse_url_query(kwargs.get("query")) files = utils.read_files_for_upload(file) - r = req.post(self.url, headers=headers, files=files) + r = req.post(self.url, headers=headers, files=files, timeout=30) # Added timeout r.raise_for_status() - utils.close_files_after_upload(files) + + # Always ensure files are closed + if files: + utils.close_files_after_upload(files) + try: return r.json() - except Exception: + except req.exceptions.JSONDecodeError: + # Handle special case where response contains multiple lines temp = r.text.split("\n") - return json.loads(temp[len(temp) - 2]) + return json.loads(temp[-2]) # Use -2 index instead of len(temp) - 2 except Exception as e: - utils.close_files_after_upload(files) - raise e + # Ensure files are closed even if an error occurs + if files: + utils.close_files_after_upload(files) + raise RequestException(f"File upload failed: {str(e)}") + + def post_blob(self, file: BufferedReader, filename: str, headers: Optional[Dict[str, str]] = None, **kwargs) -> Dict[str, Any]: + """ + Upload blob data via POST request. + + :param file: BufferedReader instance containing file data + :param filename: Name of the file + :param headers: Request headers + :param kwargs: Additional parameters including query + :return: JSON response + :raises RequestException: If request fails + :raises ValueError: If file or filename is invalid + """ + if not isinstance(file, BufferedReader): + raise ValueError("file must be a BufferedReader instance") + if not filename or not isinstance(filename, str): + raise ValueError("filename must be a non-empty string") - def post_blob( - self, file: BufferedReader, filename: str, headers = None, **kwargs - ) : try: - self.parse_url_query(kwargs.get("query", None)) + self.parse_url_query(kwargs.get("query")) files = [( "file", ( @@ -70,15 +139,19 @@ def post_blob( file.read(), "application/octet-stream", ), - ),] - r = req.post(self.url, headers=headers, files=files) + )] + + r = req.post(self.url, headers=headers, files=files, timeout=30) # Added timeout r.raise_for_status() - file.close() + try: return r.json() - except Exception: + except req.exceptions.JSONDecodeError: + # Handle special case where response contains multiple lines temp = r.text.split("\n") - return json.loads(temp[len(temp) - 2]) + return json.loads(temp[-2]) # Use -2 index instead of len(temp) - 2 except Exception as e: + raise RequestException(f"Blob upload failed: {str(e)}") + finally: + # Always ensure file is closed file.close() - raise e diff --git a/src/lighthouseweb3/functions/upload.py b/src/lighthouseweb3/functions/upload.py index 652cd8a..9bb7fc7 100644 --- a/src/lighthouseweb3/functions/upload.py +++ b/src/lighthouseweb3/functions/upload.py @@ -1,88 +1,122 @@ #!/usr/bin/env python3 from io import BufferedReader -from typing import Dict, List, Tuple +from typing import Dict, List, Tuple, Union, Any from .axios import Axios from .utils import is_dir, walk_dir_tree, extract_file_name, NamedBufferedReader from .config import Config -def upload(source, token: str, tag: str = ""): +def upload(source: Union[str, BufferedReader], token: str, tag: str = "") -> Dict[str, Any]: """ - Deploy a file or directory to the lighthouse network - @params {source}: str, path to file or directory - @params {token}: str, lighthouse api token + Deploy a file or directory to the lighthouse network. + + :param source: Path to file/directory or BufferedReader instance + :param token: Lighthouse API token + :param tag: Optional tag for the upload + :return: Dictionary containing upload response data + :raises ValueError: If source or token is invalid + :raises Exception: If upload fails """ - # create headers - headers = { + if not token or not isinstance(token, str): + raise ValueError("Token must be a non-empty string") + + if not source: + raise ValueError("Source must be provided") + + # Create headers with proper typing + headers: Dict[str, str] = { "Authorization": f"Bearer {token}", - # "Content-Type": "multipart/form-data", "Encryption": "false", "Mime-Type": "application/octet-stream", } + try: - # create http object + # Create HTTP object axios = Axios(Config.lighthouse_node + "/api/v0/add") - # create list of files to upload - if (isinstance(source, str)): - file_dict = {} + if isinstance(source, str): + file_dict: Dict[str, Union[List[str], bool, str]] = {} - # check if source is a directory + # Check if source is a directory if is_dir(source): - # walk directory tree and add files to list - file_dict["files"], root = walk_dir_tree(source) + # Walk directory tree and add files to list + files, root = walk_dir_tree(source) + file_dict["files"] = files file_dict["is_dir"] = True file_dict["path"] = root else: - # add file to list + # Add single file file_dict["files"] = [source] file_dict["is_dir"] = False file_dict["path"] = source - hashData = axios.post_files(file_dict, headers) + + hash_data = axios.post_files(file_dict, headers) else: - hashData = axios.post_blob(source, source.name, headers) - - if len(tag): - _axios = Axios(Config.lighthouse_api + "/api/user/create_tag") - data = _axios.post({ - "tag": tag, - "cid": hashData.get("Hash") - }, { - "Authorization": f"Bearer {token}", }) - return {"data": hashData} + if not hasattr(source, 'name'): + raise ValueError("Source object must have a 'name' attribute") + hash_data = axios.post_blob(source, source.name, headers) + + # Create tag if provided + if tag: + tag_axios = Axios(Config.lighthouse_api + "/api/user/create_tag") + tag_axios.post( + body={ + "tag": tag, + "cid": hash_data.get("Hash") + }, + headers={"Authorization": f"Bearer {token}"} + ) + + return {"data": hash_data} except Exception as e: - print(e) - raise e + # Don't print the error, just raise it with context + raise Exception(f"Upload failed: {str(e)}") -def uploadBlob(source: BufferedReader, filename: str, token: str, tag: str = ""): +def uploadBlob(source: BufferedReader, filename: str, token: str, tag: str = "") -> Dict[str, Any]: """ - Upload a Buffer or readable Object - @params {source}: str, path to file or directory - @params {token}: str, lighthouse api token + Upload a Buffer or readable Object to the lighthouse network. + + :param source: BufferedReader instance containing file data + :param filename: Name of the file to be uploaded + :param token: Lighthouse API token + :param tag: Optional tag for the upload + :return: Dictionary containing upload response data + :raises ValueError: If parameters are invalid + :raises Exception: If upload fails """ - # create headers - headers = { + if not isinstance(source, BufferedReader): + raise ValueError("Source must be a BufferedReader instance") + if not filename or not isinstance(filename, str): + raise ValueError("Filename must be a non-empty string") + if not token or not isinstance(token, str): + raise ValueError("Token must be a non-empty string") + + # Create headers with proper typing + headers: Dict[str, str] = { "Authorization": f"Bearer {token}", - # "Content-Type": "multipart/form-data", "Encryption": "false", "Mime-Type": "application/octet-stream", } + try: - # create http object + # Create HTTP object axios = Axios(Config.lighthouse_node + "/api/v0/add") - # create list of files to upload - - hashData = axios.post_blob(source, filename, headers) - if len(tag): - _axios = Axios(Config.lighthouse_api + "/api/user/create_tag") - data = _axios.post({ - "tag": tag, - "cid": hashData.get("Hash") - }, { - "Authorization": f"Bearer {token}", }) - return {"data": hashData} + hash_data = axios.post_blob(source, filename, headers) + + # Create tag if provided + if tag: + tag_axios = Axios(Config.lighthouse_api + "/api/user/create_tag") + tag_axios.post( + body={ + "tag": tag, + "cid": hash_data.get("Hash") + }, + headers={"Authorization": f"Bearer {token}"} + ) + + return {"data": hash_data} except Exception as e: - print(e) - raise e + # Don't print the error, just raise it with context + raise Exception(f"Blob upload failed: {str(e)}") diff --git a/src/lighthouseweb3/functions/utils.py b/src/lighthouseweb3/functions/utils.py index 691759e..239675f 100644 --- a/src/lighthouseweb3/functions/utils.py +++ b/src/lighthouseweb3/functions/utils.py @@ -2,79 +2,170 @@ from io import BufferedReader, BytesIO import os +from typing import List, Tuple, Dict, Any, BinaryIO class NamedBufferedReader: - def __init__(self, buffer, name:str): + """ + A wrapper class for BufferedReader that includes a name attribute. + Useful for handling named file-like objects. + """ + + def __init__(self, buffer: BytesIO, name: str): + """ + Initialize NamedBufferedReader. + + :param buffer: BytesIO object to read from + :param name: Name to associate with the buffer + :raises ValueError: If name is empty or invalid + """ + if not name or not isinstance(name, str): + raise ValueError("Name must be a non-empty string") + if not isinstance(buffer, BytesIO): + raise ValueError("Buffer must be a BytesIO instance") + self.reader = BufferedReader(buffer) self.name = name - def read(self, *args, **kwargs): + def read(self, *args, **kwargs) -> bytes: + """Read from the underlying buffer.""" return self.reader.read(*args, **kwargs) - def close(self): + def close(self) -> None: + """Close the underlying buffer.""" self.reader.close() -# walk path and return list of file paths -def walk_dir_tree(path: str): +def walk_dir_tree(path: str) -> Tuple[List[str], str]: + """ + Walk through directory tree and collect file paths. + + :param path: Root directory path to walk through + :return: Tuple of (list of file paths, root directory path) + :raises ValueError: If path is invalid or doesn't exist + """ + if not path or not isinstance(path, str): + raise ValueError("Path must be a non-empty string") + if not os.path.exists(path): + raise ValueError(f"Path does not exist: {path}") + file_list = [] roots = [] - for root, dirs, files in os.walk(path): - roots.append(root) - for file in files: - file_list.append(os.path.join(root, file)) - return file_list, roots[0] - - -# check if file is a directory -def is_dir(path: str): + + try: + for root, _, files in os.walk(path): + roots.append(root) + for file in files: + file_list.append(os.path.join(root, file)) + + if not roots: + raise ValueError(f"No valid directory found at path: {path}") + + return file_list, roots[0] + except Exception as e: + raise ValueError(f"Failed to walk directory tree: {str(e)}") + + +def is_dir(path: str) -> bool: + """ + Check if path points to a directory. + + :param path: Path to check + :return: True if path is a directory, False otherwise + :raises ValueError: If path is invalid + """ + if not path or not isinstance(path, str): + raise ValueError("Path must be a non-empty string") return os.path.isdir(path) -def extract_file_name(file: str): - return file.split("/")[-1] - - -def extract_file_name_with_source(file: str, source: str): - if source.endswith("/"): - source = source[: len(source) - 1] - base = source.split("/")[-1] +def extract_file_name(file: str) -> str: + """ + Extract filename from file path. + + :param file: File path + :return: Extracted filename + :raises ValueError: If file path is invalid + """ + if not file or not isinstance(file, str): + raise ValueError("File path must be a non-empty string") + return os.path.basename(file) # Using os.path.basename instead of split + + +def extract_file_name_with_source(file: str, source: str) -> str: + """ + Extract filename while preserving source directory structure. + + :param file: File path + :param source: Source directory path + :return: Extracted filename with source structure + :raises ValueError: If file or source paths are invalid + """ + if not file or not isinstance(file, str): + raise ValueError("File path must be a non-empty string") + if not source or not isinstance(source, str): + raise ValueError("Source path must be a non-empty string") + + source = source.rstrip('/') # Remove trailing slash if present + base = os.path.basename(source) return base + file.split(base)[-1] -def read_files_for_upload( - files -): +def read_files_for_upload(files: Dict[str, Any]) -> List[Tuple[str, Tuple[str, BinaryIO, str]]]: + """ + Prepare files for upload by creating appropriate tuples with file information. + + :param files: Dictionary containing file information + :return: List of tuples containing file upload information + :raises ValueError: If files dictionary is invalid + """ + if not isinstance(files, dict): + raise ValueError("Files must be a dictionary") + if "files" not in files or "is_dir" not in files or "path" not in files: + raise ValueError("Files dictionary missing required keys") + file_list = [] - for file in files["files"]: - if files["is_dir"]: + try: + for file in files["files"]: + if not os.path.exists(file): + raise ValueError(f"File does not exist: {file}") + + name = (extract_file_name_with_source(file, files["path"]) + if files["is_dir"] + else extract_file_name(file)) + file_list.append( ( "file", ( - extract_file_name_with_source(file, files["path"]), + name, open(file, "rb"), "application/octet-stream", ), ) ) - else: - file_list.append( - ( - "file", - ( - extract_file_name(file), - open(file, "rb"), - "application/octet-stream", - ), - ), - ) - return file_list - - -def close_files_after_upload( - files -) -> None: - for file in files: - file[1][1].close() + return file_list + except Exception as e: + # Clean up any opened files if an error occurs + for item in file_list: + try: + item[1][1].close() + except: + pass + raise ValueError(f"Failed to prepare files for upload: {str(e)}") + + +def close_files_after_upload(files: List[Tuple[str, Tuple[str, BinaryIO, str]]]) -> None: + """ + Close all file handles after upload. + + :param files: List of file tuples containing file handles + """ + if files: + for file in files: + try: + if file and len(file) > 1 and len(file[1]) > 1: + file[1][1].close() + except Exception: + # Continue closing other files even if one fails + pass