Compression amount now chunk-based, also versioned for PyPI

emcf · May 17, 2024 · a2aa550 · a2aa550
1 parent e7059a3
commit a2aa550
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/setup.py b/setup.py
@@ -2,7 +2,7 @@
 
 setup(
     name='thepipe_api',
-    version='0.3.4',
+    version='0.3.5',
     author='Emmett McFarlane',
     author_email='[email protected]',
     description='Automate information extraction for multimodal LLMs.',

diff --git a/thepipe_api/compressor.py b/thepipe_api/compressor.py
@@ -11,7 +11,7 @@
 CTAGS_EXECUTABLE_PATH = "C:\ctags.exe" if os.name == 'nt' else "ctags-universal"
 CTAGS_LANGUAGES = {'py': "Python", "cpp": "C++", "c": "C"}
 CTAGS_OUTPUT_FILE = 'ctags_output.json'
-MAX_COMPRESSION_ATTEMPTS = 10
+MAX_COMPRESSION_ATTEMPTS = 25
 
 def compress_with_ctags(chunk: Chunk, extension: str) -> Chunk:
     if chunk.text is None:
@@ -96,7 +96,7 @@ def calculate_tokens(chunk: Chunk) -> int:
 
 def compress_chunks(chunks: List[Chunk], verbose: bool = False, limit: Optional[int] = None) -> List[Chunk]:
     new_chunks = chunks
-    for _ in range(MAX_COMPRESSION_ATTEMPTS):
+    for _ in range(min(MAX_COMPRESSION_ATTEMPTS, len(chunks))):
         if count_tokens(new_chunks) <= limit:
             break
         if verbose: print_status(f"Compressing prompt ({count_tokens(chunks)} tokens / {limit} limit)", status='info')