Skip to content

Commit

Permalink
removed unused compression function
Browse files Browse the repository at this point in the history
  • Loading branch information
emcf committed Apr 28, 2024
1 parent 75301ed commit cc5c466
Showing 1 changed file with 0 additions and 11 deletions.
11 changes: 0 additions & 11 deletions thepipe_api/compressor.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,17 +68,6 @@ def compress_with_llmlingua(chunk: Chunk) -> Chunk:
new_chunk = Chunk(path=chunk.path, text=new_chunk_text, image=chunk.image, source_type=chunk.source_type)
return new_chunk

def compress_spreadsheet(chunk: Chunk) -> Chunk:
loaded_json = json.loads(chunk.text)
row_one = loaded_json[0]
colnames = []
coltypes = []
for key, value in row_one.items():
colnames.append(key)
coltypes.append(type(value))
new_chunk_text = "Column names and types: " + str(list(zip(colnames, coltypes)))
return Chunk(path=chunk.path, text=new_chunk_text, image=chunk.image, source_type=chunk.source_type)

def compress_chunks(chunks: List[Chunk], verbose: bool = False, limit: int = 1e5) -> List[Chunk]:
new_chunks = chunks
for _ in range(MAX_COMPRESSION_ATTEMPTS):
Expand Down

0 comments on commit cc5c466

Please sign in to comment.