lang.csv

import json
import csv
import glob
import os
from collections import defaultdict

input_folder = "/mnt/sdc/genius/original"
output_folder = "/mnt/sdc/genius/statistics"

language_count = defaultdict(int)

for file in glob.glob(f"{input_folder}/*.jsonl"):
    with open(file, 'r', encoding='utf-8') as f:
        for line in f:
            data = json.loads(line)
            if 'language' in data and data['language']:
                language_count[data['language']] += 1

os.makedirs(output_folder, exist_ok=True)

with open(f"{output_folder}/lang.csv", 'w', newline='', encoding='utf-8') as csvfile:
    writer = csv.writer(csvfile)
    writer.writerow(["Language", "Count"])
    for lang, count in sorted(language_count.items()):
        writer.writerow([lang, count])

print(f"Total unique languages found: {len(language_count)}")