First_LongestWordFinder.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "365c32a4-672f-40a4-b4cd-60931c7f0af6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports\n",
    "import os           # module to access operating system \n",
    "import findspark    # find pyspark to make it importable\n",
    "import re           # regular expressions\n",
    "from pyspark import SparkConf, SparkContext\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8f4a5d95-966f-4257-a182-edd007a0ba2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize\n",
    "findspark.init()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "00275ed9-5258-43c1-a76a-7c0071bd6a41",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n",
      "25/01/13 21:54:21 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n"
     ]
    }
   ],
   "source": [
    "conf = SparkConf().setMaster(\"local[*]\").setAppName(\"LongestWordFinder\")\n",
    "sc = SparkContext(conf=conf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1aaf4904-2cd8-4b70-83c3-bb1d1bf5b733",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to find the longest word in a folder\n",
    "def find_longest_word(file_path):\n",
    "    rdd = sc.textFile(file_path)                            # set RDD (= resilient distributed dataset)\n",
    "    longest_word = (\n",
    "        rdd.flatMap(lambda t: re.findall(r\"\\w+\", t))         # Tokenize words (r = raw string)\n",
    "        .map(lambda w: (w, len(w)))                          # Map: Pair each word (key) with its length (value)\n",
    "        .reduce(lambda a, b: a if a[1] > b[1] else b)        # Reduce: Find the longest word by comparison\n",
    "    )\n",
    "    print(longest_word)\n",
    "    return longest_word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8dea25df-329e-4135-8f71-653825f3782b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load text files from this folder\n",
    "parent_folder = \"/Users/sophia1/Desktop/htw-longest-word/data\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "93361e5f-034a-4629-81d4-2b44680e88e1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/sophia1/Desktop/Texte8mal/Dutch/*.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('landbouwgereedschappen', 22)\n",
      "/Users/sophia1/Desktop/Texte8mal/German/*.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('eindusendsöbenhunnertuneiunsösstig', 34)\n",
      "/Users/sophia1/Desktop/Texte8mal/Russian/*.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('засвидетельствованных', 21)\n",
      "/Users/sophia1/Desktop/Texte8mal/Italian/*.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('quattrocentoquarantatremila', 27)\n",
      "/Users/sophia1/Desktop/Texte8mal/English/*.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Mekkamuselmannenmassenmenchenmoerdermohrenmuttermarmormonumentenmacher', 70)\n",
      "/Users/sophia1/Desktop/Texte8mal/French/*.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('constitutionnellement', 21)\n",
      "/Users/sophia1/Desktop/Texte8mal/Spanish/*.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('circunstanciadamente', 20)\n",
      "/Users/sophia1/Desktop/Texte8mal/Ukrainian/*.txt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 7:=========================================>             (280 + 8) / 368]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('благочестивомудренно', 20)\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "results = []\n",
    "\n",
    "# Loop through each language folder\n",
    "for language_folder in os.listdir(parent_folder):\n",
    "    language_path = os.path.join(parent_folder, language_folder)\n",
    "\n",
    "    # if it's a directory (it is a language folder) -> get all txt files\n",
    "    if os.path.isdir(language_path):\n",
    "        text_files = os.path.join(language_path, \"*.txt\")\n",
    "        print(text_files)\n",
    "        # Find the longest word in this language and save it\n",
    "        longest_word, length = find_longest_word(text_files)\n",
    "        results.append((language_folder, longest_word, length))\n",
    "print(f\"Execution time: {time.time() - start_time}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d8dbda0a-ec19-45f2-990a-c3d934f130f5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('English', 'Mekkamuselmannenmassenmenchenmoerdermohrenmuttermarmormonumentenmacher', 70), ('German', 'eindusendsöbenhunnertuneiunsösstig', 34), ('Italian', 'quattrocentoquarantatremila', 27), ('Dutch', 'landbouwgereedschappen', 22), ('Russian', 'засвидетельствованных', 21), ('French', 'constitutionnellement', 21), ('Spanish', 'circunstanciadamente', 20), ('Ukrainian', 'благочестивомудренно', 20)]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "# Sort results by the length of the longest word\n",
    "sorted_results = sorted(results, key=lambda x: x[2], reverse=True)\n",
    "print(sorted_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "762aa5aa-31ec-425b-a8ea-713f842cd493",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "English – Mekkamuselmannenmassenmenchenmoerdermohrenmuttermarmormonumentenmacher – 70\n",
      "German – eindusendsöbenhunnertuneiunsösstig – 34\n",
      "Italian – quattrocentoquarantatremila – 27\n",
      "Dutch – landbouwgereedschappen – 22\n",
      "Russian – засвидетельствованных – 21\n",
      "French – constitutionnellement – 21\n",
      "Spanish – circunstanciadamente – 20\n",
      "Ukrainian – благочестивомудренно – 20\n"
     ]
    }
   ],
   "source": [
    "for language, word, length in sorted_results:\n",
    "    print(f\"{language} – {word} – {length}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}