Preprocess-Corpora.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 172,
   "id": "f8dc8ee5-b0aa-4de2-bd09-757b5a3b3c0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "#words = pd.read_csv(\"./corpora/Abhidharmakosa.txt\",\n",
    "#                    delimiter=\" \", dtype=str, index_col=False)\n",
    "with open(\"./corpora/Abhidharmakosa.txt\") as f:\n",
    "    lines = f.readlines()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "id": "dd4f9ad4-76f1-4b49-a21e-d5cef4b52b52",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['\\tom namo buddhāya / (1.1)\\n', '\\tyaḥ sarvathā sarvahatāndhakāraḥ saṃsārapaṅkājjagadujjahāra / (1.2)\\n', '\\ttasmai namaskṛtya yathārthaśāstre śāstraṃ pravakṣyāmyabhidharmakośam // (1.3)\\n', '\\tprajñāmalā sānucarābhidharmaḥ tatprāptaye yāpi ca yacca śāstram / (2.1)\\n', \"\\ttasyārthato'smin samanupraveśāt sa cāśrayo 'syetyabhidharmakośam // (2.2)\\n\", \"\\tdharmāṇāṃ pravicayamantareṇa nāsti kleśānāṃ yata upaśāntaye'bhyupāyaḥ / (3.1)\\n\", \"\\tkleśaiśca bhramati bhavārṇave'tra lokastaddhetorata uditaḥ kilaiṣa śāstrā // (3.2)\\n\", '\\tsāsravānāsravā dharmāḥ saṃskṛtā mārgavarjitāḥ / (4.1)\\n', '\\tsāsravāḥ āsravāsteṣu yasmāt samanuśerate // (4.2)\\n', '\\tanāsravā mārgasatyaṃ trividhaṃ cāpyasaṃskṛtam / (5.1)\\n', '\\tākāśaṃ dvau nirodhau ca tatrākāśam anāvṛtiḥ // (5.2)\\n', '\\tpratisaṃkhyānirodho yo visaṃyogaḥ pṛthak pṛthak / (6.1)\\n', \"\\tutpādātyantavighno'nyo nirodho'pratisaṃkhyayā // (6.2)\\n\", '\\tte punaḥ saṃskṛtā dharmā rūpādiskandhapañcakam / (7.1)\\n', '\\tta evādhvā kathāvastu saniḥsārāḥ savastukāḥ // (7.2)\\n', '\\tye sāsravā upādānaskandhāste saraṇā api / (8.1)\\n', '\\tduḥkhaṃ samudayo loko dṛṣṭisthānaṃ bhavaśca te // (8.2)\\n', '\\trūpaṃ pañcendriyāṇyarthāḥ pañcāvijñaptireva ca / (9.1)\\n', '\\ttadvijñānāśrayā rūpaprasādāścakṣurādayaḥ // (9.2)\\n', '\\trūpaṃ dvidhā viṃśatidhā śabdastvaṣṭavidhaḥ rasaḥ / (10.1)\\n', '\\tṣoḍhā caturvidho gandhaḥ spṛśyamekādaśātmakam // (10.2)\\n', \"\\tvikṣiptācittakasyāpi yo'nubandhaḥ śubhāśubhaḥ / (11.1)\\n\", '\\tmahābhūtānyupādāya sa hyavijñaptirucyate // (11.2)\\n', '\\tbhūtāni pṛthividhāturaptejovāyudhātavaḥ / (12.1)\\n', '\\tdhṛtyādikarmasaṃsiddhāḥ kharasnehoṣṇateraṇāḥ // (12.2)\\n', '\\tpṛthivī varṇasaṃsthānamucyate lokasaṃjñayā / (13.1)\\n', '\\tāpastejaśca vāyustu dhātureva tathāpi ca // (13.2)\\n', '\\tindriyārthāsta eveṣṭā daśāyatanadhātavaḥ / (14.1)\\n', '\\tvedanānubhavaḥ saṃjñā nimittodgrahaṇātmikā // (14.2)\\n', \"\\tcaturbhyo'nye tu saṃskāraskandhaḥ ete punastrayaḥ / (15.1)\\n\", '\\tdharmāyatanadhātvākhyāḥ sahāvijñaptyasaṃskṛtaiḥ // (15.2)\\n', '\\tvijñānaṃ prativijñaptiḥ mana āyatanaṃ ca tat / (16.1)\\n', '\\tdhātavaḥ sapta ca matāḥ ṣaḍ vijñānānyatho manaḥ // (16.2)\\n', '\\tṣaṇṇām anantarātītaṃ vijñānaṃ yaddhi tanmanaḥ / (17.1)\\n', \"\\tṣaṣṭhāśrayaprasiddhyarthaṃ dhātavo 'ṣṭādaśa smṛtāḥ // (17.2)\\n\", '\\tsarvasaṃgraha ekena skandhenāyatanena ca / (18.1)\\n', '\\tdhātunā ca svabhāvena parabhāvaviyogataḥ // (18.2)\\n', '\\tjātigocaravijñānasāmānyādekadhātutā / (19.1)\\n', \"\\tdvitve'pi cakṣurādīnāṃ śobhārthaṃ tu dvayodbhavaḥ // (19.2)\\n\", '\\trāśyāyadvāragotrārthāḥ skandhāyatanadhātavaḥ / (20.1)\\n', '\\tmohendriyarucitraidhāt tisraḥ skandhādideśanāḥ // (20.2)\\n', '\\tvivādamūlasaṃsārahetutvāt kramakāraṇāt / (21.1)\\n', '\\tcaittebhyo vedanāsaṃjñe pṛthakskandhau niveśitau // (21.2)\\n', '\\tskandheṣvasaṃskṛtaṃ noktamarthāyogāt kramaḥ punaḥ / (22.1)\\n', '\\tyathaudārikasaṃkleśabhājanādyarthadhātutaḥ // (22.2)\\n', '\\tprāk pañca vārttamānārthyāt bhautikārthyāccatuṣṭayam / (23.1)\\n', \"\\tdūrāśutaravṛttyānyat yathāsthānaṃ kramo'thavā // (23.2)\\n\", '\\tviśeṣaṇārthaṃ prādhānyād bahudharmāgrasaṃgrahāt / (24.1)\\n', '\\tekamāyatanaṃ rūpamekaṃ dharmākhyamucyate // (24.2)\\n', '\\tdharmaskandhasahasrāṇi yānyaśītiṃ jagau muniḥ / (25.1)\\n', '\\ttāni vāṅnāma vetyeṣāṃ rūpasaṃskārasaṃgrahaḥ // (25.2)\\n', '\\tśāstrapramāṇā ityeke skandhādīnāṃ kathaikaśaḥ / (26.1)\\n', \"\\tcaritapratipakṣastu dharmaskandho'nuvarṇitaḥ // (26.2)\\n\", \"\\ttathānye'pi yathāyogaṃ skandhāyatanadhātavaḥ / (27.1)\\n\", '\\tpratipādyā yathokteṣu sampradhārya svalakṣaṇam // (27.2)\\n', '\\tchidramākāśadhātvākhyam ālokatamasī kila / (28.1)\\n', '\\tvijñānadhāturvijñānaṃ sāsravaṃ janmaniśrayāḥ // (28.2)\\n', \"\\tsanidarśana eko'tra rūpaṃ sapratighā daśa / (29.1)\\n\", '\\trūpiṇaḥ avyākṛtā aṣṭau ta evārūpaśabdakāḥ // (29.2)\\n', '\\ttridhānye kāmadhātvāptāḥ sarve rūpe caturdaśa / (30.1)\\n', '\\tvinā gandharasaghrāṇajihvāvijñānadhātubhiḥ // (30.2)\\n', '\\tārūpyāptā manodharmamanovijñānadhātavaḥ / (31.1)\\n', '\\tsāsravānāsravā ete trayaḥ śeṣāstu sāsravāḥ // (31.2)\\n', '\\tsavitarkavicārā hi pañca vijñānadhātavaḥ / (32.1)\\n', '\\tantyāstrayastriprakārāḥ śeṣā ubhayavarjitāḥ // (32.2)\\n', '\\tnirūpaṇānusmaraṇavikalpenāvikalpakāḥ / (33.1)\\n', '\\ttau prajñā mānasī vyagrā smṛtiḥ sarvaiva mānasī // (33.2)\\n', '\\tsapta sālambanāścittadhātavaḥ ardhaṃ ca dharmataḥ / (34.1)\\n', '\\tnavānupāttās te cāṣṭau śabdaśca anye nava dvidhā // (34.2)\\n', '\\tspraṣṭavyaṃ dvividhaṃ śeṣā rūpiṇo nava bhautikāḥ / (35.1)\\n', '\\tdharmadhātvekadeśaśca saṃcitā daśa rūpiṇaḥ // (35.2)\\n', '\\tchinatti chidyate caiva bāhyaṃ dhātucatuṣṭayam / (36.1)\\n', '\\tdahyate tulayatyevaṃ vivādo dagdhṛtulyayoḥ // (36.2)\\n', '\\tvipākajaupacayikāḥ pañcādhyātmaṃ vipākajaḥ / (37.1)\\n', '\\tna śabdaḥ apratighā aṣṭau naiḥṣyandikavipākajāḥ // (37.2)\\n', '\\ttridhānye dravyavānekaḥ kṣaṇikāḥ paścimāstrayaḥ / (38.1)\\n', '\\tcakṣurvijñānadhātvoḥ syāt pṛthak lābhaḥ sahāpi ca // (38.2)\\n', '\\tdvādaśādhyātmikāḥ hitvā rūpādīn dharmasaṃjñakaḥ / (39.1)\\n', '\\tsabhāgaḥ tatsabhāgāśca śeṣāḥ yo na svakarmakṛt // (39.2)\\n', '\\tdaśa bhāvanayā heyāḥ pañca ca antyāstrayastridhā / (40.1)\\n', '\\tna dṛṣṭiheyamakliṣṭaṃ na rūpaṃ nāpyaṣaṣṭhajam // (40.2)\\n', '\\tcakṣuśca dharmadhātośca pradeśo dṛṣṭiḥ aṣṭadhā / (41.1)\\n', '\\tpañcavijñānasahajā dhīrna dṛṣṭiratīraṇāt // (41.2)\\n', '\\tcakṣuḥ paśyati rūpāṇi sabhāgaṃ na tadāśritam / (42.1)\\n', '\\tvijñānaṃ dṛśyate rūpaṃ na kilāntaritaṃ yataḥ // (42.2)\\n', '\\tubhābhyāmapi cakṣurbhyāṃ paśyati vyaktadarśanāt / (43.1)\\n', \"\\tcakṣuḥśrotramano'prāptaviṣayaṃ trayamanyathā // (43.2)\\n\", '\\ttribhirghrāṇādibhistulyaviṣayagrahaṇaṃ matam / (44.1)\\n', \"\\tcaramasyāśrayo'tītaḥ pañcānāṃ sahajaśca taiḥ // (44.2)\\n\", '\\ttadvikāravikāritvād āśrayāś cakṣurādayaḥ / (45.1)\\n', \"\\tato'sādhāraṇatvāddhi vijñānaṃ tairnirucyate // (45.2)\\n\", '\\tna kāyasyādharaṃ cakṣuḥ ūrdhvaṃ rūpaṃ na cakṣuṣaḥ / (46.1)\\n', '\\tvijñānaṃ ca asya rūpaṃ tu kāyasyobhe ca sarvataḥ // (46.2)\\n', '\\ttathā śrotraṃ trayāṇāṃ tu sarvameva svabhūmikam / (47.1)\\n', '\\tkāyavijñānamadharasvabhūmi aniyataṃ manaḥ // (47.2)\\n', '\\tpañca bāhyā dvivijñeyāḥ nityā dharmā asaṃskṛtāḥ / (48.1)\\n', '\\tdharmārdhamindriyaṃ ye ca dvādaśādhyātmikāḥ smṛtāḥ // (48.2)\\n', '\\tabhidharmakośabhāṣye dhātunirdeśo nāma prathamaṃ kośasthānaṃ samāptamiti / (49.1)\\n', '\\t\\n']\n"
     ]
    }
   ],
   "source": [
    "print(lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "id": "0ff7af15-f2e3-4d3d-9308-7cc9d06a594e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "om namo buddhāya \n",
      "\n"
     ]
    }
   ],
   "source": [
    "df = pd.DataFrame(data=lines, columns=['text'])\n",
    "df = df.replace(to_replace=r'\\/.*$', value='', regex=True)\n",
    "df = df.replace(to_replace=r'^\\s*\\t*', value='', regex=True)\n",
    "print(df['text'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "id": "3d5e3013-c1ff-4851-8401-647dda500e04",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dharmāyatanadhātvākhyāḥ sahāvijñaptyasaṃskṛtaiḥ \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(df['text'][30])\n",
    "df = df['text'].str.split(expand=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "id": "ad84088d-d1d3-4cb0-afb1-7088bb28c40a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>om</td>\n",
       "      <td>namo</td>\n",
       "      <td>buddhāya</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>yaḥ</td>\n",
       "      <td>sarvathā</td>\n",
       "      <td>sarvahatāndhakāraḥ</td>\n",
       "      <td>saṃsārapaṅkājjagadujjahāra</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tasmai</td>\n",
       "      <td>namaskṛtya</td>\n",
       "      <td>yathārthaśāstre</td>\n",
       "      <td>śāstraṃ</td>\n",
       "      <td>pravakṣyāmyabhidharmakośam</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>prajñāmalā</td>\n",
       "      <td>sānucarābhidharmaḥ</td>\n",
       "      <td>tatprāptaye</td>\n",
       "      <td>yāpi</td>\n",
       "      <td>ca</td>\n",
       "      <td>yacca</td>\n",
       "      <td>śāstram</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>tasyārthato'smin</td>\n",
       "      <td>samanupraveśāt</td>\n",
       "      <td>sa</td>\n",
       "      <td>cāśrayo</td>\n",
       "      <td>'syetyabhidharmakośam</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>kāyavijñānamadharasvabhūmi</td>\n",
       "      <td>aniyataṃ</td>\n",
       "      <td>manaḥ</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>pañca</td>\n",
       "      <td>bāhyā</td>\n",
       "      <td>dvivijñeyāḥ</td>\n",
       "      <td>nityā</td>\n",
       "      <td>dharmā</td>\n",
       "      <td>asaṃskṛtāḥ</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>dharmārdhamindriyaṃ</td>\n",
       "      <td>ye</td>\n",
       "      <td>ca</td>\n",
       "      <td>dvādaśādhyātmikāḥ</td>\n",
       "      <td>smṛtāḥ</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>abhidharmakośabhāṣye</td>\n",
       "      <td>dhātunirdeśo</td>\n",
       "      <td>nāma</td>\n",
       "      <td>prathamaṃ</td>\n",
       "      <td>kośasthānaṃ</td>\n",
       "      <td>samāptamiti</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>99 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             0                   1                   2  \\\n",
       "0                           om                namo            buddhāya   \n",
       "1                          yaḥ            sarvathā  sarvahatāndhakāraḥ   \n",
       "2                       tasmai          namaskṛtya     yathārthaśāstre   \n",
       "3                   prajñāmalā  sānucarābhidharmaḥ         tatprāptaye   \n",
       "4             tasyārthato'smin      samanupraveśāt                  sa   \n",
       "..                         ...                 ...                 ...   \n",
       "94  kāyavijñānamadharasvabhūmi            aniyataṃ               manaḥ   \n",
       "95                       pañca               bāhyā         dvivijñeyāḥ   \n",
       "96         dharmārdhamindriyaṃ                  ye                  ca   \n",
       "97        abhidharmakośabhāṣye        dhātunirdeśo                nāma   \n",
       "98                        None                None                None   \n",
       "\n",
       "                             3                           4            5  \\\n",
       "0                         None                        None         None   \n",
       "1   saṃsārapaṅkājjagadujjahāra                        None         None   \n",
       "2                      śāstraṃ  pravakṣyāmyabhidharmakośam         None   \n",
       "3                         yāpi                          ca        yacca   \n",
       "4                      cāśrayo       'syetyabhidharmakośam         None   \n",
       "..                         ...                         ...          ...   \n",
       "94                        None                        None         None   \n",
       "95                       nityā                      dharmā   asaṃskṛtāḥ   \n",
       "96           dvādaśādhyātmikāḥ                      smṛtāḥ         None   \n",
       "97                   prathamaṃ                 kośasthānaṃ  samāptamiti   \n",
       "98                        None                        None         None   \n",
       "\n",
       "          6     7  \n",
       "0      None  None  \n",
       "1      None  None  \n",
       "2      None  None  \n",
       "3   śāstram  None  \n",
       "4      None  None  \n",
       "..      ...   ...  \n",
       "94     None  None  \n",
       "95     None  None  \n",
       "96     None  None  \n",
       "97     None  None  \n",
       "98     None  None  \n",
       "\n",
       "[99 rows x 8 columns]"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "id": "89f773c1-e36e-4adb-85e6-e0e46f29eb5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "new = df.values.flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "id": "5ce9edf6-081d-4d0d-b867-bde02df236bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['om', 'namo', 'buddhāya', None, None, None, None, None, 'yaḥ',\n",
       "       'sarvathā', 'sarvahatāndhakāraḥ', 'saṃsārapaṅkājjagadujjahāra',\n",
       "       None, None, None, None, 'tasmai', 'namaskṛtya', 'yathārthaśāstre',\n",
       "       'śāstraṃ', 'pravakṣyāmyabhidharmakośam', None, None, None,\n",
       "       'prajñāmalā', 'sānucarābhidharmaḥ', 'tatprāptaye', 'yāpi', 'ca',\n",
       "       'yacca', 'śāstram', None, \"tasyārthato'smin\", 'samanupraveśāt',\n",
       "       'sa', 'cāśrayo', \"'syetyabhidharmakośam\", None, None, None,\n",
       "       'dharmāṇāṃ', 'pravicayamantareṇa', 'nāsti', 'kleśānāṃ', 'yata',\n",
       "       \"upaśāntaye'bhyupāyaḥ\", None, None, 'kleśaiśca', 'bhramati',\n",
       "       \"bhavārṇave'tra\", 'lokastaddhetorata', 'uditaḥ', 'kilaiṣa',\n",
       "       'śāstrā', None, 'sāsravānāsravā', 'dharmāḥ', 'saṃskṛtā',\n",
       "       'mārgavarjitāḥ', None, None, None, None, 'sāsravāḥ', 'āsravāsteṣu',\n",
       "       'yasmāt', 'samanuśerate', None, None, None, None, 'anāsravā',\n",
       "       'mārgasatyaṃ', 'trividhaṃ', 'cāpyasaṃskṛtam', None, None, None,\n",
       "       None, 'ākāśaṃ', 'dvau', 'nirodhau', 'ca', 'tatrākāśam', 'anāvṛtiḥ',\n",
       "       None, None, 'pratisaṃkhyānirodho', 'yo', 'visaṃyogaḥ', 'pṛthak',\n",
       "       'pṛthak', None, None, None, \"utpādātyantavighno'nyo\",\n",
       "       \"nirodho'pratisaṃkhyayā\", None, None, None, None, None, None, 'te',\n",
       "       'punaḥ', 'saṃskṛtā', 'dharmā', 'rūpādiskandhapañcakam', None, None,\n",
       "       None, 'ta', 'evādhvā', 'kathāvastu', 'saniḥsārāḥ', 'savastukāḥ',\n",
       "       None, None, None, 'ye', 'sāsravā', 'upādānaskandhāste', 'saraṇā',\n",
       "       'api', None, None, None, 'duḥkhaṃ', 'samudayo', 'loko',\n",
       "       'dṛṣṭisthānaṃ', 'bhavaśca', 'te', None, None, 'rūpaṃ',\n",
       "       'pañcendriyāṇyarthāḥ', 'pañcāvijñaptireva', 'ca', None, None, None,\n",
       "       None, 'tadvijñānāśrayā', 'rūpaprasādāścakṣurādayaḥ', None, None,\n",
       "       None, None, None, None, 'rūpaṃ', 'dvidhā', 'viṃśatidhā',\n",
       "       'śabdastvaṣṭavidhaḥ', 'rasaḥ', None, None, None, 'ṣoḍhā',\n",
       "       'caturvidho', 'gandhaḥ', 'spṛśyamekādaśātmakam', None, None, None,\n",
       "       None, 'vikṣiptācittakasyāpi', \"yo'nubandhaḥ\", 'śubhāśubhaḥ', None,\n",
       "       None, None, None, None, 'mahābhūtānyupādāya', 'sa',\n",
       "       'hyavijñaptirucyate', None, None, None, None, None, 'bhūtāni',\n",
       "       'pṛthividhāturaptejovāyudhātavaḥ', None, None, None, None, None,\n",
       "       None, 'dhṛtyādikarmasaṃsiddhāḥ', 'kharasnehoṣṇateraṇāḥ', None,\n",
       "       None, None, None, None, None, 'pṛthivī', 'varṇasaṃsthānamucyate',\n",
       "       'lokasaṃjñayā', None, None, None, None, None, 'āpastejaśca',\n",
       "       'vāyustu', 'dhātureva', 'tathāpi', 'ca', None, None, None,\n",
       "       'indriyārthāsta', 'eveṣṭā', 'daśāyatanadhātavaḥ', None, None, None,\n",
       "       None, None, 'vedanānubhavaḥ', 'saṃjñā', 'nimittodgrahaṇātmikā',\n",
       "       None, None, None, None, None, \"caturbhyo'nye\", 'tu',\n",
       "       'saṃskāraskandhaḥ', 'ete', 'punastrayaḥ', None, None, None,\n",
       "       'dharmāyatanadhātvākhyāḥ', 'sahāvijñaptyasaṃskṛtaiḥ', None, None,\n",
       "       None, None, None, None, 'vijñānaṃ', 'prativijñaptiḥ', 'mana',\n",
       "       'āyatanaṃ', 'ca', 'tat', None, None, 'dhātavaḥ', 'sapta', 'ca',\n",
       "       'matāḥ', 'ṣaḍ', 'vijñānānyatho', 'manaḥ', None, 'ṣaṇṇām',\n",
       "       'anantarātītaṃ', 'vijñānaṃ', 'yaddhi', 'tanmanaḥ', None, None,\n",
       "       None, 'ṣaṣṭhāśrayaprasiddhyarthaṃ', 'dhātavo', \"'ṣṭādaśa\",\n",
       "       'smṛtāḥ', None, None, None, None, 'sarvasaṃgraha', 'ekena',\n",
       "       'skandhenāyatanena', 'ca', None, None, None, None, 'dhātunā', 'ca',\n",
       "       'svabhāvena', 'parabhāvaviyogataḥ', None, None, None, None,\n",
       "       'jātigocaravijñānasāmānyādekadhātutā', None, None, None, None,\n",
       "       None, None, None, \"dvitve'pi\", 'cakṣurādīnāṃ', 'śobhārthaṃ', 'tu',\n",
       "       'dvayodbhavaḥ', None, None, None, 'rāśyāyadvāragotrārthāḥ',\n",
       "       'skandhāyatanadhātavaḥ', None, None, None, None, None, None,\n",
       "       'mohendriyarucitraidhāt', 'tisraḥ', 'skandhādideśanāḥ', None, None,\n",
       "       None, None, None, 'vivādamūlasaṃsārahetutvāt', 'kramakāraṇāt',\n",
       "       None, None, None, None, None, None, 'caittebhyo', 'vedanāsaṃjñe',\n",
       "       'pṛthakskandhau', 'niveśitau', None, None, None, None,\n",
       "       'skandheṣvasaṃskṛtaṃ', 'noktamarthāyogāt', 'kramaḥ', 'punaḥ', None,\n",
       "       None, None, None, 'yathaudārikasaṃkleśabhājanādyarthadhātutaḥ',\n",
       "       None, None, None, None, None, None, None, 'prāk', 'pañca',\n",
       "       'vārttamānārthyāt', 'bhautikārthyāccatuṣṭayam', None, None, None,\n",
       "       None, 'dūrāśutaravṛttyānyat', 'yathāsthānaṃ', \"kramo'thavā\", None,\n",
       "       None, None, None, None, 'viśeṣaṇārthaṃ', 'prādhānyād',\n",
       "       'bahudharmāgrasaṃgrahāt', None, None, None, None, None,\n",
       "       'ekamāyatanaṃ', 'rūpamekaṃ', 'dharmākhyamucyate', None, None, None,\n",
       "       None, None, 'dharmaskandhasahasrāṇi', 'yānyaśītiṃ', 'jagau',\n",
       "       'muniḥ', None, None, None, None, 'tāni', 'vāṅnāma', 'vetyeṣāṃ',\n",
       "       'rūpasaṃskārasaṃgrahaḥ', None, None, None, None, 'śāstrapramāṇā',\n",
       "       'ityeke', 'skandhādīnāṃ', 'kathaikaśaḥ', None, None, None, None,\n",
       "       'caritapratipakṣastu', \"dharmaskandho'nuvarṇitaḥ\", None, None,\n",
       "       None, None, None, None, \"tathānye'pi\", 'yathāyogaṃ',\n",
       "       'skandhāyatanadhātavaḥ', None, None, None, None, None,\n",
       "       'pratipādyā', 'yathokteṣu', 'sampradhārya', 'svalakṣaṇam', None,\n",
       "       None, None, None, 'chidramākāśadhātvākhyam', 'ālokatamasī', 'kila',\n",
       "       None, None, None, None, None, 'vijñānadhāturvijñānaṃ', 'sāsravaṃ',\n",
       "       'janmaniśrayāḥ', None, None, None, None, None, 'sanidarśana',\n",
       "       \"eko'tra\", 'rūpaṃ', 'sapratighā', 'daśa', None, None, None,\n",
       "       'rūpiṇaḥ', 'avyākṛtā', 'aṣṭau', 'ta', 'evārūpaśabdakāḥ', None,\n",
       "       None, None, 'tridhānye', 'kāmadhātvāptāḥ', 'sarve', 'rūpe',\n",
       "       'caturdaśa', None, None, None, 'vinā',\n",
       "       'gandharasaghrāṇajihvāvijñānadhātubhiḥ', None, None, None, None,\n",
       "       None, None, 'ārūpyāptā', 'manodharmamanovijñānadhātavaḥ', None,\n",
       "       None, None, None, None, None, 'sāsravānāsravā', 'ete', 'trayaḥ',\n",
       "       'śeṣāstu', 'sāsravāḥ', None, None, None, 'savitarkavicārā', 'hi',\n",
       "       'pañca', 'vijñānadhātavaḥ', None, None, None, None,\n",
       "       'antyāstrayastriprakārāḥ', 'śeṣā', 'ubhayavarjitāḥ', None, None,\n",
       "       None, None, None, 'nirūpaṇānusmaraṇavikalpenāvikalpakāḥ', None,\n",
       "       None, None, None, None, None, None, 'tau', 'prajñā', 'mānasī',\n",
       "       'vyagrā', 'smṛtiḥ', 'sarvaiva', 'mānasī', None, 'sapta',\n",
       "       'sālambanāścittadhātavaḥ', 'ardhaṃ', 'ca', 'dharmataḥ', None, None,\n",
       "       None, 'navānupāttās', 'te', 'cāṣṭau', 'śabdaśca', 'anye', 'nava',\n",
       "       'dvidhā', None, 'spraṣṭavyaṃ', 'dvividhaṃ', 'śeṣā', 'rūpiṇo',\n",
       "       'nava', 'bhautikāḥ', None, None, 'dharmadhātvekadeśaśca',\n",
       "       'saṃcitā', 'daśa', 'rūpiṇaḥ', None, None, None, None, 'chinatti',\n",
       "       'chidyate', 'caiva', 'bāhyaṃ', 'dhātucatuṣṭayam', None, None, None,\n",
       "       'dahyate', 'tulayatyevaṃ', 'vivādo', 'dagdhṛtulyayoḥ', None, None,\n",
       "       None, None, 'vipākajaupacayikāḥ', 'pañcādhyātmaṃ', 'vipākajaḥ',\n",
       "       None, None, None, None, None, 'na', 'śabdaḥ', 'apratighā', 'aṣṭau',\n",
       "       'naiḥṣyandikavipākajāḥ', None, None, None, 'tridhānye',\n",
       "       'dravyavānekaḥ', 'kṣaṇikāḥ', 'paścimāstrayaḥ', None, None, None,\n",
       "       None, 'cakṣurvijñānadhātvoḥ', 'syāt', 'pṛthak', 'lābhaḥ', 'sahāpi',\n",
       "       'ca', None, None, 'dvādaśādhyātmikāḥ', 'hitvā', 'rūpādīn',\n",
       "       'dharmasaṃjñakaḥ', None, None, None, None, 'sabhāgaḥ',\n",
       "       'tatsabhāgāśca', 'śeṣāḥ', 'yo', 'na', 'svakarmakṛt', None, None,\n",
       "       'daśa', 'bhāvanayā', 'heyāḥ', 'pañca', 'ca', 'antyāstrayastridhā',\n",
       "       None, None, 'na', 'dṛṣṭiheyamakliṣṭaṃ', 'na', 'rūpaṃ',\n",
       "       'nāpyaṣaṣṭhajam', None, None, None, 'cakṣuśca', 'dharmadhātośca',\n",
       "       'pradeśo', 'dṛṣṭiḥ', 'aṣṭadhā', None, None, None,\n",
       "       'pañcavijñānasahajā', 'dhīrna', 'dṛṣṭiratīraṇāt', None, None, None,\n",
       "       None, None, 'cakṣuḥ', 'paśyati', 'rūpāṇi', 'sabhāgaṃ', 'na',\n",
       "       'tadāśritam', None, None, 'vijñānaṃ', 'dṛśyate', 'rūpaṃ', 'na',\n",
       "       'kilāntaritaṃ', 'yataḥ', None, None, 'ubhābhyāmapi', 'cakṣurbhyāṃ',\n",
       "       'paśyati', 'vyaktadarśanāt', None, None, None, None,\n",
       "       \"cakṣuḥśrotramano'prāptaviṣayaṃ\", 'trayamanyathā', None, None,\n",
       "       None, None, None, None, 'tribhirghrāṇādibhistulyaviṣayagrahaṇaṃ',\n",
       "       'matam', None, None, None, None, None, None,\n",
       "       \"caramasyāśrayo'tītaḥ\", 'pañcānāṃ', 'sahajaśca', 'taiḥ', None,\n",
       "       None, None, None, 'tadvikāravikāritvād', 'āśrayāś', 'cakṣurādayaḥ',\n",
       "       None, None, None, None, None, \"ato'sādhāraṇatvāddhi\", 'vijñānaṃ',\n",
       "       'tairnirucyate', None, None, None, None, None, 'na',\n",
       "       'kāyasyādharaṃ', 'cakṣuḥ', 'ūrdhvaṃ', 'rūpaṃ', 'na', 'cakṣuṣaḥ',\n",
       "       None, 'vijñānaṃ', 'ca', 'asya', 'rūpaṃ', 'tu', 'kāyasyobhe', 'ca',\n",
       "       'sarvataḥ', 'tathā', 'śrotraṃ', 'trayāṇāṃ', 'tu', 'sarvameva',\n",
       "       'svabhūmikam', None, None, 'kāyavijñānamadharasvabhūmi',\n",
       "       'aniyataṃ', 'manaḥ', None, None, None, None, None, 'pañca',\n",
       "       'bāhyā', 'dvivijñeyāḥ', 'nityā', 'dharmā', 'asaṃskṛtāḥ', None,\n",
       "       None, 'dharmārdhamindriyaṃ', 'ye', 'ca', 'dvādaśādhyātmikāḥ',\n",
       "       'smṛtāḥ', None, None, None, 'abhidharmakośabhāṣye', 'dhātunirdeśo',\n",
       "       'nāma', 'prathamaṃ', 'kośasthānaṃ', 'samāptamiti', None, None,\n",
       "       None, None, None, None, None, None, None, None], dtype=object)"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "id": "1d2b53c0-60aa-43c2-bfc0-3d14c24bec7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "clean = np.extract(new != None, new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "id": "efef1601-2ee6-4fd3-af34-7fe5724bb195",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['om', 'namo', 'buddhāya', 'yaḥ', 'sarvathā', 'sarvahatāndhakāraḥ',\n",
       "       'saṃsārapaṅkājjagadujjahāra', 'tasmai', 'namaskṛtya',\n",
       "       'yathārthaśāstre', 'śāstraṃ', 'pravakṣyāmyabhidharmakośam',\n",
       "       'prajñāmalā', 'sānucarābhidharmaḥ', 'tatprāptaye', 'yāpi', 'ca',\n",
       "       'yacca', 'śāstram', \"tasyārthato'smin\", 'samanupraveśāt', 'sa',\n",
       "       'cāśrayo', \"'syetyabhidharmakośam\", 'dharmāṇāṃ',\n",
       "       'pravicayamantareṇa', 'nāsti', 'kleśānāṃ', 'yata',\n",
       "       \"upaśāntaye'bhyupāyaḥ\", 'kleśaiśca', 'bhramati', \"bhavārṇave'tra\",\n",
       "       'lokastaddhetorata', 'uditaḥ', 'kilaiṣa', 'śāstrā',\n",
       "       'sāsravānāsravā', 'dharmāḥ', 'saṃskṛtā', 'mārgavarjitāḥ',\n",
       "       'sāsravāḥ', 'āsravāsteṣu', 'yasmāt', 'samanuśerate', 'anāsravā',\n",
       "       'mārgasatyaṃ', 'trividhaṃ', 'cāpyasaṃskṛtam', 'ākāśaṃ', 'dvau',\n",
       "       'nirodhau', 'ca', 'tatrākāśam', 'anāvṛtiḥ', 'pratisaṃkhyānirodho',\n",
       "       'yo', 'visaṃyogaḥ', 'pṛthak', 'pṛthak', \"utpādātyantavighno'nyo\",\n",
       "       \"nirodho'pratisaṃkhyayā\", 'te', 'punaḥ', 'saṃskṛtā', 'dharmā',\n",
       "       'rūpādiskandhapañcakam', 'ta', 'evādhvā', 'kathāvastu',\n",
       "       'saniḥsārāḥ', 'savastukāḥ', 'ye', 'sāsravā', 'upādānaskandhāste',\n",
       "       'saraṇā', 'api', 'duḥkhaṃ', 'samudayo', 'loko', 'dṛṣṭisthānaṃ',\n",
       "       'bhavaśca', 'te', 'rūpaṃ', 'pañcendriyāṇyarthāḥ',\n",
       "       'pañcāvijñaptireva', 'ca', 'tadvijñānāśrayā',\n",
       "       'rūpaprasādāścakṣurādayaḥ', 'rūpaṃ', 'dvidhā', 'viṃśatidhā',\n",
       "       'śabdastvaṣṭavidhaḥ', 'rasaḥ', 'ṣoḍhā', 'caturvidho', 'gandhaḥ',\n",
       "       'spṛśyamekādaśātmakam', 'vikṣiptācittakasyāpi', \"yo'nubandhaḥ\",\n",
       "       'śubhāśubhaḥ', 'mahābhūtānyupādāya', 'sa', 'hyavijñaptirucyate',\n",
       "       'bhūtāni', 'pṛthividhāturaptejovāyudhātavaḥ',\n",
       "       'dhṛtyādikarmasaṃsiddhāḥ', 'kharasnehoṣṇateraṇāḥ', 'pṛthivī',\n",
       "       'varṇasaṃsthānamucyate', 'lokasaṃjñayā', 'āpastejaśca', 'vāyustu',\n",
       "       'dhātureva', 'tathāpi', 'ca', 'indriyārthāsta', 'eveṣṭā',\n",
       "       'daśāyatanadhātavaḥ', 'vedanānubhavaḥ', 'saṃjñā',\n",
       "       'nimittodgrahaṇātmikā', \"caturbhyo'nye\", 'tu', 'saṃskāraskandhaḥ',\n",
       "       'ete', 'punastrayaḥ', 'dharmāyatanadhātvākhyāḥ',\n",
       "       'sahāvijñaptyasaṃskṛtaiḥ', 'vijñānaṃ', 'prativijñaptiḥ', 'mana',\n",
       "       'āyatanaṃ', 'ca', 'tat', 'dhātavaḥ', 'sapta', 'ca', 'matāḥ', 'ṣaḍ',\n",
       "       'vijñānānyatho', 'manaḥ', 'ṣaṇṇām', 'anantarātītaṃ', 'vijñānaṃ',\n",
       "       'yaddhi', 'tanmanaḥ', 'ṣaṣṭhāśrayaprasiddhyarthaṃ', 'dhātavo',\n",
       "       \"'ṣṭādaśa\", 'smṛtāḥ', 'sarvasaṃgraha', 'ekena',\n",
       "       'skandhenāyatanena', 'ca', 'dhātunā', 'ca', 'svabhāvena',\n",
       "       'parabhāvaviyogataḥ', 'jātigocaravijñānasāmānyādekadhātutā',\n",
       "       \"dvitve'pi\", 'cakṣurādīnāṃ', 'śobhārthaṃ', 'tu', 'dvayodbhavaḥ',\n",
       "       'rāśyāyadvāragotrārthāḥ', 'skandhāyatanadhātavaḥ',\n",
       "       'mohendriyarucitraidhāt', 'tisraḥ', 'skandhādideśanāḥ',\n",
       "       'vivādamūlasaṃsārahetutvāt', 'kramakāraṇāt', 'caittebhyo',\n",
       "       'vedanāsaṃjñe', 'pṛthakskandhau', 'niveśitau',\n",
       "       'skandheṣvasaṃskṛtaṃ', 'noktamarthāyogāt', 'kramaḥ', 'punaḥ',\n",
       "       'yathaudārikasaṃkleśabhājanādyarthadhātutaḥ', 'prāk', 'pañca',\n",
       "       'vārttamānārthyāt', 'bhautikārthyāccatuṣṭayam',\n",
       "       'dūrāśutaravṛttyānyat', 'yathāsthānaṃ', \"kramo'thavā\",\n",
       "       'viśeṣaṇārthaṃ', 'prādhānyād', 'bahudharmāgrasaṃgrahāt',\n",
       "       'ekamāyatanaṃ', 'rūpamekaṃ', 'dharmākhyamucyate',\n",
       "       'dharmaskandhasahasrāṇi', 'yānyaśītiṃ', 'jagau', 'muniḥ', 'tāni',\n",
       "       'vāṅnāma', 'vetyeṣāṃ', 'rūpasaṃskārasaṃgrahaḥ', 'śāstrapramāṇā',\n",
       "       'ityeke', 'skandhādīnāṃ', 'kathaikaśaḥ', 'caritapratipakṣastu',\n",
       "       \"dharmaskandho'nuvarṇitaḥ\", \"tathānye'pi\", 'yathāyogaṃ',\n",
       "       'skandhāyatanadhātavaḥ', 'pratipādyā', 'yathokteṣu',\n",
       "       'sampradhārya', 'svalakṣaṇam', 'chidramākāśadhātvākhyam',\n",
       "       'ālokatamasī', 'kila', 'vijñānadhāturvijñānaṃ', 'sāsravaṃ',\n",
       "       'janmaniśrayāḥ', 'sanidarśana', \"eko'tra\", 'rūpaṃ', 'sapratighā',\n",
       "       'daśa', 'rūpiṇaḥ', 'avyākṛtā', 'aṣṭau', 'ta', 'evārūpaśabdakāḥ',\n",
       "       'tridhānye', 'kāmadhātvāptāḥ', 'sarve', 'rūpe', 'caturdaśa',\n",
       "       'vinā', 'gandharasaghrāṇajihvāvijñānadhātubhiḥ', 'ārūpyāptā',\n",
       "       'manodharmamanovijñānadhātavaḥ', 'sāsravānāsravā', 'ete', 'trayaḥ',\n",
       "       'śeṣāstu', 'sāsravāḥ', 'savitarkavicārā', 'hi', 'pañca',\n",
       "       'vijñānadhātavaḥ', 'antyāstrayastriprakārāḥ', 'śeṣā',\n",
       "       'ubhayavarjitāḥ', 'nirūpaṇānusmaraṇavikalpenāvikalpakāḥ', 'tau',\n",
       "       'prajñā', 'mānasī', 'vyagrā', 'smṛtiḥ', 'sarvaiva', 'mānasī',\n",
       "       'sapta', 'sālambanāścittadhātavaḥ', 'ardhaṃ', 'ca', 'dharmataḥ',\n",
       "       'navānupāttās', 'te', 'cāṣṭau', 'śabdaśca', 'anye', 'nava',\n",
       "       'dvidhā', 'spraṣṭavyaṃ', 'dvividhaṃ', 'śeṣā', 'rūpiṇo', 'nava',\n",
       "       'bhautikāḥ', 'dharmadhātvekadeśaśca', 'saṃcitā', 'daśa', 'rūpiṇaḥ',\n",
       "       'chinatti', 'chidyate', 'caiva', 'bāhyaṃ', 'dhātucatuṣṭayam',\n",
       "       'dahyate', 'tulayatyevaṃ', 'vivādo', 'dagdhṛtulyayoḥ',\n",
       "       'vipākajaupacayikāḥ', 'pañcādhyātmaṃ', 'vipākajaḥ', 'na', 'śabdaḥ',\n",
       "       'apratighā', 'aṣṭau', 'naiḥṣyandikavipākajāḥ', 'tridhānye',\n",
       "       'dravyavānekaḥ', 'kṣaṇikāḥ', 'paścimāstrayaḥ',\n",
       "       'cakṣurvijñānadhātvoḥ', 'syāt', 'pṛthak', 'lābhaḥ', 'sahāpi', 'ca',\n",
       "       'dvādaśādhyātmikāḥ', 'hitvā', 'rūpādīn', 'dharmasaṃjñakaḥ',\n",
       "       'sabhāgaḥ', 'tatsabhāgāśca', 'śeṣāḥ', 'yo', 'na', 'svakarmakṛt',\n",
       "       'daśa', 'bhāvanayā', 'heyāḥ', 'pañca', 'ca', 'antyāstrayastridhā',\n",
       "       'na', 'dṛṣṭiheyamakliṣṭaṃ', 'na', 'rūpaṃ', 'nāpyaṣaṣṭhajam',\n",
       "       'cakṣuśca', 'dharmadhātośca', 'pradeśo', 'dṛṣṭiḥ', 'aṣṭadhā',\n",
       "       'pañcavijñānasahajā', 'dhīrna', 'dṛṣṭiratīraṇāt', 'cakṣuḥ',\n",
       "       'paśyati', 'rūpāṇi', 'sabhāgaṃ', 'na', 'tadāśritam', 'vijñānaṃ',\n",
       "       'dṛśyate', 'rūpaṃ', 'na', 'kilāntaritaṃ', 'yataḥ', 'ubhābhyāmapi',\n",
       "       'cakṣurbhyāṃ', 'paśyati', 'vyaktadarśanāt',\n",
       "       \"cakṣuḥśrotramano'prāptaviṣayaṃ\", 'trayamanyathā',\n",
       "       'tribhirghrāṇādibhistulyaviṣayagrahaṇaṃ', 'matam',\n",
       "       \"caramasyāśrayo'tītaḥ\", 'pañcānāṃ', 'sahajaśca', 'taiḥ',\n",
       "       'tadvikāravikāritvād', 'āśrayāś', 'cakṣurādayaḥ',\n",
       "       \"ato'sādhāraṇatvāddhi\", 'vijñānaṃ', 'tairnirucyate', 'na',\n",
       "       'kāyasyādharaṃ', 'cakṣuḥ', 'ūrdhvaṃ', 'rūpaṃ', 'na', 'cakṣuṣaḥ',\n",
       "       'vijñānaṃ', 'ca', 'asya', 'rūpaṃ', 'tu', 'kāyasyobhe', 'ca',\n",
       "       'sarvataḥ', 'tathā', 'śrotraṃ', 'trayāṇāṃ', 'tu', 'sarvameva',\n",
       "       'svabhūmikam', 'kāyavijñānamadharasvabhūmi', 'aniyataṃ', 'manaḥ',\n",
       "       'pañca', 'bāhyā', 'dvivijñeyāḥ', 'nityā', 'dharmā', 'asaṃskṛtāḥ',\n",
       "       'dharmārdhamindriyaṃ', 'ye', 'ca', 'dvādaśādhyātmikāḥ', 'smṛtāḥ',\n",
       "       'abhidharmakośabhāṣye', 'dhātunirdeśo', 'nāma', 'prathamaṃ',\n",
       "       'kośasthānaṃ', 'samāptamiti'], dtype=object)"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "e857ca5d-f2ab-4ec6-962d-7e89f2f96306",
   "metadata": {},
   "outputs": [],
   "source": [
    "def getWords(filepath):\n",
    "    with open(filepath) as f:\n",
    "        lines = f.readlines()\n",
    "    df = pd.DataFrame(data=lines, columns=['text'])\n",
    "    df = df.replace(to_replace=r'\\/.*$', value='', regex=True)\n",
    "    df = df.replace(to_replace=r'^\\s*\\t*', value='', regex=True)\n",
    "    df = df.replace(to_replace=r\"'*\", value='', regex=True)\n",
    "    df = df['text'].str.split(expand=True)\n",
    "    new = df.values.flatten()\n",
    "    return np.extract(new != None, new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "id": "b5312885-d091-472f-a687-52629d32bc34",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['om' 'namo' 'buddhāya' 'yaḥ' 'sarvathā' 'sarvahatāndhakāraḥ'\n",
      " 'saṃsārapaṅkājjagadujjahāra' 'tasmai' 'namaskṛtya' 'yathārthaśāstre'\n",
      " 'śāstraṃ' 'pravakṣyāmyabhidharmakośam' 'prajñāmalā' 'sānucarābhidharmaḥ'\n",
      " 'tatprāptaye' 'yāpi' 'ca' 'yacca' 'śāstram' 'tasyārthatosmin'\n",
      " 'samanupraveśāt' 'sa' 'cāśrayo' 'syetyabhidharmakośam' 'dharmāṇāṃ'\n",
      " 'pravicayamantareṇa' 'nāsti' 'kleśānāṃ' 'yata' 'upaśāntayebhyupāyaḥ'\n",
      " 'kleśaiśca' 'bhramati' 'bhavārṇavetra' 'lokastaddhetorata' 'uditaḥ'\n",
      " 'kilaiṣa' 'śāstrā' 'sāsravānāsravā' 'dharmāḥ' 'saṃskṛtā' 'mārgavarjitāḥ'\n",
      " 'sāsravāḥ' 'āsravāsteṣu' 'yasmāt' 'samanuśerate' 'anāsravā' 'mārgasatyaṃ'\n",
      " 'trividhaṃ' 'cāpyasaṃskṛtam' 'ākāśaṃ' 'dvau' 'nirodhau' 'ca' 'tatrākāśam'\n",
      " 'anāvṛtiḥ' 'pratisaṃkhyānirodho' 'yo' 'visaṃyogaḥ' 'pṛthak' 'pṛthak'\n",
      " 'utpādātyantavighnonyo' 'nirodhopratisaṃkhyayā' 'te' 'punaḥ' 'saṃskṛtā'\n",
      " 'dharmā' 'rūpādiskandhapañcakam' 'ta' 'evādhvā' 'kathāvastu' 'saniḥsārāḥ'\n",
      " 'savastukāḥ' 'ye' 'sāsravā' 'upādānaskandhāste' 'saraṇā' 'api' 'duḥkhaṃ'\n",
      " 'samudayo' 'loko' 'dṛṣṭisthānaṃ' 'bhavaśca' 'te' 'rūpaṃ'\n",
      " 'pañcendriyāṇyarthāḥ' 'pañcāvijñaptireva' 'ca' 'tadvijñānāśrayā'\n",
      " 'rūpaprasādāścakṣurādayaḥ' 'rūpaṃ' 'dvidhā' 'viṃśatidhā'\n",
      " 'śabdastvaṣṭavidhaḥ' 'rasaḥ' 'ṣoḍhā' 'caturvidho' 'gandhaḥ'\n",
      " 'spṛśyamekādaśātmakam' 'vikṣiptācittakasyāpi' 'yonubandhaḥ' 'śubhāśubhaḥ'\n",
      " 'mahābhūtānyupādāya' 'sa' 'hyavijñaptirucyate' 'bhūtāni'\n",
      " 'pṛthividhāturaptejovāyudhātavaḥ' 'dhṛtyādikarmasaṃsiddhāḥ'\n",
      " 'kharasnehoṣṇateraṇāḥ' 'pṛthivī' 'varṇasaṃsthānamucyate' 'lokasaṃjñayā'\n",
      " 'āpastejaśca' 'vāyustu' 'dhātureva' 'tathāpi' 'ca' 'indriyārthāsta'\n",
      " 'eveṣṭā' 'daśāyatanadhātavaḥ' 'vedanānubhavaḥ' 'saṃjñā'\n",
      " 'nimittodgrahaṇātmikā' 'caturbhyonye' 'tu' 'saṃskāraskandhaḥ' 'ete'\n",
      " 'punastrayaḥ' 'dharmāyatanadhātvākhyāḥ' 'sahāvijñaptyasaṃskṛtaiḥ'\n",
      " 'vijñānaṃ' 'prativijñaptiḥ' 'mana' 'āyatanaṃ' 'ca' 'tat' 'dhātavaḥ'\n",
      " 'sapta' 'ca' 'matāḥ' 'ṣaḍ' 'vijñānānyatho' 'manaḥ' 'ṣaṇṇām'\n",
      " 'anantarātītaṃ' 'vijñānaṃ' 'yaddhi' 'tanmanaḥ'\n",
      " 'ṣaṣṭhāśrayaprasiddhyarthaṃ' 'dhātavo' 'ṣṭādaśa' 'smṛtāḥ' 'sarvasaṃgraha'\n",
      " 'ekena' 'skandhenāyatanena' 'ca' 'dhātunā' 'ca' 'svabhāvena'\n",
      " 'parabhāvaviyogataḥ' 'jātigocaravijñānasāmānyādekadhātutā' 'dvitvepi'\n",
      " 'cakṣurādīnāṃ' 'śobhārthaṃ' 'tu' 'dvayodbhavaḥ' 'rāśyāyadvāragotrārthāḥ'\n",
      " 'skandhāyatanadhātavaḥ' 'mohendriyarucitraidhāt' 'tisraḥ'\n",
      " 'skandhādideśanāḥ' 'vivādamūlasaṃsārahetutvāt' 'kramakāraṇāt'\n",
      " 'caittebhyo' 'vedanāsaṃjñe' 'pṛthakskandhau' 'niveśitau'\n",
      " 'skandheṣvasaṃskṛtaṃ' 'noktamarthāyogāt' 'kramaḥ' 'punaḥ'\n",
      " 'yathaudārikasaṃkleśabhājanādyarthadhātutaḥ' 'prāk' 'pañca'\n",
      " 'vārttamānārthyāt' 'bhautikārthyāccatuṣṭayam' 'dūrāśutaravṛttyānyat'\n",
      " 'yathāsthānaṃ' 'kramothavā' 'viśeṣaṇārthaṃ' 'prādhānyād'\n",
      " 'bahudharmāgrasaṃgrahāt' 'ekamāyatanaṃ' 'rūpamekaṃ' 'dharmākhyamucyate'\n",
      " 'dharmaskandhasahasrāṇi' 'yānyaśītiṃ' 'jagau' 'muniḥ' 'tāni' 'vāṅnāma'\n",
      " 'vetyeṣāṃ' 'rūpasaṃskārasaṃgrahaḥ' 'śāstrapramāṇā' 'ityeke'\n",
      " 'skandhādīnāṃ' 'kathaikaśaḥ' 'caritapratipakṣastu'\n",
      " 'dharmaskandhonuvarṇitaḥ' 'tathānyepi' 'yathāyogaṃ'\n",
      " 'skandhāyatanadhātavaḥ' 'pratipādyā' 'yathokteṣu' 'sampradhārya'\n",
      " 'svalakṣaṇam' 'chidramākāśadhātvākhyam' 'ālokatamasī' 'kila'\n",
      " 'vijñānadhāturvijñānaṃ' 'sāsravaṃ' 'janmaniśrayāḥ' 'sanidarśana' 'ekotra'\n",
      " 'rūpaṃ' 'sapratighā' 'daśa' 'rūpiṇaḥ' 'avyākṛtā' 'aṣṭau' 'ta'\n",
      " 'evārūpaśabdakāḥ' 'tridhānye' 'kāmadhātvāptāḥ' 'sarve' 'rūpe' 'caturdaśa'\n",
      " 'vinā' 'gandharasaghrāṇajihvāvijñānadhātubhiḥ' 'ārūpyāptā'\n",
      " 'manodharmamanovijñānadhātavaḥ' 'sāsravānāsravā' 'ete' 'trayaḥ' 'śeṣāstu'\n",
      " 'sāsravāḥ' 'savitarkavicārā' 'hi' 'pañca' 'vijñānadhātavaḥ'\n",
      " 'antyāstrayastriprakārāḥ' 'śeṣā' 'ubhayavarjitāḥ'\n",
      " 'nirūpaṇānusmaraṇavikalpenāvikalpakāḥ' 'tau' 'prajñā' 'mānasī' 'vyagrā'\n",
      " 'smṛtiḥ' 'sarvaiva' 'mānasī' 'sapta' 'sālambanāścittadhātavaḥ' 'ardhaṃ'\n",
      " 'ca' 'dharmataḥ' 'navānupāttās' 'te' 'cāṣṭau' 'śabdaśca' 'anye' 'nava'\n",
      " 'dvidhā' 'spraṣṭavyaṃ' 'dvividhaṃ' 'śeṣā' 'rūpiṇo' 'nava' 'bhautikāḥ'\n",
      " 'dharmadhātvekadeśaśca' 'saṃcitā' 'daśa' 'rūpiṇaḥ' 'chinatti' 'chidyate'\n",
      " 'caiva' 'bāhyaṃ' 'dhātucatuṣṭayam' 'dahyate' 'tulayatyevaṃ' 'vivādo'\n",
      " 'dagdhṛtulyayoḥ' 'vipākajaupacayikāḥ' 'pañcādhyātmaṃ' 'vipākajaḥ' 'na'\n",
      " 'śabdaḥ' 'apratighā' 'aṣṭau' 'naiḥṣyandikavipākajāḥ' 'tridhānye'\n",
      " 'dravyavānekaḥ' 'kṣaṇikāḥ' 'paścimāstrayaḥ' 'cakṣurvijñānadhātvoḥ' 'syāt'\n",
      " 'pṛthak' 'lābhaḥ' 'sahāpi' 'ca' 'dvādaśādhyātmikāḥ' 'hitvā' 'rūpādīn'\n",
      " 'dharmasaṃjñakaḥ' 'sabhāgaḥ' 'tatsabhāgāśca' 'śeṣāḥ' 'yo' 'na'\n",
      " 'svakarmakṛt' 'daśa' 'bhāvanayā' 'heyāḥ' 'pañca' 'ca'\n",
      " 'antyāstrayastridhā' 'na' 'dṛṣṭiheyamakliṣṭaṃ' 'na' 'rūpaṃ'\n",
      " 'nāpyaṣaṣṭhajam' 'cakṣuśca' 'dharmadhātośca' 'pradeśo' 'dṛṣṭiḥ' 'aṣṭadhā'\n",
      " 'pañcavijñānasahajā' 'dhīrna' 'dṛṣṭiratīraṇāt' 'cakṣuḥ' 'paśyati'\n",
      " 'rūpāṇi' 'sabhāgaṃ' 'na' 'tadāśritam' 'vijñānaṃ' 'dṛśyate' 'rūpaṃ' 'na'\n",
      " 'kilāntaritaṃ' 'yataḥ' 'ubhābhyāmapi' 'cakṣurbhyāṃ' 'paśyati'\n",
      " 'vyaktadarśanāt' 'cakṣuḥśrotramanoprāptaviṣayaṃ' 'trayamanyathā'\n",
      " 'tribhirghrāṇādibhistulyaviṣayagrahaṇaṃ' 'matam' 'caramasyāśrayotītaḥ'\n",
      " 'pañcānāṃ' 'sahajaśca' 'taiḥ' 'tadvikāravikāritvād' 'āśrayāś'\n",
      " 'cakṣurādayaḥ' 'atosādhāraṇatvāddhi' 'vijñānaṃ' 'tairnirucyate' 'na'\n",
      " 'kāyasyādharaṃ' 'cakṣuḥ' 'ūrdhvaṃ' 'rūpaṃ' 'na' 'cakṣuṣaḥ' 'vijñānaṃ'\n",
      " 'ca' 'asya' 'rūpaṃ' 'tu' 'kāyasyobhe' 'ca' 'sarvataḥ' 'tathā' 'śrotraṃ'\n",
      " 'trayāṇāṃ' 'tu' 'sarvameva' 'svabhūmikam' 'kāyavijñānamadharasvabhūmi'\n",
      " 'aniyataṃ' 'manaḥ' 'pañca' 'bāhyā' 'dvivijñeyāḥ' 'nityā' 'dharmā'\n",
      " 'asaṃskṛtāḥ' 'dharmārdhamindriyaṃ' 'ye' 'ca' 'dvādaśādhyātmikāḥ' 'smṛtāḥ'\n",
      " 'abhidharmakośabhāṣye' 'dhātunirdeśo' 'nāma' 'prathamaṃ' 'kośasthānaṃ'\n",
      " 'samāptamiti']\n"
     ]
    }
   ],
   "source": [
    "abh = getWords('./corpora/Abhidharmakosa.txt')\n",
    "print(abh)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "id": "b091eac5-1901-4367-b03d-96e9d4be8f45",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "om\n",
      "om\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(abh[0])\n",
    "print(abh[0][:3])\n",
    "abh[0][3:]\n",
    "len(abh[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "id": "bbf3bba9-0e58-4eb9-9411-84e89fb4a6cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extractPhonemes2(word, i, arr):\n",
    "    vowel = ['a','ā','e','è','i','o','ö','u','ū']\n",
    "    terminals = ['ṃ']\n",
    "    exceptions = ['r', 'ḥ', 'm']\n",
    "    if len(word) == 0:\n",
    "        return arr\n",
    "    if len(word) == i:\n",
    "        arr.append(word)\n",
    "        return arr\n",
    "    if word[i] in vowel:\n",
    "        #a,e\n",
    "        if i == 0:\n",
    "            arr.append(word[:i+1])\n",
    "            return extractPhonemes2(word[i+1:], 0,arr)\n",
    "        # next can be a terminal or exception\n",
    "        else:\n",
    "            return extractPhonemes2(word, i+1,arr)\n",
    "    #if 'ṃ'\n",
    "    if word[i] in terminals:\n",
    "        arr.append(word[:i+1])\n",
    "        return extractPhonemes2(word[i+1:], 0,arr)\n",
    "    #exception is a consonant if first\n",
    "    if word[i] in exceptions:\n",
    "        if i == 0:\n",
    "            return extractPhonemes2(word, i+1, arr)\n",
    "        else:\n",
    "            #exceptions are terminals if preceded by a vowel\n",
    "            if word[i-1] in vowel and i+1 < len(word) and word[i+1] in vowel:\n",
    "                arr.append(word[:i])\n",
    "                return extractPhonemes2(word[i:], 0,arr)\n",
    "            else:\n",
    "                if i+1 < len(word) and word[i+1] in vowel:\n",
    "                    return extractPhonemes2(word, i+1, arr)\n",
    "                else: \n",
    "                    arr.append(word[:i+1])\n",
    "                    return extractPhonemes2(word[i+1:], 0,arr)\n",
    "    #finally if it is a consonant\n",
    "    if i == 0:\n",
    "        return extractPhonemes2(word, i+1, arr)\n",
    "    else:\n",
    "        if word[i-1] in vowel:\n",
    "            arr.append(word[:i])\n",
    "            return extractPhonemes2(word[i:], 0,arr)\n",
    "        else:\n",
    "            return extractPhonemes2(word, i+1, arr)\n",
    "    return arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "id": "382b2941-5003-4df4-bad4-17e45e9bab4b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['o', 'm', 'na', 'mo', 'bu', 'ddhā', 'ya', 'yaḥ', 'sar', 'va', 'thā', 'sar', 'va', 'ha', 'tā', 'ndha', 'kā', 'raḥ', 'saṃ', 'sā', 'ra', 'pa', 'ṅkā', 'jja', 'ga', 'du', 'jja', 'hā', 'ra', 'ta', 'smai', 'na', 'ma', 'skṛtya', 'ya', 'thār', 'tha', 'śā', 'stre', 'śā', 'straṃ', 'pra', 'va', 'kṣyām', 'ya', 'bhi', 'dhar', 'ma', 'ko', 'śam', 'pra', 'jñā', 'ma', 'lā', 'sā', 'nu', 'ca', 'rā', 'bhi', 'dhar', 'maḥ', 'ta', 'tprā', 'pta', 'ye', 'yā', 'pi', 'ca', 'ya', 'cca', 'śā', 'stram', 'ta', 'syār', 'tha', 'to', 'smi', 'n', 'sa', 'ma', 'nu', 'pra', 've', 'śā', 't', 'sa', 'cā', 'śra', 'yo', 'sye', 'tya', 'bhi', 'dhar', 'ma', 'ko', 'śam', 'dhar', 'mā', 'ṇāṃ', 'pra', 'vi', 'ca', 'ya', 'ma', 'nta', 're', 'ṇa', 'nā', 'sti', 'kle', 'śā', 'nāṃ', 'ya', 'ta', 'u', 'pa', 'śā', 'nta', 'ye', 'bhyu', 'pā', 'yaḥ', 'kle', 'śai', 'śca', 'bhra', 'ma', 'ti', 'bha', 'vār', 'ṇa', 've', 'tra', 'lo', 'ka', 'sta', 'ddhe', 'to', 'ra', 'ta', 'u', 'di', 'taḥ', 'ki', 'lai', 'ṣa', 'śā', 'strā', 'sā', 'sra', 'vā', 'nā', 'sra', 'vā', 'dhar', 'māḥ', 'saṃ', 'skṛtā', 'mār', 'ga', 'var', 'ji', 'tāḥ', 'sā', 'sra', 'vāḥ', 'ā', 'sra', 'vā', 'ste', 'ṣu', 'ya', 'smā', 't', 'sa', 'ma', 'nu', 'śe', 'ra', 'te', 'a', 'nā', 'sra', 'vā', 'mār', 'ga', 'sa', 'tyaṃ', 'tri', 'vi', 'dhaṃ', 'cā', 'pya', 'saṃ', 'skṛtam', 'ā', 'kā', 'śaṃ', 'dvau', 'ni', 'ro', 'dhau', 'ca', 'ta', 'trā', 'kā', 'śam', 'a', 'nā', 'vṛtiḥ', 'pra', 'ti', 'saṃ', 'khyā', 'ni', 'ro', 'dho', 'yo', 'vi', 'saṃ', 'yo', 'gaḥ', 'pṛtha', 'k', 'pṛtha', 'k', 'u', 'tpā', 'dā', 'tya', 'nta', 'vi', 'ghno', 'nyo', 'ni', 'ro', 'dho', 'pra', 'ti', 'saṃ', 'khya', 'yā', 'te', 'pu', 'naḥ', 'saṃ', 'skṛtā', 'dhar', 'mā', 'rū', 'pā', 'di', 'ska', 'ndha', 'pa', 'ñca', 'kam', 'ta', 'e', 'vā', 'dhvā', 'ka', 'thā', 'va', 'stu', 'sa', 'niḥ', 'sā', 'rāḥ', 'sa', 'va', 'stu', 'kāḥ', 'ye', 'sā', 'sra', 'vā', 'u', 'pā', 'dā', 'na', 'ska', 'ndhā', 'ste', 'sa', 'ra', 'ṇā', 'a', 'pi', 'duḥ', 'khaṃ', 'sa', 'mu', 'da', 'yo', 'lo', 'ko', 'dṛṣṭi', 'sthā', 'naṃ', 'bha', 'va', 'śca', 'te', 'rū', 'paṃ', 'pa', 'ñce', 'ndri', 'yā', 'ṇyar', 'thāḥ', 'pa', 'ñcā', 'vi', 'jña', 'pti', 're', 'va', 'ca', 'ta', 'dvi', 'jñā', 'nā', 'śra', 'yā', 'rū', 'pa', 'pra', 'sā', 'dā', 'śca', 'kṣu', 'rā', 'da', 'yaḥ', 'rū', 'paṃ', 'dvi', 'dhā', 'viṃ', 'śa', 'ti', 'dhā', 'śa', 'bda', 'stva', 'ṣṭa', 'vi', 'dhaḥ', 'ra', 'saḥ', 'ṣo', 'ḍhā', 'ca', 'tur', 'vi', 'dho', 'ga', 'ndhaḥ', 'spṛśya', 'me', 'kā', 'da', 'śā', 'tma', 'kam', 'vi', 'kṣi', 'ptā', 'ci', 'tta', 'ka', 'syā', 'pi', 'yo', 'nu', 'ba', 'ndhaḥ', 'śu', 'bhā', 'śu', 'bhaḥ', 'ma', 'hā', 'bhū', 'tā', 'nyu', 'pā', 'dā', 'ya', 'sa', 'hya', 'vi', 'jña', 'pti', 'ru', 'cya', 'te', 'bhū', 'tā', 'ni', 'pṛthi', 'vi', 'dhā', 'tu', 'ra', 'pte', 'jo', 'vā', 'yu', 'dhā', 'ta', 'vaḥ', 'dhṛtyā', 'di', 'kar', 'ma', 'saṃ', 'si', 'ddhāḥ', 'kha', 'ra', 'sne', 'ho', 'ṣṇa', 'te', 'ra', 'ṇāḥ', 'pṛthi', 'vī', 'var', 'ṇa', 'saṃ', 'sthā', 'na', 'mu', 'cya', 'te', 'lo', 'ka', 'saṃ', 'jña', 'yā', 'ā', 'pa', 'ste', 'ja', 'śca', 'vā', 'yu', 'stu', 'dhā', 'tu', 're', 'va', 'ta', 'thā', 'pi', 'ca', 'i', 'ndri', 'yār', 'thā', 'sta', 'e', 've', 'ṣṭā', 'da', 'śā', 'ya', 'ta', 'na', 'dhā', 'ta', 'vaḥ', 've', 'da', 'nā', 'nu', 'bha', 'vaḥ', 'saṃ', 'jñā', 'ni', 'mi', 'tto', 'dgra', 'ha', 'ṇā', 'tmi', 'kā', 'ca', 'tur', 'bhyo', 'nye', 'tu', 'saṃ', 'skā', 'ra', 'ska', 'ndhaḥ', 'e', 'te', 'pu', 'na', 'stra', 'yaḥ', 'dhar', 'mā', 'ya', 'ta', 'na', 'dhā', 'tvā', 'khyāḥ', 'sa', 'hā', 'vi', 'jña', 'ptya', 'saṃ', 'skṛtaiḥ', 'vi', 'jñā', 'naṃ', 'pra', 'ti', 'vi', 'jña', 'ptiḥ', 'ma', 'na', 'ā', 'ya', 'ta', 'naṃ', 'ca', 'ta', 't', 'dhā', 'ta', 'vaḥ', 'sa', 'pta', 'ca', 'ma', 'tāḥ', 'ṣa', 'ḍ', 'vi', 'jñā', 'nā', 'nya', 'tho', 'ma', 'naḥ', 'ṣa', 'ṇṇām', 'a', 'na', 'nta', 'rā', 'tītaṃ', 'vi', 'jñā', 'naṃ', 'ya', 'ddhi', 'ta', 'nma', 'naḥ', 'ṣa', 'ṣṭhā', 'śra', 'ya', 'pra', 'si', 'ddhyar', 'thaṃ', 'dhā', 'ta', 'vo', 'ṣṭā', 'da', 'śa', 'sm', 'ṛtāḥ', 'sar', 'va', 'saṃ', 'gra', 'ha', 'e', 'ke', 'na', 'ska', 'ndhe', 'nā', 'ya', 'ta', 'ne', 'na', 'ca', 'dhā', 'tu', 'nā', 'ca', 'sva', 'bhā', 've', 'na', 'pa', 'ra', 'bhā', 'va', 'vi', 'yo', 'ga', 'taḥ', 'jā', 'ti', 'go', 'ca', 'ra', 'vi', 'jñā', 'na', 'sā', 'mā', 'nyā', 'de', 'ka', 'dhā', 'tu', 'tā', 'dvi', 'tve', 'pi', 'ca', 'kṣu', 'rā', 'dīnāṃ', 'śo', 'bhār', 'thaṃ', 'tu', 'dva', 'yo', 'dbha', 'vaḥ', 'rā', 'śyā', 'ya', 'dvā', 'ra', 'go', 'trār', 'thāḥ', 'ska', 'ndhā', 'ya', 'ta', 'na', 'dhā', 'ta', 'vaḥ', 'mo', 'he', 'ndri', 'ya', 'ru', 'ci', 'trai', 'dhā', 't', 'ti', 'sraḥ', 'ska', 'ndhā', 'di', 'de', 'śa', 'nāḥ', 'vi', 'vā', 'da', 'mū', 'la', 'saṃ', 'sā', 'ra', 'he', 'tu', 'tvā', 't', 'kra', 'ma', 'kā', 'ra', 'ṇā', 't', 'cai', 'tte', 'bhyo', 've', 'da', 'nā', 'saṃ', 'jñe', 'pṛtha', 'kska', 'ndhau', 'ni', 've', 'śi', 'tau', 'ska', 'ndhe', 'ṣva', 'saṃ', 'skṛtaṃ', 'no', 'kta', 'mar', 'thā', 'yo', 'gā', 't', 'kra', 'maḥ', 'pu', 'naḥ', 'ya', 'thau', 'dā', 'ri', 'ka', 'saṃ', 'kle', 'śa', 'bhā', 'ja', 'nā', 'dyar', 'tha', 'dhā', 'tu', 'taḥ', 'prā', 'k', 'pa', 'ñca', 'vār', 'tta', 'mā', 'nār', 'thyā', 't', 'bhau', 'ti', 'kār', 'thyā', 'cca', 'tu', 'ṣṭa', 'yam', 'dū', 'rā', 'śu', 'ta', 'ra', 'vṛttyā', 'nya', 't', 'ya', 'thā', 'sthā', 'naṃ', 'kra', 'mo', 'tha', 'vā', 'vi', 'śe', 'ṣa', 'ṇār', 'thaṃ', 'prā', 'dhā', 'nyā', 'd', 'ba', 'hu', 'dhar', 'mā', 'gra', 'saṃ', 'gra', 'hā', 't', 'e', 'ka', 'mā', 'ya', 'ta', 'naṃ', 'rū', 'pa', 'me', 'kaṃ', 'dhar', 'mā', 'khya', 'mu', 'cya', 'te', 'dhar', 'ma', 'ska', 'ndha', 'sa', 'ha', 'srā', 'ṇi', 'yā', 'nya', 'śītiṃ', 'ja', 'gau', 'mu', 'niḥ', 'tā', 'ni', 'vā', 'ṅnā', 'ma', 've', 'tye', 'ṣāṃ', 'rū', 'pa', 'saṃ', 'skā', 'ra', 'saṃ', 'gra', 'haḥ', 'śā', 'stra', 'pra', 'mā', 'ṇā', 'i', 'tye', 'ke', 'ska', 'ndhā', 'dīnāṃ', 'ka', 'thai', 'ka', 'śaḥ', 'ca', 'ri', 'ta', 'pra', 'ti', 'pa', 'kṣa', 'stu', 'dhar', 'ma', 'ska', 'ndho', 'nu', 'var', 'ṇi', 'taḥ', 'ta', 'thā', 'nye', 'pi', 'ya', 'thā', 'yo', 'gaṃ', 'ska', 'ndhā', 'ya', 'ta', 'na', 'dhā', 'ta', 'vaḥ', 'pra', 'ti', 'pā', 'dyā', 'ya', 'tho', 'kte', 'ṣu', 'sam', 'pra', 'dhār', 'ya', 'sva', 'la', 'kṣa', 'ṇam', 'chi', 'dra', 'mā', 'kā', 'śa', 'dhā', 'tvā', 'khyam', 'ā', 'lo', 'ka', 'ta', 'ma', 'sī', 'ki', 'la', 'vi', 'jñā', 'na', 'dhā', 'tur', 'vi', 'jñā', 'naṃ', 'sā', 'sra', 'vaṃ', 'ja', 'nma', 'ni', 'śra', 'yāḥ', 'sa', 'ni', 'dar', 'śa', 'na', 'e', 'ko', 'tra', 'rū', 'paṃ', 'sa', 'pra', 'ti', 'ghā', 'da', 'śa', 'rū', 'pi', 'ṇaḥ', 'a', 'vyā', 'kṛtā', 'a', 'ṣṭau', 'ta', 'e', 'vā', 'rū', 'pa', 'śa', 'bda', 'kāḥ', 'tri', 'dhā', 'nye', 'kā', 'ma', 'dhā', 'tvā', 'ptāḥ', 'sar', 've', 'rū', 'pe', 'ca', 'tur', 'da', 'śa', 'vi', 'nā', 'ga', 'ndha', 'ra', 'sa', 'ghrā', 'ṇa', 'ji', 'hvā', 'vi', 'jñā', 'na', 'dhā', 'tu', 'bhiḥ', 'ā', 'rū', 'pyā', 'ptā', 'ma', 'no', 'dhar', 'ma', 'ma', 'no', 'vi', 'jñā', 'na', 'dhā', 'ta', 'vaḥ', 'sā', 'sra', 'vā', 'nā', 'sra', 'vā', 'e', 'te', 'tra', 'yaḥ', 'śe', 'ṣā', 'stu', 'sā', 'sra', 'vāḥ', 'sa', 'vi', 'tar', 'ka', 'vi', 'cā', 'rā', 'hi', 'pa', 'ñca', 'vi', 'jñā', 'na', 'dhā', 'ta', 'vaḥ', 'a', 'ntyā', 'stra', 'ya', 'stri', 'pra', 'kā', 'rāḥ', 'śe', 'ṣā', 'u', 'bha', 'ya', 'var', 'ji', 'tāḥ', 'ni', 'rū', 'pa', 'ṇā', 'nu', 'sma', 'ra', 'ṇa', 'vi', 'ka', 'lpe', 'nā', 'vi', 'ka', 'lpa', 'kāḥ', 'tau', 'pra', 'jñā', 'mā', 'na', 'sī', 'vya', 'grā', 'sm', 'ṛtiḥ', 'sar', 'vai', 'va', 'mā', 'na', 'sī', 'sa', 'pta', 'sā', 'lam', 'ba', 'nā', 'ści', 'tta', 'dhā', 'ta', 'vaḥ', 'a', 'rdhaṃ', 'ca', 'dhar', 'ma', 'taḥ', 'na', 'vā', 'nu', 'pā', 'ttā', 's', 'te', 'cā', 'ṣṭau', 'śa', 'bda', 'śca', 'a', 'nye', 'na', 'va', 'dvi', 'dhā', 'spra', 'ṣṭa', 'vyaṃ', 'dvi', 'vi', 'dhaṃ', 'śe', 'ṣā', 'rū', 'pi', 'ṇo', 'na', 'va', 'bhau', 'ti', 'kāḥ', 'dhar', 'ma', 'dhā', 'tve', 'ka', 'de', 'śa', 'śca', 'saṃ', 'ci', 'tā', 'da', 'śa', 'rū', 'pi', 'ṇaḥ', 'chi', 'na', 'tti', 'chi', 'dya', 'te', 'cai', 'va', 'bā', 'hyaṃ', 'dhā', 'tu', 'ca', 'tu', 'ṣṭa', 'yam', 'da', 'hya', 'te', 'tu', 'la', 'ya', 'tye', 'vaṃ', 'vi', 'vā', 'do', 'da', 'gdhṛtu', 'lya', 'yoḥ', 'vi', 'pā', 'ka', 'jau', 'pa', 'ca', 'yi', 'kāḥ', 'pa', 'ñcā', 'dhyā', 'tmaṃ', 'vi', 'pā', 'ka', 'jaḥ', 'na', 'śa', 'bdaḥ', 'a', 'pra', 'ti', 'ghā', 'a', 'ṣṭau', 'naiḥ', 'ṣya', 'ndi', 'ka', 'vi', 'pā', 'ka', 'jāḥ', 'tri', 'dhā', 'nye', 'dra', 'vya', 'vā', 'ne', 'kaḥ', 'kṣa', 'ṇi', 'kāḥ', 'pa', 'ści', 'mā', 'stra', 'yaḥ', 'ca', 'kṣur', 'vi', 'jñā', 'na', 'dhā', 'tvoḥ', 'syā', 't', 'pṛtha', 'k', 'lā', 'bhaḥ', 'sa', 'hā', 'pi', 'ca', 'dvā', 'da', 'śā', 'dhyā', 'tmi', 'kāḥ', 'hi', 'tvā', 'rū', 'pā', 'dīn', 'dhar', 'ma', 'saṃ', 'jña', 'kaḥ', 'sa', 'bhā', 'gaḥ', 'ta', 'tsa', 'bhā', 'gā', 'śca', 'śe', 'ṣāḥ', 'yo', 'na', 'sva', 'kar', 'ma', 'kṛt', 'da', 'śa', 'bhā', 'va', 'na', 'yā', 'he', 'yāḥ', 'pa', 'ñca', 'ca', 'a', 'ntyā', 'stra', 'ya', 'stri', 'dhā', 'na', 'dṛṣṭi', 'he', 'ya', 'ma', 'kli', 'ṣṭaṃ', 'na', 'rū', 'paṃ', 'nā', 'pya', 'ṣa', 'ṣṭha', 'jam', 'ca', 'kṣu', 'śca', 'dhar', 'ma', 'dhā', 'to', 'śca', 'pra', 'de', 'śo', 'dṛṣṭiḥ', 'a', 'ṣṭa', 'dhā', 'pa', 'ñca', 'vi', 'jñā', 'na', 'sa', 'ha', 'jā', 'dhīr', 'na', 'dṛṣṭi', 'ra', 'tīra', 'ṇā', 't', 'ca', 'kṣuḥ', 'pa', 'śya', 'ti', 'rū', 'pā', 'ṇi', 'sa', 'bhā', 'gaṃ', 'na', 'ta', 'dā', 'śri', 'tam', 'vi', 'jñā', 'naṃ', 'dṛśya', 'te', 'rū', 'paṃ', 'na', 'ki', 'lā', 'nta', 'ri', 'taṃ', 'ya', 'taḥ', 'u', 'bhā', 'bhyā', 'ma', 'pi', 'ca', 'kṣur', 'bhyāṃ', 'pa', 'śya', 'ti', 'vya', 'kta', 'dar', 'śa', 'nā', 't', 'ca', 'kṣuḥ', 'śro', 'tra', 'ma', 'no', 'prā', 'pta', 'vi', 'ṣa', 'yaṃ', 'tra', 'ya', 'ma', 'nya', 'thā', 'tri', 'bhir', 'ghrā', 'ṇā', 'di', 'bhi', 'stu', 'lya', 'vi', 'ṣa', 'ya', 'gra', 'ha', 'ṇaṃ', 'ma', 'tam', 'ca', 'ra', 'ma', 'syā', 'śra', 'yo', 'tītaḥ', 'pa', 'ñcā', 'nāṃ', 'sa', 'ha', 'ja', 'śca', 'taiḥ', 'ta', 'dvi', 'kā', 'ra', 'vi', 'kā', 'ri', 'tvā', 'd', 'ā', 'śra', 'yā', 'ś', 'ca', 'kṣu', 'rā', 'da', 'yaḥ', 'a', 'to', 'sā', 'dhā', 'ra', 'ṇa', 'tvā', 'ddhi', 'vi', 'jñā', 'naṃ', 'tair', 'ni', 'ru', 'cya', 'te', 'na', 'kā', 'ya', 'syā', 'dha', 'raṃ', 'ca', 'kṣuḥ', 'ū', 'rdhvaṃ', 'rū', 'paṃ', 'na', 'ca', 'kṣu', 'ṣaḥ', 'vi', 'jñā', 'naṃ', 'ca', 'a', 'sya', 'rū', 'paṃ', 'tu', 'kā', 'ya', 'syo', 'bhe', 'ca', 'sar', 'va', 'taḥ', 'ta', 'thā', 'śro', 'traṃ', 'tra', 'yā', 'ṇāṃ', 'tu', 'sar', 'va', 'me', 'va', 'sva', 'bhū', 'mi', 'kam', 'kā', 'ya', 'vi', 'jñā', 'na', 'ma', 'dha', 'ra', 'sva', 'bhū', 'mi', 'a', 'ni', 'ya', 'taṃ', 'ma', 'naḥ', 'pa', 'ñca', 'bā', 'hyā', 'dvi', 'vi', 'jñe', 'yāḥ', 'ni', 'tyā', 'dhar', 'mā', 'a', 'saṃ', 'skṛtāḥ', 'dhar', 'mār', 'dha', 'mi', 'ndri', 'yaṃ', 'ye', 'ca', 'dvā', 'da', 'śā', 'dhyā', 'tmi', 'kāḥ', 'sm', 'ṛtāḥ', 'a', 'bhi', 'dhar', 'ma', 'ko', 'śa', 'bhā', 'ṣye', 'dhā', 'tu', 'nir', 'de', 'śo', 'nā', 'ma', 'pra', 'tha', 'maṃ', 'ko', 'śa', 'sthā', 'naṃ', 'sa', 'mā', 'pta', 'mi', 'ti']\n"
     ]
    }
   ],
   "source": [
    "all = []\n",
    "for i in range(len(abh)):\n",
    "    arr = []\n",
    "    all = all + extractPhonemes2(abh[i], 0, arr)\n",
    "print(all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "id": "82f870d0-3bdd-440e-ad1a-d664882dd92b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1614"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "id": "bcdac04a-d6c1-413b-904a-b5e0877cfed8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "411"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b = set(all)\n",
    "len(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "id": "17f3d78a-d86f-43e5-bfa5-192a5e94d0db",
   "metadata": {},
   "outputs": [],
   "source": [
    "c = list(b)+ all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "id": "9044ce9f-6ff3-424a-bb24-b32d46cfc043",
   "metadata": {},
   "outputs": [],
   "source": [
    "c = set(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "id": "d0943f32-c21b-4f35-a781-39ee7678943c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "411"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8eba013f-3144-400c-99ae-d343bd71a7a9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}